diff --git a/build_scripts/update_redis_server.py b/build_scripts/update_redis_server.py index 6a708a3..ac8e306 100755 --- a/build_scripts/update_redis_server.py +++ b/build_scripts/update_redis_server.py @@ -7,7 +7,7 @@ import tempfile -url = 'http://download.redis.io/releases/redis-stable.tar.gz' +url = 'http://download.redis.io/releases/redis-6.2.6.tar.gz' if __name__ == "__main__": diff --git a/redis.submodule/.github/ISSUE_TEMPLATE/bug_report.md b/redis.submodule/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..00f21a8 --- /dev/null +++ b/redis.submodule/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,24 @@ +--- +name: Bug report +about: Help us improve Redis by reporting a bug +title: '[BUG]' +labels: '' +assignees: '' + +--- + +**Describe the bug** + +A short description of the bug. + +**To reproduce** + +Steps to reproduce the behavior and/or a minimal code sample. + +**Expected behavior** + +A description of what you expected to happen. + +**Additional information** + +Any additional information that is relevant to the problem. diff --git a/redis.submodule/.github/ISSUE_TEMPLATE/crash_report.md b/redis.submodule/.github/ISSUE_TEMPLATE/crash_report.md new file mode 100644 index 0000000..c608ccd --- /dev/null +++ b/redis.submodule/.github/ISSUE_TEMPLATE/crash_report.md @@ -0,0 +1,20 @@ +--- +name: Crash report +about: Submit a crash report +title: '[CRASH]' +labels: '' +assignees: '' + +--- + +**Crash report** + +Paste the complete crash log between the quotes below. Please include a few lines from the log preceding the crash report to provide some context. + +``` +``` + +**Aditional information** + +1. OS distribution and version +2. Steps to reproduce (if any) diff --git a/redis.submodule/.github/ISSUE_TEMPLATE/feature_request.md b/redis.submodule/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..732dc64 --- /dev/null +++ b/redis.submodule/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,24 @@ +--- +name: Feature request +about: Suggest a feature for Redis +title: '[NEW]' +labels: '' +assignees: '' + +--- + +**The problem/use-case that the feature addresses** + +A description of the problem that the feature will solve, or the use-case with which the feature will be used. + +**Description of the feature** + +A description of what you want to happen. + +**Alternatives you've considered** + +Any alternative solutions or features you've considered, including references to existing open and closed feature requests in this repository. + +**Additional information** + +Any additional information that is relevant to the feature request. diff --git a/redis.submodule/.github/ISSUE_TEMPLATE/other_stuff.md b/redis.submodule/.github/ISSUE_TEMPLATE/other_stuff.md new file mode 100644 index 0000000..e82cf00 --- /dev/null +++ b/redis.submodule/.github/ISSUE_TEMPLATE/other_stuff.md @@ -0,0 +1,8 @@ +--- +name: Other +about: Can't find the right issue type? Use this one! +title: '' +labels: '' +assignees: '' + +--- diff --git a/redis.submodule/.github/ISSUE_TEMPLATE/question.md b/redis.submodule/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000..6acde0d --- /dev/null +++ b/redis.submodule/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,21 @@ +--- +name: Question +about: Ask the Redis developers +title: '[QUESTION]' +labels: '' +assignees: '' + +--- + +Please keep in mind that this issue tracker should be used for reporting bugs or proposing improvements to the Redis server. + +Generally, questions about using Redis should be directed to the [community](https://redis.io/community): + +* [the mailing list](https://groups.google.com/forum/#!forum/redis-db) +* [the `redis` tag at StackOverflow](http://stackoverflow.com/questions/tagged/redis) +* [/r/redis subreddit](http://www.reddit.com/r/redis) +* [the irc channel #redis](http://webchat.freenode.net/?channels=redis) on freenode + +It is also possible that your question was already asked here, so please do a quick issues search before submitting. Lastly, if your question is about one of Redis' [clients](https://redis.io/clients), you may to contact your client's developers for help. + +That said, please feel free to replace all this with your question :) \ No newline at end of file diff --git a/redis.submodule/.github/workflows/ci.yml b/redis.submodule/.github/workflows/ci.yml new file mode 100644 index 0000000..39b332a --- /dev/null +++ b/redis.submodule/.github/workflows/ci.yml @@ -0,0 +1,60 @@ +name: CI + +on: [push, pull_request] + +jobs: + + test-ubuntu-latest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: make + # Fail build if there are warnings + # build with TLS just for compilation coverage + run: make REDIS_CFLAGS='-Werror' BUILD_TLS=yes + - name: test + run: | + sudo apt-get install tcl8.6 + ./runtest --verbose + - name: module api test + run: ./runtest-moduleapi --verbose + + build-ubuntu-old: + runs-on: ubuntu-16.04 + steps: + - uses: actions/checkout@v2 + - name: make + run: make REDIS_CFLAGS='-Werror' + + build-macos-latest: + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + - name: make + run: make REDIS_CFLAGS='-Werror' + + build-32bit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: make + run: | + sudo apt-get update && sudo apt-get install libc6-dev-i386 + make REDIS_CFLAGS='-Werror' 32bit + + build-libc-malloc: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: make + run: make REDIS_CFLAGS='-Werror' MALLOC=libc + + build-centos7-jemalloc: + runs-on: ubuntu-latest + container: centos:7 + steps: + - uses: actions/checkout@v2 + - name: make + run: | + yum -y install gcc make + make REDIS_CFLAGS='-Werror' diff --git a/redis.submodule/.github/workflows/daily.yml b/redis.submodule/.github/workflows/daily.yml new file mode 100644 index 0000000..432971a --- /dev/null +++ b/redis.submodule/.github/workflows/daily.yml @@ -0,0 +1,308 @@ +name: Daily + +on: + pull_request: + branches: + # any PR to a release branch. + - '[0-9].[0-9]' + schedule: + - cron: '0 0 * * *' + +jobs: + + test-ubuntu-jemalloc: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: make REDIS_CFLAGS='-Werror -DREDIS_TEST' + - name: test + run: | + sudo apt-get install tcl8.6 + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + - name: unittest + run: ./src/redis-server test all + + test-ubuntu-libc-malloc: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: make MALLOC=libc + - name: test + run: | + sudo apt-get install tcl8.6 + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + + test-ubuntu-no-malloc-usable-size: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: make MALLOC=libc CFLAGS=-DNO_MALLOC_USABLE_SIZE + - name: test + run: | + sudo apt-get install tcl8.6 + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + + test-ubuntu-32bit: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: | + sudo apt-get update && sudo apt-get install libc6-dev-i386 + make 32bit REDIS_CFLAGS='-Werror -DREDIS_TEST' + - name: test + run: | + sudo apt-get install tcl8.6 + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: | + make -C tests/modules 32bit # the script below doesn't have an argument, we must build manually ahead of time + ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + - name: unittest + run: ./src/redis-server test all + + test-ubuntu-tls: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: | + make BUILD_TLS=yes + - name: test + run: | + sudo apt-get install tcl8.6 tcl-tls + ./utils/gen-test-certs.sh + ./runtest --accurate --verbose --tls --dump-logs + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: | + ./runtest-moduleapi --verbose --tls + ./runtest-moduleapi --verbose + - name: sentinel tests + run: | + ./runtest-sentinel --tls + ./runtest-sentinel + - name: cluster tests + run: | + ./runtest-cluster --tls + ./runtest-cluster + + test-ubuntu-io-threads: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: | + make + - name: test + run: | + sudo apt-get install tcl8.6 tcl-tls + ./runtest --config io-threads 4 --config io-threads-do-reads yes --accurate --verbose --tags network --dump-logs + - name: cluster tests + run: | + ./runtest-cluster --config io-threads 4 --config io-threads-do-reads yes + + test-valgrind: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: make valgrind REDIS_CFLAGS='-Werror -DREDIS_TEST' + - name: test + run: | + sudo apt-get update + sudo apt-get install tcl8.6 valgrind -y + ./runtest --valgrind --verbose --clients 1 --tags -large-memory --dump-logs + - name: module api test + run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 + - name: unittest + run: | + valgrind --track-origins=yes --suppressions=./src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full --log-file=err.txt ./src/redis-server test all + if grep -q 0x err.txt; then cat err.txt; exit 1; fi + + test-valgrind-no-malloc-usable-size: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: make valgrind CFLAGS="-DNO_MALLOC_USABLE_SIZE" + - name: test + run: | + sudo apt-get update + sudo apt-get install tcl8.6 valgrind -y + ./runtest --valgrind --verbose --clients 1 --tags -large-memory --dump-logs + - name: module api test + run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 + + test-centos7-jemalloc: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + container: centos:7 + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: | + yum -y install gcc make + make + - name: test + run: | + yum -y install which tcl + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + + test-centos7-tls: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + container: centos:7 + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: | + yum -y install centos-release-scl epel-release + yum -y install devtoolset-7 openssl-devel openssl + scl enable devtoolset-7 "make BUILD_TLS=yes" + - name: test + run: | + yum -y install tcl tcltls + ./utils/gen-test-certs.sh + ./runtest --accurate --verbose --tls --dump-logs + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: | + ./runtest-moduleapi --verbose --tls + ./runtest-moduleapi --verbose + - name: sentinel tests + run: | + ./runtest-sentinel --tls + ./runtest-sentinel + - name: cluster tests + run: | + ./runtest-cluster --tls + ./runtest-cluster + + test-macos-latest: + runs-on: macos-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: make + run: make + - name: test + run: | + ./runtest --accurate --verbose --no-latency --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + + test-freebsd: + runs-on: macos-latest + if: github.repository == 'redis/redis' + timeout-minutes: 14400 + steps: + - uses: actions/checkout@v2 + - name: test + uses: vmactions/freebsd-vm@v0.1.4 + with: + usesh: true + sync: rsync + prepare: pkg install -y bash gmake lang/tcl86 + run: > + gmake && + ./runtest --accurate --verbose --no-latency --tags -large-memory --dump-logs && + MAKE=gmake ./runtest-moduleapi --verbose && + ./runtest-sentinel && + ./runtest-cluster + + test-alpine-jemalloc: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + container: alpine:latest + steps: + - uses: actions/checkout@v2 + - name: make + run: | + apk add build-base + make REDIS_CFLAGS='-Werror' + - name: test + run: | + apk add tcl procps + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster + + test-alpine-libc-malloc: + runs-on: ubuntu-latest + if: github.repository == 'redis/redis' + container: alpine:latest + steps: + - uses: actions/checkout@v2 + - name: make + run: | + apk add build-base + make REDIS_CFLAGS='-Werror' USE_JEMALLOC=no CFLAGS=-DUSE_MALLOC_USABLE_SIZE + - name: test + run: | + apk add tcl procps + ./runtest --accurate --verbose --dump-logs + - name: module api test + run: ./runtest-moduleapi --verbose + - name: sentinel tests + run: ./runtest-sentinel + - name: cluster tests + run: ./runtest-cluster diff --git a/redis.submodule/.gitignore b/redis.submodule/.gitignore index a188cfc..dd9d4b8 100644 --- a/redis.submodule/.gitignore +++ b/redis.submodule/.gitignore @@ -1,5 +1,8 @@ .*.swp *.o +*.xo +*.so +*.d *.log dump.rdb redis-benchmark @@ -24,7 +27,14 @@ src/nodes.conf deps/lua/src/lua deps/lua/src/luac deps/lua/src/liblua.a +tests/tls/* .make-* .prerequisites *.dSYM Makefile.dep +.vscode/* +.idea/* +.ccls +.ccls-cache/* +compile_commands.json +redis.code-workspace diff --git a/redis.submodule/00-RELEASENOTES b/redis.submodule/00-RELEASENOTES index 285ae84..62d1def 100644 --- a/redis.submodule/00-RELEASENOTES +++ b/redis.submodule/00-RELEASENOTES @@ -1,4 +1,4 @@ -Redis 5.0 release notes +Redis 6.2 release notes ======================= -------------------------------------------------------------------------------- @@ -12,3172 +12,681 @@ SECURITY: There are security fixes in the release. -------------------------------------------------------------------------------- ================================================================================ -Redis 5.0.6 Released Wed Sep 25 12:33:56 CEST 2019 +Redis 6.2.6 Released Mon Oct 4 12:00:00 IDT 2021 ================================================================================ -Upgrade urgency CRITICAL: Only in case of exposed instances to untrusted users. +Upgrade urgency: SECURITY, contains fixes to security issues. + +Security Fixes: +* (CVE-2021-41099) Integer to heap buffer overflow handling certain string + commands and network payloads, when proto-max-bulk-len is manually configured + to a non-default, very large value [reported by yiyuaner]. +* (CVE-2021-32762) Integer to heap buffer overflow issue in redis-cli and + redis-sentinel parsing large multi-bulk replies on some older and less common + platforms [reported by Microsoft Vulnerability Research]. +* (CVE-2021-32687) Integer to heap buffer overflow with intsets, when + set-max-intset-entries is manually configured to a non-default, very large + value [reported by Pawel Wieczorkiewicz, AWS]. +* (CVE-2021-32675) Denial Of Service when processing RESP request payloads with + a large number of elements on many connections. +* (CVE-2021-32672) Random heap reading issue with Lua Debugger [reported by + Meir Shpilraien]. +* (CVE-2021-32628) Integer to heap buffer overflow handling ziplist-encoded + data types, when configuring a large, non-default value for + hash-max-ziplist-entries, hash-max-ziplist-value, zset-max-ziplist-entries + or zset-max-ziplist-value [reported by sundb]. +* (CVE-2021-32627) Integer to heap buffer overflow issue with streams, when + configuring a non-default, large value for proto-max-bulk-len and + client-query-buffer-limit [reported by sundb]. +* (CVE-2021-32626) Specially crafted Lua scripts may result with Heap buffer + overflow [reported by Meir Shpilraien]. + +Bug fixes that involve behavior changes: +* GEO* STORE with empty source key deletes the destination key and return 0 (#9271) + Previously it would have returned an empty array like the non-STORE variant. +* PUBSUB NUMPAT replies with number of patterns rather than number of subscriptions (#9209) + This actually changed in 6.2.0 but was overlooked and omitted from the release notes. + +Bug fixes that are only applicable to previous releases of Redis 6.2: +* Fix CLIENT PAUSE, used an old timeout from previous PAUSE (#9477) +* Fix CLIENT PAUSE in a replica would mess the replication offset (#9448) +* Add some missing error statistics in INFO errorstats (#9328) + +Other bug fixes: +* Fix incorrect reply of COMMAND command key positions for MIGRATE command (#9455) +* Fix appendfsync to always guarantee fsync before reply, on MacOS and FreeBSD (kqueue) (#9416) +* Fix the wrong mis-detection of sync_file_range system call, affecting performance (#9371) + +CLI tools: +* When redis-cli received ASK response, it didn't handle it (#8930) -This Redis release, 5.0.6, is a bugfix and enhancement release. The most -important bugfix is a corruption related to the HyperLogLog. A malformed -HyperLogLog string could cause an invalid access to the memory. At a first -glance the vulnerability appears to be not exploitable but just a DoS. The -way to trigger the issue is complex, we'll not provide any information about -how to do that for the users safety. - -Other significant changes in this release: - -* New modules APIs merged from Redis unstable to Redis 5. -* Some memory optimization related to objects creation. -* Fixes to flushSlaveOutputBuffer() that make sure that SHUTDOWN will - transfer pending buffers to replicas. - -This is the full list of commits: - -antirez in commit 7a41047a: - RDB: fix MODULE_AUX loading by continuing to next opcode. - 1 file changed, 1 insertion(+) - -Oran Agra in commit 4eb3028b: - missing per-skiplist overheads in MEMORY USAGE - 1 file changed, 3 insertions(+), 1 deletion(-) - -Oran Agra in commit 5d09f9bc: - RM_Log - add support for logging without a context or context without module - 1 file changed, 6 insertions(+), 4 deletions(-) - -antirez in commit 2810de9f: - Cluster: abort loading nodes data if vars arguments are unbalanced. - 1 file changed, 1 insertion(+) - -antirez in commit f5c63ce0: - More strict checks and better comments in flushSlaveOutputBuffers(). - 1 file changed, 18 insertions(+), 7 deletions(-) - -antirez in commit 7f289c3b: - Improve comment in flushSlavesOutputBuffers(). - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 7ab62d4b: - Replication: clarify why repl_put_online_on_ack exists at all. - 2 files changed, 34 insertions(+), 10 deletions(-) - -zhaozhao.zz in commit 495dd0da: - networking: flushSlavesOutputBuffers bugfix - 1 file changed, 2 insertions(+), 4 deletions(-) - -Salvatore Sanfilippo in commit c1ccf0f1: - Merge pull request #6366 from oranagra/5.0_rm_reply_cstring -Salvatore Sanfilippo in commit a50dad73: - Merge pull request #6365 from oranagra/5.0_module_aux -Oran Agra in commit d6294d05: - RM_ReplyWithCString was missing registration - 1 file changed, 1 insertion(+) - -Oran Agra in commit 8c56fc86: - Fix to module aux data rdb format for backwards compatibility with old check-rdb - 1 file changed, 9 insertions(+), 1 deletion(-) - -Oran Agra in commit 98b1314f: - Implement module api for aux data in rdb - 9 files changed, 431 insertions(+), 18 deletions(-) - -antirez in commit 08b03e23: - redis-cli: always report server errors on read errors. - 1 file changed, 8 insertions(+), 1 deletion(-) - -wubostc in commit 239069de: - Reduce the calling stack - 1 file changed, 2 insertions(+), 3 deletions(-) - -antirez in commit 90bf6313: - Make EMBSTR case of #6261 more obvious. - 1 file changed, 1 insertion(+), 1 deletion(-) - -chendianqiang in commit 2f8a0749: - make memory usage consistent of robj with OBJ_ENCODING_INT - 1 file changed, 9 insertions(+), 4 deletions(-) - -antirez in commit 436ed56d: - HyperLogLog: fix the fix of a corruption bug. - 1 file changed, 1 insertion(+), 2 deletions(-) - -John Sully in commit 680f89fb: - Fix HLL corruption bug - 1 file changed, 1 insertion(+) - -swilly22 in commit 388efbf8: - Extend REDISMODULE_CTX_FLAGS to indicate if redis is currently loading from either RDB or AOF - 2 files changed, 5 insertions(+) - -Itamar Haber in commit 0ccbdcee: - Uses addReplyBulkCString - 1 file changed, 1 insertion(+), 1 deletion(-) - -Itamar Haber in commit 707a59c6: - Adds RedisModule_ReplyWithCString - 2 files changed, 13 insertions(+) - -================================================================================ -Redis 5.0.5 Released Wed May 15 17:57:41 CEST 2019 -================================================================================ - -Upgrade urgency CRITICAL: This release fixes an important AOF fysnc bug - and other less critical issues. - - -Dear user, - -Redis 5.0.5 fixes an important issue with AOF and adds multiple very useful -modules APIs. Moreover smaller bugs in other parts of Redis are fixed in -this release. - -The AOF bug ------------ - -The AOF bug happens when the fsync policy is set to "everysec", which is the -default: if the write load in the server drops immediately, the commands -executed in the latest second may not be fsync-ed to disk as it should. -This may lead to data loss in case the write load drops immediately and -successively a server crash happens. - -Other things in this release ----------------------------- - -* Streams: a bug in the iterator could prevent certain items to be returned in - range queries under specific conditions. -* Memleak in bitfieldCommand fixed. -* Modules API: Preserve client->id for blocked clients. -* Fix memory leak when rewriting config file in case of write errors. -* New modules API: RedisModule_GetKeyNameFromIO(). -* Fix non critical bugs in diskless replication. -* New mdouels API: command filtering. See RedisModule_RegisterCommandFilter(); -* Tests improved to be more deterministic. -* Fix a Redis Cluster bug, manual failover may abort because of the master - sending PINGs to the replicas. - -The following is the full list of commmits. - -Regards, -Salvatore - -Christian Zeller in commit 1cac9b4b: - Typo fixes in CONTRIBUTING - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit f63c5c7b: - Update CONTRIBUTING with present info. - 1 file changed, 15 insertions(+), 5 deletions(-) - -antirez in commit 668661da: - Narrow the effects of PR #6029 to the exact state. - 1 file changed, 17 insertions(+), 5 deletions(-) - -chendianqiang in commit 3c2800e3: - stop ping when client pause - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 7ac7ffd5: - Test: fix slowlog test false positive. - 1 file changed, 3 insertions(+), 1 deletion(-) - -antirez in commit cc101721: - Make comment in getClientOutputBufferMemoryUsage() describing the present. - 1 file changed, 1 insertion(+), 8 deletions(-) - -WuYunlong in commit 72420b0d: - Do not active expire keys in the background when the switch is off. - 1 file changed, 6 insertions(+), 4 deletions(-) - -liaotonglang in commit 33a50d24: - delete sdsTest() from REDIS_TEST - 1 file changed, 2 deletions(-) - -zhaozhao.zz in commit 6a92836f: - test cases: skiptill -> skip-till - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit f179f71e: - make replication tests more stable on slow machines - 3 files changed, 34 insertions(+), 4 deletions(-) - -Yossi Gottlieb in commit 1825a4ec: - Add runtest-moduleapi with commandfilter coverage. - 5 files changed, 63 insertions(+), 28 deletions(-) - -Yossi Gottlieb in commit 9d20fdb4: - fix: missing initialization. - 3 files changed, 1 insertion(+) - -antirez in commit ded1980e: - Test: disable module testing for now. - 1 file changed, 1 deletion(-) - -Yossi Gottlieb in commit c3df78c2: - CommandFilter API: REDISMODULE_CMDFILTER_NOSELF. - 4 files changed, 62 insertions(+), 15 deletions(-) - -Yossi Gottlieb in commit 8d38ef20: - CommandFilter API: fix UnregisterCommandFilter. - 1 file changed, 2 insertions(+), 3 deletions(-) - -Yossi Gottlieb in commit 9b7009b1: - CommandFilter API: Add unregister option. - 4 files changed, 126 insertions(+), 32 deletions(-) - -Yossi Gottlieb in commit 05802442: - CommandFilter API: Extend documentation. - 1 file changed, 43 insertions(+), 5 deletions(-) - -Yossi Gottlieb in commit d5194daf: - CommandFilter API: hellofilter and tests. - 2 files changed, 47 insertions(+), 5 deletions(-) - -Yossi Gottlieb in commit 8897c154: - CommandFilter API: Support Lua and RM_call() flows. - 2 files changed, 18 insertions(+), 7 deletions(-) - -Yossi Gottlieb in commit 6dd5bad4: - CommandFilter API: More cleanup. - 2 files changed, 10 insertions(+), 29 deletions(-) - -Yossi Gottlieb in commit 83026101: - Add command filter Module API tests. - 2 files changed, 28 insertions(+) - -Yossi Gottlieb in commit dc5edc7b: - Add command filtering argument handling API. - 3 files changed, 132 insertions(+), 13 deletions(-) - -Yossi Gottlieb in commit 5f29e2e2: - Initial command filter experiment. - 6 files changed, 161 insertions(+), 2 deletions(-) - -Oran Agra in commit e1839ab3: - diskless fork kept streaming RDB to a disconnected slave - 1 file changed, 10 insertions(+) - -Oran Agra in commit 3b207b89: - diskless replication - notify slave when rdb transfer failed - 1 file changed, 1 insertion(+) - -antirez in commit 7e350b09: - More sensible name for function: restartAOFAfterSYNC(). - 1 file changed, 7 insertions(+), 3 deletions(-) - -antirez in commit 91238a60: - Mostly aesthetic changes to restartAOF(). - 1 file changed, 7 insertions(+), 3 deletions(-) - -oranagra in commit ee2da67c: - bugfix to restartAOF, exit will never happen since retry will get negative. - 1 file changed, 5 insertions(+), 4 deletions(-) - -Oran Agra in commit 78022492: - Add log when server dies of SIGTERM during loading - 1 file changed, 1 insertion(+) - -Yossi Gottlieb in commit 232dca7f: - Add RedisModule_GetKeyNameFromIO(). - 8 files changed, 30 insertions(+), 17 deletions(-) - -antirez in commit 7f98129a: - MANIFESTO: simplicity and lock-in. - 1 file changed, 5 insertions(+), 1 deletion(-) - -antirez in commit 71265fe3: - MANIFESTO v2. - 1 file changed, 41 insertions(+), 6 deletions(-) - -yongman in commit 8115be6e: - Fix uint64_t hash value in active defrag - 1 file changed, 3 insertions(+), 3 deletions(-) - -Angus Pearson in commit 90e7b5a9: - Enlarge error buffer in redis-check-aof.c to remove compiler warning of output truncation through snprintf format string - 1 file changed, 1 insertion(+), 1 deletion(-) - -zhaozhao.zz in commit 43151baf: - fix memory leak when rewrite config file - 1 file changed, 3 insertions(+), 4 deletions(-) - -唐权 in commit d3c17c9d: - Update ziplist.c - 1 file changed, 1 insertion(+), 1 deletion(-) - -stan011 in commit 296bd097: - change the comments there may have a mis type - 1 file changed, 1 insertion(+), 1 deletion(-) - -Yossi Gottlieb in commit e08c9c15: - Preserve client->id for blocked clients. - 1 file changed, 4 insertions(+), 1 deletion(-) - -zhaozhao.zz in commit c6b1252f: - aof: enhance AOF_FSYNC_EVERYSEC, more details in #5985 - 2 files changed, 32 insertions(+), 3 deletions(-) - -David Carlier in commit ce54e299: - build fix - 1 file changed, 1 insertion(+) - -yongman in commit c9274498: - Fix memleak in bitfieldCommand - 1 file changed, 8 insertions(+), 2 deletions(-) - -James Rouzier in commit 635d8d83: - Fix start and end key initialize - 1 file changed, 2 insertions(+), 2 deletions(-) - -Salvatore Sanfilippo in commit 7c23e534: - Merge pull request #6047 from abhaynahar/removed-obsolete-warning-5.0 -abhay in commit 9ea8ec42: - removed obsolete warning as per - https://github.com/antirez/redis/issues/5291 - 1 file changed, 1 insertion(+), 7 deletions(-) - -antirez in commit 1b7407fa: - Aesthetic change to #5962 to conform to Redis style. - 1 file changed, 1 insertion(+), 3 deletions(-) - -Oran Agra in commit 3bbf9747: - slave corrupts replication stream when module blocked client uses large reply (or POSTPONED_ARRAY) - 3 files changed, 15 insertions(+), 6 deletions(-) +Improvements: +* Add latency monitor sample when key is deleted via lazy expire (#9317) +* Sanitize corrupt payload improvements (#9321, #9399) +* Delete empty keys when loading RDB file or handling a RESTORE command (#9297, #9349) ================================================================================ -Redis 5.0.4 Released Mon Mar 18 17:12:53 CET 2019 +Redis 6.2.5 Released Wed Jul 21 16:32:19 IDT 2021 ================================================================================ -Upgrade urgency HIGH: This release fixes several Redis stability issues. - -Dear Redis users, this release includes a number of fixes for bugs that may -result in Redis crashing in special conditions (not normal usage, but specific -artificial conditions), fixes to certain Redis behaviors especially around -Redis streams, and finally a set of new APIs for Redis Modules. - -Specifically: - -* Hyperloglog different coding errors leading to potential crashes were fixed. -* A replication bug leading to a potential crash in case of plain misuse of handshake commands was fixed. -* XCLAIM command incrementing of number of deliveries was fixed. -* LFU field management in objects was improved. -* A potential overflow in the redis-check-aof was fixed. -* A memory leak in case of API misuse was fixed. -* ZPOP* behavior when count is 0 is fixed. -* A few redis-cli --cluster bugs were fixed, plus a few improvements. -* Many other smaller bugs. - -We suggest to upgrade Redis, especially in case your instance is facing -untrusted users (for instance Cloud providers) because several of these -bugs could result in unwanted crashes. - -This is the list of commits: - -antirez in commit 84bdd440: - HyperLogLog: fix comment in hllCount(). - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit ef1833b3: - HyperLogLog: handle wrong offset in the base case. - 1 file changed, 2 insertions(+), 6 deletions(-) - -antirez in commit 623afd5e: - HyperLogLog: speedup fuzz test. - 1 file changed, 1 insertion(+), 2 deletions(-) - -antirez in commit 12b5ff10: - HyperLogLog: enlarge reghisto variable for safety. - 1 file changed, 6 insertions(+), 1 deletion(-) - -antirez in commit 254d897e: - HyperLogLog: dense/sparse repr parsing fuzz test. - 1 file changed, 29 insertions(+) - -John Sully in commit 7f79849c: - Fix hyperloglog corruption - 1 file changed, 6 insertions(+) - -Brad Solomon in commit 3ef2c831: - Provide an uninstall target in Makefile - 1 file changed, 3 insertions(+) - -antirez in commit 57aea463: - redis-check-aof: fix potential overflow. - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit ba5145b8: - Fix objectSetLRUOrLFU() when LFU underflows. - 1 file changed, 11 insertions(+), 7 deletions(-) - -antirez in commit 76c59f0e: - Fix ZPOP return type when COUNT=0. Related to #5799. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 1c636714: - Improve comments after merging #5834. - 2 files changed, 14 insertions(+), 8 deletions(-) - -Guy Benoish in commit 6a3fca4c: - Trim SDS free space of retained module strings - 4 files changed, 28 insertions(+), 5 deletions(-) - -Guy Benoish in commit 9ec144ea: - Fix mismatching keyspace notification classes - 2 files changed, 2 insertions(+), 2 deletions(-) - -Guy Benoish in commit d04b5211: - Fix zlexrangespec mem-leak in genericZrangebylexCommand - 1 file changed, 4 insertions(+), 1 deletion(-) - -Guy Benoish in commit 516f1c77: - Use memtoll() in 'CONFIG SET client-output-buffer-limit' - 1 file changed, 2 insertions(+), 2 deletions(-) - -Guy Benoish in commit 8db67a55: - Increase string2ld's buffer size (and fix HINCRBYFLOAT) - 2 files changed, 5 insertions(+), 1 deletion(-) - -Guy Benoish in commit db3d626b: - Check server.verbosity in RM_LogRaw - 1 file changed, 2 insertions(+) - -Guy Benoish in commit 71439a07: - ZPOP should return an empty array if COUNT=0 - 1 file changed, 4 insertions(+), 1 deletion(-) - -antirez in commit c8a26834: - Modules shared API: export new core APIs. - 2 files changed, 6 insertions(+) - -antirez in commit a13ba750: - Modules shared API: also unregister the module as user. - 1 file changed, 23 insertions(+) - -antirez in commit 500e5117: - Modules shared API: prevent unloading of used modules. - 1 file changed, 10 insertions(+), 4 deletions(-) - -antirez in commit 7854daa1: - Modules shared API: unregister APIs function. - 1 file changed, 25 insertions(+) - -antirez in commit d38d82af: - Modules shared API: initial core functions. - 2 files changed, 89 insertions(+), 1 deletion(-) - -antirez in commit 4d747bb8: - Revert shared APIs to modify the design. - 3 files changed, 120 deletions(-) - -MeirShpilraien in commit 8824b509: - added module ability to register api to be used by other modules - 3 files changed, 120 insertions(+) - -zhaozhao.zz in commit 000b055b: - Streams: checkType before XGROUP CREATE - 1 file changed, 7 insertions(+), 5 deletions(-) - -antirez in commit 9b2a0d54: - Fix BZPOP arity, backport from fix in cd2743c. - 1 file changed, 2 insertions(+), 2 deletions(-) - -chendianqiang in commit 134b2582: - optimize cluster failover - 1 file changed, 1 insertion(+) - -Steve Webster in commit 1293e2a5: - Only increment delivery count if JUSTID option is omitted - 2 files changed, 18 insertions(+), 3 deletions(-) - -Steve Webster in commit 3cc4f469: - Increment delivery counter on XCLAIM unless RETRYCOUNT specified - 2 files changed, 35 insertions(+), 2 deletions(-) - -antirez in commit f4edd2b9: - Merge branch '5.0' of github.com:/antirez/redis into 5.0 -swilly22 in commit cedcc54e: - document additional flag of RM_GetContextFlags - 1 file changed, 3 insertions(+) - -swilly22 in commit 26e98da2: - Extend REDISMODULE_CTX_FLAGS to indicate if command was sent by master - 2 files changed, 6 insertions(+) - -Salvatore Sanfilippo in commit 0e910939: - Merge pull request #5879 from meierfra-ergon/redis-cli-assume-yes -antirez in commit 67452e91: - Make comment in #5911 stay inside 80 cols. - 1 file changed, 2 insertions(+), 1 deletion(-) - -John Sully in commit 30f666ef: - Replicas aren't allowed to run the replicaof command - 1 file changed, 8 insertions(+) - -Frank Meier in commit bc6c1c40: - extend use of cluster-yes option to other confimation questions - 1 file changed, 9 insertions(+) - -antirez in commit 76419d8d: - Merge branch '5.0' of github.com:/antirez/redis into 5.0 -Oran Agra in commit 72ba6069: - redis-cli add support for --memkeys, fix --bigkeys for module types - 1 file changed, 132 insertions(+), 81 deletions(-) - -chendianqiang in commit 2ca21753: - fix replicationid will not change for server.masterhost==NULL in cluster mode when restart slave - 1 file changed, 1 insertion(+), 1 deletion(-) - -Salvatore Sanfilippo in commit bd7ddd79: - Merge pull request #5870 from fengweiyuan/5.0 -varianfeng in commit d13bc143: - fix corrupt_rdb.c bug.Let the name of input rdb file name be valid. - 1 file changed, 2 insertions(+), 1 deletion(-) - -artix in commit 44c5bce0: - Cluster Manager: fix replica assigment anti-affinity (create) - 1 file changed, 6 insertions(+) - -artix in commit f066e526: - Cluster Manager: remove unused code elements - 1 file changed, 8 insertions(+), 13 deletions(-) - -Zhicheng Wei in commit 23214966: - fix clusterManagerGetAntiAffinityScore double free otypes - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 80bccd71: - Remove debugging printf from replication.tcl test. - 1 file changed, 1 deletion(-) +Upgrade urgency: SECURITY, contains fixes to security issues that affect +authenticated client connections on 32-bit versions. MODERATE otherwise. + +Fix integer overflow in BITFIELD on 32-bit versions (CVE-2021-32761). +An integer overflow bug in Redis version 2.2 or newer can be exploited using the +BITFIELD command to corrupt the heap and potentially result with remote code +execution. + +Bug fixes that involve behavior changes: +* Change reply type for ZPOPMAX/MIN with count in RESP3 to nested array (#8981). + Was using a flat array like in RESP2 instead of a nested array like ZRANGE does. +* Fix reply type for HRANDFIELD and ZRANDMEMBER when key is missing (#9178). + Was using a null array instead of an empty array. +* Fix reply type for ZRANGESTORE when source key is missing (#9089). + Was using an empty array like ZRANGE instead of 0 (used in the STORE variant). + +Bug fixes that are only applicable to previous releases of Redis 6.2: +* ZRANDMEMBER WITHSCORES with negative COUNT may return bad score (#9162) +* Fix crash after CLIENT UNPAUSE when threaded I/O config is enabled (#9041) +* Fix XTRIM or XADD with LIMIT may delete more entries than the limit (#9048) +* Fix build issue with OpenSSL 1.1.0 (#9233) + +Other bug fixes: +* Fail EXEC command in case a watched key is expired (#9194) +* Fix SMOVE not to invalidate dest key (WATCH and tracking) when member already exists (#9244) +* Fix SINTERSTORE not to delete dest key when getting a wrong type error (#9032) +* Fix overflows on 32-bit versions in GETBIT, SETBIT, BITCOUNT, BITPOS, and BITFIELD (#9191) +* Improve MEMORY USAGE on stream keys (#9164) +* Set TCP keepalive on inbound cluster bus connections (#9230) +* Fix diskless replica loading to recover from RDB short read on module AUX data (#9199) +* Fix race in client side tracking (#9116) +* Fix ziplist length updates on big-endian platforms (#2080) + +CLI tools: +* redis-cli cluster import command may issue wrong MIGRATE command, sending COPY instead of REPLACE (#8945) +* redis-cli --rdb fixes when using "-" to write to stdout (#9136, #9135) +* redis-cli support for RESP3 set type in CSV and RAW output (#7338) + +Modules: +* Module API for getting current command name (#8792) +* Fix RM_StringTruncate when newlen is 0 (#3718) +* Fix CLIENT UNBLOCK crashing modules without timeout callback (#9167) ================================================================================ -Redis 5.0.3 Released Tue Dec 11 18:17:26 CET 2018 +Redis 6.2.4 Released Tue June 1 12:00:00 IST 2021 ================================================================================ -Upgrade urgency HIGH: Redis 5 is consolidating, upgrading is a good idea. - However there is nothing very critical here, but certain - issues resolved could lead to very rare crashes. - -Welcome to Redis 5.0.3, several interesting bug fixes here: - -* Redis no longer panics when you send data to a replica-mode connection that - is in MONITOR or SYNC mode. - -* Fixes to certain sorted set edge cases. You are unlikely to ever notice those - issues, but now it is more correct. - -* Certain BSD variants now are better supported: build & register logging - on crash. - -* The networking core now recovers if an IPv6 address is listed in bind but - is actually not able to work because there is no such protocol in the - system. - -* redis-cli cluster mode improved in many ways. Especially the fix subcommand - work was enhanced to cover other edge cases that were still not covered - after the work done for Redis 5. +Upgrade urgency: SECURITY, Contains fixes to security issues that affect +authenticated client connections. MODERATE otherwise. -* MEMORY USAGE is now more accurate. +Fix integer overflow in STRALGO LCS (CVE-2021-32625) +An integer overflow bug in Redis version 6.0 or newer can be exploited using the +STRALGO LCS command to corrupt the heap and potentially result with remote code +execution. This is a result of an incomplete fix by CVE-2021-29477. -* DEBUG DIGEST-VALUE added in case you want to make sure a given set of keys - (and not the whole DB) are excatly the same between two instances. +Bug fixes that are only applicable to previous releases of Redis 6.2: +* Fix crash after a diskless replication fork child is terminated (#8991) +* Fix redis-benchmark crash on unsupported configs (#8916) -* Fix a potential crash in the networking code related to recent changes - to the way the reply is consumed. +Other bug fixes: +* Fix crash in UNLINK on a stream key with deleted consumer groups (#8932) +* SINTERSTORE: Add missing keyspace del event when none of the sources exist (#8949) +* Sentinel: Fix CONFIG SET of empty string sentinel-user/sentinel-pass configs (#8958) +* Enforce client output buffer soft limit when no traffic (#8833) -* Reject EXEC containing write commands against an instance that changed role - from master to replica during our transaction. - -* Fix a crash in KEYS and other commands using pattern matching, in an edge - case where the pattern contains a zero byte. - -* Fix eviction during AOF loading due to maxmemory triggered by commands - executed in loading state. - -The following is the list of commmits if you want to check credits or dig -further in the details. - -commit 2c6ee0f9b3d9ca48c6da8bd18796186784216bff -Author: antirez -Date: Wed Dec 12 11:37:15 2018 +0100 - - freeMemoryIfNeeded() small refactoring. - - Related to issue #5686 and PR #5689. - -commit 107e93e75acfd5def0252efb6870751940816395 -Author: zhaozhao.zz -Date: Wed Dec 12 00:25:24 2018 +0800 - - evict: don't care about mem if loading - - When loading data, we call processEventsWhileBlocked - to process events and execute commands. - But if we are loading AOF it's dangerous, because - processCommand would call freeMemoryIfNeeded to evict, - and that will break data consistency, see issue #5686. - -antirez in commit ee93dc0b: - Crashing is too much in addReplyErrorLength(). - 1 file changed, 6 deletions(-) - -hdmg in commit c55254a5: - fix comments fault discription - 1 file changed, 1 insertion(+), 1 deletion(-) - -lsytj0413 in commit dfd25013: - fix a typo: craeted -> created - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 392a2566: - stringmatchlen() fuzz test added. - 3 files changed, 22 insertions(+) - -antirez in commit 7602f695: - Fix stringmatchlen() read past buffer bug. - 1 file changed, 1 insertion(+), 1 deletion(-) - -zhaozhao.zz in commit c4f3585e: - multi: ignore multiState's cmd_flags when loading AOF - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit d037e987: - Reject EXEC containing write commands against RO replica. - 2 files changed, 20 insertions(+) - -artix in commit e00ab324: - Cluster Manager: - Multiple owners checking in 'fix'/'check' commands is - now optional (using --cluster-search-multiple-owners). - Updated help. - 1 file changed, 14 insertions(+), 5 deletions(-) - -artix in commit 94f64de3: - Cluster Manager: FixOpenSlot now correctly updates in-memory cluster - configuration. Improved output messages. - 1 file changed, 17 insertions(+), 5 deletions(-) - -artix in commit 752d636f: - Cluster Manager: 'fix' command now handles open slots with migrating state - in one node and importing state in multiple nodes. - 1 file changed, 74 insertions(+), 6 deletions(-) - -artix in commit 552091f9: - Cluster Manager: setting new slot owner is now handled atomically in - 'fix' command. - 1 file changed, 72 insertions(+), 31 deletions(-) - -artix in commit 2280f4f7: - Cluster Manager: code cleanup. - 1 file changed, 41 insertions(+), 87 deletions(-) - -artix in commit e084b8cc: - Cluster Manager: check/fix commands now handle multiple owners even - if all slots are covered and not open. - 1 file changed, 129 insertions(+), 6 deletions(-) - -zhaozhao.zz in commit fa726e2a: - remove useless tryObjectEncoding in debug assert - 1 file changed, 1 deletion(-) - -Oran Agra in commit 40244b10: - fix #5580, display fragmentation and rss overhead bytes as signed - 2 files changed, 6 insertions(+), 6 deletions(-) - -zhaozhao.zz in commit beab3151: - networking: current_client should not be NULL when trim qb_pos - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 07ccb642: - Remove no longer relevant comment in processCommand(). - 1 file changed, 2 insertions(+), 6 deletions(-) - -antirez in commit 60fdaf07: - DEBUG DIGEST-VALUE implemented. - 1 file changed, 17 insertions(+), 3 deletions(-) - -antirez in commit 48b31b0d: - DEBUG DIGEST refactoring: extract function to digest a value. - 1 file changed, 142 insertions(+), 131 deletions(-) - -yura in commit ef3ff402: - redis-cli reshard/rebalance: ability to force replacement on existing keys - 1 file changed, 6 insertions(+), 5 deletions(-) - -Thomas Orozco in commit ee223fb8: - cli: pass auth through REDISCLI_AUTH - 1 file changed, 14 insertions(+) - -yongman in commit 41295e55: - Fix cluster call reply format readable - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit 0ed3970f: - fix small test suite race conditions - 3 files changed, 11 insertions(+) - -zhaozhao.zz in commit 605dddbb: - MEMORY command: make USAGE more accurate - 1 file changed, 7 insertions(+), 6 deletions(-) - -yongman in commit 1f43bf29: - Fix choose a random master node for slot assignment - 1 file changed, 29 insertions(+), 5 deletions(-) - -Weiliang Li in commit 69f0c678: - fix comment typo in util.c - 1 file changed, 1 insertion(+), 1 deletion(-) - -Chris Lamb in commit bc53a3ab: - Clarify the "Creating Server TCP listening socket" error. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Chris Lamb in commit fefe5460: - Don't treat unsupported protocols as fatal errors - 1 file changed, 4 insertions(+) - -David Carlier in commit a8862972: - OpenBSD support. - 3 files changed, 74 insertions(+), 1 deletion(-) - -David Carlier in commit 5e86daf9: - Backtrace/register dump on BSD. - 3 files changed, 97 insertions(+), 3 deletions(-) - -Guy Benoish in commit 7c8cf5ac: - Don't call sdscmp() with shared.maxstring or shared.minstring - 2 files changed, 23 insertions(+), 9 deletions(-) - -Qu Chen in commit 39e9eda3: - Add unit test for stream XCLAIM command. - 1 file changed, 48 insertions(+) - -antirez in commit 62485232: - Abort instead of crashing when loading bad stream master key. - 1 file changed, 3 insertions(+) - -Madelyn Olson in commit a5487309: - Fixed a serverPanic when sending an invalid command to a monitor client - 1 file changed, 1 insertion(+), 1 deletion(-) +Improvements: +* Hide AUTH passwords in MIGRATE command from slowlog (#8859) ================================================================================ -Redis 5.0.2 Released Thu Nov 22 11:22:37 CET 2018 +Redis 6.2.3 Released Mon May 3 19:00:00 IST 2021 ================================================================================ -Upgrade urgency: CRITICAL if you use streams and consumer groups. - HIGH if you use redis-cli with Redis Cluster. - LOW otherwise. - -Welcome to Redis 5.0.2. This release fixes two issues with Streams consumer -groups, where items could be returned duplicated by XREADGROUP when accessing -the history, and another bug where XREADGROUP can report some history even -if the comsumer pending list is empty. Both problems were addressed and unit -tests to avoid regressions implemented. Moreover this release fixes some -issue with redis-cli when in cluster mode. Finally some FreeBSD and DragonFly -build problems are now resolved. The list of the commits is below. - -Enjoy, -Salvatore - -David Carlier in commit e8b4291a: - DragonFlyBSD little build fix - 2 files changed, 6 insertions(+), 1 deletion(-) - -yongman in commit 8fcfd374: - skip slave nodes when sending cluster setslot command - 1 file changed, 1 insertion(+) - -yongman in commit d7089ddd: - Fix pointer access and memory leak in redis-cli. - 1 file changed, 6 insertions(+), 3 deletions(-) +Upgrade urgency: SECURITY, Contains fixes to security issues that affect +authenticated client connections. LOW otherwise. -antirez in commit 17b4cd83: - Test: regression test for #5570. - 1 file changed, 15 insertions(+) +Integer overflow in STRALGO LCS command (CVE-2021-29477): +An integer overflow bug in Redis version 6.0 or newer could be exploited using +the STRALGO LCS command to corrupt the heap and potentially result in remote +code execution. The integer overflow bug exists in all versions of Redis +starting with 6.0. -antirez in commit 45123169: - Stream: fix XREADGROUP history reading of deleted messages. - 1 file changed, 1 insertion(+), 1 deletion(-) +Integer overflow in COPY command for large intsets (CVE-2021-29478): +An integer overflow bug in Redis 6.2 could be exploited to corrupt the heap and +potentially result with remote code execution. The vulnerability involves +changing the default set-max-intset-entries configuration value, creating a +large set key that consists of integer values and using the COPY command to +duplicate it. The integer overflow bug exists in all versions of Redis starting +with 2.6, where it could result with a corrupted RDB or DUMP payload, but not +exploited through COPY (which did not exist before 6.2). -David Carlier in commit 5ad588f0: - only FreeBSD change/little warning addressing - 2 files changed, 7 insertions(+), 4 deletions(-) +Bug fixes that are only applicable to previous releases of Redis 6.2: +* Fix memory leak in moduleDefragGlobals (#8853) +* Fix memory leak when doing lazy freeing client tracking table (#8822) +* Block abusive replicas from sending command that could assert and crash redis (#8868) -David Carlier in commit 11801e1a: - tweak form feedback - 1 file changed, 1 insertion(+), 1 deletion(-) +Other bug fixes: +* Use a monotonic clock to check for Lua script timeout (#8812) +* redis-cli: Do not use unix socket when we got redirected in cluster mode (#8870) -David Carlier in commit c1f13575: - allow flavors - 1 file changed, 1 insertion(+), 1 deletion(-) - -David Carlier in commit 275a2d49: - Fix clang build. - 1 file changed, 5 insertions(+), 1 deletion(-) - -antirez in commit 44ad5141: - Test: regression test for #5577. - 1 file changed, 24 insertions(+) - -antirez in commit c7951f43: - Streams: fix XREADGROUP history reading when CG last_id is low. - 1 file changed, 12 insertions(+), 9 deletions(-) - -antirez in commit a69bc5be: - t_stream.c comment resized to 80 cols. - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 5314099d: - Redis 5 changelog: don't expect Lua replies to be ordered. - 1 file changed, 14 insertions(+), 5 deletions(-) +Modules: +* Fix RM_GetClusterNodeInfo() to correctly populate master id (#8846) ================================================================================ -Redis 5.0.1 Released Wed Nov 07 13:09:30 CET 2018 +Redis 6.2.2 Released Mon April 19 19:00:00 IST 2021 ================================================================================ -Upgrade urgency: URGENT if you use Redis Streams. MODERATE otherwise. - -Hi all, this is the first patch level release of Redis 5. It contains -both fixes and improvements. Here there is a list of the major ones, however -read the commit messages at the end of the changelog if you want to know -more about the smaller things. Let's start with the new features: - -* Sentinel now supports authentication! Check the Sentinel official doc - for more info. - -* Redis-cli cluster "fix" is now able to fix a big number of clusters put - in a bad condition. Previously many corner cases were not covered. - -Now the critical fixes: - -1. Fix RESTORE mismatch reply when certain keys already expired. -2. Fix an XCLAIM non trivial issue: sometimes the command returned a wrong - entry or desynchronized the protocol. - -And now the other fixes: - -3. Stack trace generation on the Raspberry PI (and 32bit ARM) fixed. -4. Don't evict expired keys when the KEYS command is called, in order to - avoid a mass deletion event. However expired keys are not displayed - by KEYS as usually. -5. Improvements in the computation of the memory used, when estimating - the AOF buffers. -6. XRANGE COUNT of 0 fixed. -7. "key misses" stats accounting fixed. Many cache misses were not counted. -8. When in MULTI state, return OOM while accumulating commands and there - is no longer memory available. -9. Fix build on FreeBSD and possibly others. -10. Fix a crash in Redis modules, thread safe context reply accumulation. -11. Fix a race condition when producing the RDB file for full SYNC. -12. Disable protected mode in Sentinel. -13. More commands now have the HELP subcommand. -14. Fixed an issue about adaptive server HZ timer. -15. Fix cluster-replica-no-failover option name. - -Finally, this is the list of commits. Enjoy Redis 5.0.1! - -antirez in commit c801283f: - Fix cluster-replica-no-failover option name. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 4c4f50e1: - MEMORY command: make strcasecmp() conditional like the following. - 1 file changed, 1 insertion(+), 2 deletions(-) - -Itamar Haber in commit a7b46e0e: - Uppercases subcommands in MEMORY HELP - 1 file changed, 5 insertions(+), 5 deletions(-) +Upgrade urgency: HIGH, if you're using ACL and pub/sub, CONFIG REWRITE, or +suffering from performance regression. see below. + +Bug fixes for regressions in previous releases of Redis 6.2: +* Fix BGSAVE, AOFRW, and replication slowdown due to child reporting CoW (#8645) +* Fix short busy loop when timer event is about to fire (#8764) +* Fix default user, overwritten and reset users losing pubsub channel permissions (#8723) +* Fix config rewrite with an empty `save` config resulsing in default `save` values (#8719) +* Fix not starting on alpine/libmusl without IPv6 (#8655) +* Fix issues with propagation and MULTI/EXEC in modules (#8617) + Several issues around nested calls and thread safe contexts + +Bug fixes that are only applicable to previous releases of Redis 6.2: +* ACL Pub/Sub channels permission handling for save/load scenario (#8794) +* Fix early rejection of PUBLISH inside MULTI-EXEC transaction (#8534) +* Fix missing SLOWLOG records for blocked commands (#8632) +* Allow RESET command during busy scripts (#8629) +* Fix some error replies were not counted on stats (#8659) + +Bug fixes: +* Add a timeout mechanism for replicas stuck in fullsync (#8762) +* Process HELLO command even if the default user has no permissions (#8633) +* Client issuing a long running script and using a pipeline, got disconnected (#8715) +* Fix script kill to work also on scripts that use `pcall` (#8661) +* Fix list-compress-depth may compress more node than required (#8311) +* Fix redis-cli handling of rediss:// URL scheme (#8705) +* Cluster: Skip unnecessary check which may prevent failure detection (#8585) +* Cluster: Fix hang manual failover when replica just started (#8651) +* Sentinel: Fix info-refresh time field before sentinel get first response (#8567) +* Sentinel: Fix possible crash on failed connection attempt (#8627) +* Systemd: Send the readiness notification when a replica is ready to accept connections (#8409) + +Command behavior changes: +* ZADD: fix wrong reply when INCR used with GT/LT which blocked the update (#8717) + It was responding with the incremented value rather than nil +* XAUTOCLAIM: fix response to return the next available id as the cursor (#8725) + Previous behavior was retuning the last one which was already scanned +* XAUTOCLAIM: fix JUSTID to prevent incrementing delivery_count (#8724) + +New config options: +* Add cluster-allow-replica-migration config option (#5285) +* Add replica-announced config option (#8653) +* Add support for plaintext clients in TLS cluster (#8587) +* Add support for reading encrypted keyfiles (#8644) -Itamar Haber in commit 80e129d9: - Standardizes `MEMORY HELP` subcommand - 1 file changed, 13 insertions(+), 14 deletions(-) - -valentino in commit 88805cbb: - fix short period of server.hz being uninitialized - 1 file changed, 1 insertion(+), 1 deletion(-) - -Itamar Haber in commit 6b402733: - Adds HELP to LATENCY - 1 file changed, 14 insertions(+), 2 deletions(-) - -yongman in commit 1c637de9: - fix malloc in clusterManagerComputeReshardTable - 1 file changed, 1 insertion(+), 1 deletion(-) - -artix in commit 90b52fde: - Cluster Manager: removed unused var. - 1 file changed, 1 insertion(+), 2 deletions(-) - -artix in commit 89cbb5df: - Cluster Manager: further improvements to "fix": - clusterManagerFixOpenSlot: ensure that the slot is unassigned before ADDSLOTS - clusterManagerFixSlotsCoverage: after cold migration, the slot configuration is now updated on all the nodes. - 1 file changed, 49 insertions(+), 10 deletions(-) - -artix in commit 175515c9: - Cluster Manager: fixed string parsing issue in clusterManagerGetConfigSignature - 1 file changed, 3 insertions(+), 3 deletions(-) - -artix in commit 3997dd6e: - Cluster Manager: better fix subcommand. - 1 file changed, 78 insertions(+), 20 deletions(-) - -artix in commit bd80291c: - Cluster Manager: fixed typos in comments. - 1 file changed, 3 insertions(+), 3 deletions(-) - -artix in commit 4369cbce: - Cluster Manager: fixed 'DELSLOT' subcommand typo. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 1ed821e2: - Fix XCLAIM missing entry bug. - 1 file changed, 3 insertions(+), 2 deletions(-) - -michael-grunder in commit b49bcd01: - Use typedef'd mstime_t instead of time_t - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 09d1849e: - Improve streamReplyWithRange() top comment. - 1 file changed, 9 insertions(+), 5 deletions(-) - -antirez in commit bdf6306f: - Add support for Sentinel authentication. - 1 file changed, 17 insertions(+), 3 deletions(-) - -antirez in commit 50222af5: - Disable protected mode in Sentinel mode. - 1 file changed, 1 insertion(+) - -antirez in commit 643ee6e3: - When replica kills a pending RDB save during SYNC, log it. - 1 file changed, 6 insertions(+) - -Andrey Bugaevskiy in commit 8b609c99: - Move child termination to readSyncBulkPayload - 1 file changed, 6 insertions(+), 7 deletions(-) - -Andrey Bugaevskiy in commit 27102605: - Prevent RDB autosave from overwriting full resync results - 1 file changed, 7 insertions(+) - -antirez in commit a677923d: - asyncCloseClientOnOutputBufferLimitReached(): don't free fake clients. - 1 file changed, 1 insertion(+) - -David Carlier in commit 427e440a: - needs it for the global - 1 file changed, 1 insertion(+) - -David Carlier in commit 28f9ca4e: - Fix non Linux build. - 3 files changed, 20 insertions(+), 1 deletion(-) - -zhaozhao.zz in commit 4bf9efe2: - MULTI: OOM err if cannot free enough memory in MULTI/EXEC context - 1 file changed, 5 insertions(+), 2 deletions(-) - -antirez in commit 4fbd7a39: - Add command fingerprint comment for XSETID. - 1 file changed, 3 insertions(+), 1 deletion(-) - -Itamar Haber in commit 2480db53: - Plugs a potential underflow - 1 file changed, 1 insertion(+) - -Itamar Haber in commit e5e4d2ef: - Corrects inline documentation of syntax - 1 file changed, 1 insertion(+), 1 deletion(-) - -zhaozhao.zz in commit 713800d2: - if we read a expired key, misses++ - 1 file changed, 5 insertions(+), 1 deletion(-) - -antirez in commit e79ee263: - Fix XRANGE COUNT option for value of 0. - 1 file changed, 8 insertions(+), 2 deletions(-) - -antirez in commit 505cc70f: - Fix typo in streamReplyWithRange() top comment. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Damien Tournoud in commit 3c36561d: - Overhead is the allocated size of the AOF buffer, not its length - 2 files changed, 2 insertions(+), 2 deletions(-) - -antirez in commit 3761582f: - Simplify part of the #5470 patch. - 1 file changed, 11 insertions(+), 12 deletions(-) - -zhaozhao.zz in commit edc47a3a: - do not delete expired keys in KEYS command - 1 file changed, 34 insertions(+), 27 deletions(-) - -antirez in commit 9872af6d: - Use guide comments to make changes in #5462 more obvious. - 1 file changed, 6 insertions(+) - -youjiali1995 in commit 3f399c3b: - migrate: fix mismatch of RESTORE reply when some keys have expired. - 1 file changed, 8 insertions(+), 6 deletions(-) - -hujie in commit eaaff621: - fix typo in config.c - 1 file changed, 1 insertion(+), 1 deletion(-) - -hujiecs in commit 43ebb7ee: - several typos fixed, optimize MSETNX to avoid unnecessary loop - 4 files changed, 4 insertions(+), 4 deletions(-) - -antirez in commit de8fdaac: - Remove useless complexity from MSET implementation. - 1 file changed, 5 insertions(+), 7 deletions(-) - -antirez in commit dc8f1112: - Fix again stack generation on the Raspberry Pi. - 1 file changed, 4 insertions(+) - -antirez in commit 83a6e81d: - Get rid of the word slave in the release note of Redis 5. - 1 file changed, 2 insertions(+), 2 deletions(-) +Improvements: +* Fix performance regression in BRPOP on Redis 6.0 (#8689) +* Avoid adding slowlog entries for config with sensitive data (#8584) +* Improve redis-cli non-binary safe string handling (#8566) +* Optimize CLUSTER SLOTS reply (#8541) +* Handle remaining fsync errors (#8419) + +Info fields and introspection changes: +* Strip % sign from current_fork_perc info field (#8628) +* Fix RSS memory info on FreeBSD (#8620) +* Fix client_recent_max_input/output_buffer in 'INFO CLIENTS' when all clients drop (#8588) +* Fix invalid master_link_down_since_seconds in info replication (#8785) + +Platform and deployment-related changes: +* Fix FreeBSD <12.x builds (#8603) + +Modules: +* Add macros for RedisModule_log logging levels (#4246) +* Add RedisModule_GetAbsExpire / RedisModule_SetAbsExpire (#8564) +* Add a module type for key space notification (#8759) +* Set module eviction context flag only in masters (#8631) +* Fix unusable RedisModule_IsAOFClient API (#8596) +* Fix missing EXEC on modules propagation after failed EVAL execution (#8654) +* Fix edge-case when a module client is unblocked (#8618) ================================================================================ -Redis 5.0.0 Released Wed Oct 17 13:28:26 CEST 2018 +Redis 6.2.1 Released Mon Mar 1 17:51:36 IST 2021 ================================================================================ -Upgrade urgency CRITICAL: Several fixes to streams AOF and replication. - -Hi all and welcome to the first stable release of Redis 5! \o/ - -To start a quick recap of what's new in Redis 5: - -1. The new Stream data type. https://redis.io/topics/streams-intro -2. New Redis modules APIs: Timers, Cluster and Dictionary APIs. -3. RDB now store LFU and LRU information. -4. The cluster manager was ported from Ruby (redis-trib.rb) to C code - inside redis-cli. Check `redis-cli --cluster help` for more info. -5. New sorted set commands: ZPOPMIN/MAX and blocking variants. -6. Active defragmentation version 2. -7. Improvemenets in HyperLogLog implementations. -8. Better memory reporting capabilities. -9. Many commands with sub-commands now have an HELP subcommand. -10. Better performances when clients connect and disconnect often. -11. Many bug fixes and other random improvements. -12. Jemalloc was upgraded to version 5.1 -13. CLIENT UNBLOCK and CLIENT ID. -14. The LOLWUT command was added. http://antirez.com/news/123 -15. We no longer use the "slave" word if not for API backward compatibility. -16. Differnet optimizations in the networking layer. -17. Lua improvements: - - Better propagation of Lua scripts to replicas / AOF. - - Lua scripts can now timeout and get in -BUSY state in the replica as well. -18. Dynamic HZ to balance idle CPU usage with responsiveness. -19. The Redis core was refactored and improved in many ways. - -However the list above really does not do justice to the changes of Redis 5 -since the core was improved in many ways during the development of the new -version. However certain changes were back ported into Redis 4 once they were -sensed as safe, because many improvements were hard to distinguish from fixes. - -The most important user facing improvement is without doubts the introduction -of the new general purpose data type after years: the streams. - -Note that we worked to improve and fix streams till a few hours ago, so while -we are not aware of critical bugs in this release, surely there is to handle it -with some care for the first weeks. Bug reporting will be highly appreciated and -we are ready to work immediately to release 5.0.1 once there is enough important -stuff to justify a new release (probably soon). - -People not using the streams can have probably a better production-ready -experience with Redis 5, also because many internals are shared with Redis 4 -so the jump is not as big as it was between 3.2 and 4 in terms of how things -internally work. - -Well, many thanks to the Redis community and the developers that made -this release possible, contributing bug reports, patches, new features, working -on the clients, sometimes debugging problems for days. Also thank to everybody -that adopted Redis for their use cases making things work for users worldwide. - -The list of commits in this release follows. - -Cheers, -Salvatore - -antirez in commit bcc0916d: - Fix conditional in XGROUP. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 1b2f23f3: - Update help.h for redis-cli. - 1 file changed, 57 insertions(+), 7 deletions(-) - -antirez in commit de0ae56c: - Tests for XGROUP CREATE MKSTREAM. - 1 file changed, 11 insertions(+) - -antirez in commit 56c3dfa1: - Fix XGROUP CREATE MKSTREAM handling of . - 1 file changed, 7 insertions(+), 2 deletions(-) - -antirez in commit 2687f228: - Process MKSTREAM option of XGROUP CREATE at a later time. - 1 file changed, 28 insertions(+), 17 deletions(-) - -zhaozhao.zz in commit cfbaf8f1: - Scripting & Streams: some commands need right flags - 1 file changed, 5 insertions(+), 5 deletions(-) - -antirez in commit 4e4099b9: - XGROUP CREATE: MKSTREAM option for automatic stream creation. - 1 file changed, 29 insertions(+), 5 deletions(-) - -zhaozhao.zz in commit 6dd4d864: - Streams: Tests modified XSTREAM -> XSETID - 1 file changed, 2 insertions(+), 2 deletions(-) - -zhaozhao.zz in commit 3aff0e8c: - Streams: rewrite empty streams with certain lastid - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 880b563e: - Tests modified to use XADD MAXLEN 0 + XSETID. - 1 file changed, 12 insertions(+), 26 deletions(-) - -antirez in commit 83c87835: - Streams: rewrite empty streams with XADD MAXLEN 0. Use XSETID. - 1 file changed, 18 insertions(+), 12 deletions(-) - -antirez in commit fd22e3ac: - XSETID: accept IDs based on last entry. - 1 file changed, 18 insertions(+), 5 deletions(-) - -antirez in commit dfab3cba: - Streams: XSTREAM SETID -> XSETID. - 3 files changed, 17 insertions(+), 67 deletions(-) - -zhaozhao.zz in commit a3fb28ed: - Streams: rewrite id in XSTREAM CREATE * - 1 file changed, 4 insertions(+) - -zhaozhao.zz in commit f4b4db13: - Streams: add tests for aof rewrite - 1 file changed, 23 insertions(+) - -zhaozhao.zz in commit d22f1ef0: - Stream & AOF: rewrite stream in correct way - 1 file changed, 32 insertions(+), 16 deletions(-) - -zhaozhao.zz in commit 6455274d: - Streams: add tests for XSTREAM command - 1 file changed, 39 insertions(+) - -zhaozhao.zz in commit 0edbe953: - Streams: add a new command XTREAM - 3 files changed, 67 insertions(+) +Upgrade urgency: LOW. -Hamid Alaei in commit 9714bba2: - fix timer context selected database - 1 file changed, 3 insertions(+), 1 deletion(-) +Here is a comprehensive list of changes in this release compared to 6.2.0, +each one includes the PR number that added it, so you can get more details +at https://github.com/redis/redis/pull/ -antirez in commit eb53f15a: - Make comment about nack->consumer test for minidle more obvious. - 1 file changed, 4 insertions(+), 2 deletions(-) +Bug fixes: +* Fix sanitize-dump-payload for stream with deleted records (#8568) +* Prevent client-query-buffer-limit config from being set to lower than 1mb (#8557) -antirez in commit a77f836e: - Streams: use propagate_last_id itself as streamPropagateGroupID trigger. - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit 0f0610eb: - Streams: better naming: lastid_updated -> propagate_last_id. - 1 file changed, 6 insertions(+), 6 deletions(-) - -zhaozhao.zz in commit a745e423: - Streams: panic if streamID invalid after check, should not be possible. - 1 file changed, 2 insertions(+), 1 deletion(-) - -zhaozhao.zz in commit 9974be13: - Streams: propagate lastid in XCLAIM when it has effect - 1 file changed, 13 insertions(+), 6 deletions(-) - -zhaozhao.zz in commit 69a628d0: - Streams: XCLAIM ignore minidle if NACK is created by FORCE - 1 file changed, 4 insertions(+), 2 deletions(-) - -zhaozhao.zz in commit a04b43c7: - Streams: bugfix XCLAIM should propagate group name not consumer name - 1 file changed, 1 insertion(+), 1 deletion(-) - -Sergey Chupov in commit 8977a90c: - fixed typos in readme - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit 3a745674: - redis.conf typo fixed: ingore -> ignore. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 22770d76: - Rax: radix tree updated to latest version from antirez/rax. - 2 files changed, 233 insertions(+), 68 deletions(-) - -antirez in commit fbac534f: - Test: avoid time related false positive in RESTORE test. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 49872337: - LOLWUT: capitalize Nees. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 80c471f5: - Test: cgroup propagation test also for NOACK variant. - 1 file changed, 39 insertions(+), 29 deletions(-) - -antirez in commit 8defa5da: - Test: consumer group last ID slave propagation test. - 1 file changed, 39 insertions(+) - -zhaozhao.zz in commit e1e3eaca: - Avoid recreate write handler for protected client. - 1 file changed, 4 insertions(+) +Improvements: +* Make port, tls-port and bind config options modifiable at runtime (#8510) -antirez in commit b501fd5d: - Fix propagation of consumer groups last ID. - 3 files changed, 56 insertions(+), 9 deletions(-) +Platform and deployment-related changes: +* Fix compilation error on non-glibc systems if jemalloc is not used (#8533) +* Improved memory consumption and memory usage tracking on FreeBSD (#8545) +* Fix compilation on ARM64 MacOS with jemalloc (#8458) +Modules: +* New Module API for getting user name of a client (#8508) +* Optimize RM_Call by utilizing a shared reusable client (#8516) +* Fix crash running CLIENT INFO via RM_Call (#8560) ================================================================================ -Redis 5.0-rc6 Released Wed Oct 10 11:03:54 CEST 2018 +Redis 6.2.0 GA Released Tue Feb 22 14:00:00 IST 2021 ================================================================================ -Upgrade urgency HIGH: Many bugs fixed especially in the context of streams. - -This is probably the last release candidate of Redis 5. The Redis 5 GA version -will be released 17th of October. The main highlights of this release are: - -* Critical AOF bug, as old as AOF itself: if an open MULTI/EXEC block is at - the end of the AOF file, Redis would still read the half-transaction when - reloading back the AOF. -* The slave name was removed from logs and documentation, now replica is used - instead. -* LOLWUT command added. -* New modules APIs: Disable Redis Cluster redirection. -* New modules APIs: Sorted dictionaries data type. -* Modules APIs fixes: timer / cluster messages callback now can call RM_Call(). -* Fix for #5024 - commandstats for multi-exec were logged as EXEC. -* A number of optimizations and fixes for the stream data type. -* Many other stability improvements. - -This is the list of comments and contributors: - -antirez in commit 9a6fa7d0: - changelog.tcl: get optional argument for number of commits. - 1 file changed, 8 insertions(+), 3 deletions(-) - -antirez in commit 101e419f: - Free protected clients asynchronously. - 1 file changed, 7 insertions(+) - -antirez in commit 726debb8: - Actually use the protectClient() API where needed. - 2 files changed, 8 insertions(+), 9 deletions(-) - -antirez in commit 0b87f78a: - Introduce protectClient() + some refactoring. - 2 files changed, 60 insertions(+), 18 deletions(-) - -zhaozhao.zz in commit 6aa8ac70: - debug: avoid free client unexpectedly when reload & loadaof - 1 file changed, 8 insertions(+), 2 deletions(-) - -antirez in commit 48040b02: - aof.c: improve indentation and change warning message. - 1 file changed, 11 insertions(+), 4 deletions(-) - -zhaozhao.zz in commit 7cc20569: - AOF: discard if we lost EXEC when loading aof - 2 files changed, 14 insertions(+), 3 deletions(-) - -antirez in commit 2007d30c: - Refactoring of XADD / XTRIM MAXLEN rewriting. - 1 file changed, 15 insertions(+), 22 deletions(-) - -zhaozhao.zz in commit 6a298110: - Streams: add test cases for XADD/XTRIM maxlen - 1 file changed, 46 insertions(+) - -zhaozhao.zz in commit 041161b7: - Streams: propagate specified MAXLEN instead of approximated - 1 file changed, 35 insertions(+), 6 deletions(-) - -zhaozhao.zz in commit f04d799b: - Streams: reset approx_maxlen in every maxlen loop - 1 file changed, 2 insertions(+) - -zhaozhao.zz in commit affd9365: - Streams: XTRIM will return an error if MAXLEN with a count < 0 - 1 file changed, 6 insertions(+), 1 deletion(-) - -zhaozhao.zz in commit 4c405ad0: - Streams: propagate original MAXLEN argument in XADD context - 1 file changed, 3 insertions(+), 12 deletions(-) - -antirez in commit 5c6d4b4a: - Fix typo in replicationCron() comment. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit a67a8dbf: - Fix typo in design comment of bio.c. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit c4ab5a05: - xclaimCommand(): fix comment typos. - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit dc0b628a: - streamAppendItem(): Update the radix tree pointer only if changed. - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 4566fbc7: - Listpack: optionally force reallocation on inserts. - 1 file changed, 20 insertions(+) - -antirez in commit 5eca170c: - Fix printf type mismatch in genRedisInfoString(). - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 260b53a2: - streamIteratorRemoveEntry(): set back lp only if pointer changed. - 1 file changed, 2 insertions(+), 1 deletion(-) - -zhaozhao.zz in commit 5d12f9d9: - Streams: update listpack with new pointer in XDEL - 1 file changed, 3 insertions(+) - -zhaozhao.zz in commit 6b7ad838: - bugfix: replace lastcmd with cmd when rewrite BRPOPLPUSH as RPOPLPUSH - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit 3454a043: - script cache memory in INFO and MEMORY includes both script code and overheads - 2 files changed, 3 insertions(+), 3 deletions(-) - -Oran Agra in commit d6aeca86: - fix #5024 - commandstats for multi-exec were logged as EXEC. - 2 files changed, 63 insertions(+), 2 deletions(-) - -antirez in commit a996b2a2: - Fix XINFO comment for consistency. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Bruce Merry in commit 1a8447b6: - Fix invalid use of sdsZmallocSize on an embedded string - 1 file changed, 1 insertion(+), 1 deletion(-) - -Bruce Merry in commit 8dde46ad: - Fix incorrect memory usage accounting in zrealloc - 3 files changed, 24 insertions(+), 2 deletions(-) - -Hamid Alaei in commit b362a1b7: - fix dict get on not found - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 55e9df8a: - Try to avoid issues with GCC pragmas and older compilers. - 1 file changed, 7 insertions(+), 4 deletions(-) - -antirez in commit b0d22702: - Modules: hellodict example WIP #3: KEYRANGE. - 1 file changed, 40 insertions(+) - -antirez in commit af2f6682: - Modules: Modules: dictionary API WIP #13: Compare API exported. - 2 files changed, 6 insertions(+) - -antirez in commit f9a3e6ef: - Modules: Modules: dictionary API WIP #12: DictCompare API. - 1 file changed, 8 insertions(+) - -antirez in commit 01e0341a: - Modules: Modules: dictionary API WIP #11: DictCompareC API. - 1 file changed, 18 insertions(+) - -antirez in commit f9b3ce9a: - Modules: hellodict example WIP #1: GET command. - 1 file changed, 18 insertions(+) - -antirez in commit 36e66d86: - Modules: hellodict example WIP #1: SET command. - 1 file changed, 74 insertions(+) - -antirez in commit e33fdbe8: - Modules: remove useless defines in hellotimer.c - 2 files changed, 6 insertions(+), 4 deletions(-) - -antirez in commit 1c8b2248: - Modules: fix top comment of hellotimer.c - 1 file changed, 1 insertion(+), 1 deletion(-) - -Guy Korland in commit 7ded552d: - add missing argument to function doc - 1 file changed, 1 insertion(+), 1 deletion(-) - -Pavel Skuratovich in commit f92b3273: - Fix typo in comment - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 57b6c343: - Modules: dictionary API WIP #10: export API to modules. - 2 files changed, 60 insertions(+) - -antirez in commit 3f82e59c: - Modules: dictionary API WIP #9: iterator returning string object. - 1 file changed, 23 insertions(+), 6 deletions(-) - -antirez in commit 6a73aca3: - Modules: dictionary API WIP #8: Iterator next/prev. - 1 file changed, 42 insertions(+) - -antirez in commit ef8413db: - Modules: dictionary API WIP #7: don't store the context. - 1 file changed, 7 insertions(+), 8 deletions(-) - -antirez in commit 05579e38: - Modules: dictionary API WIP #6: implement automatic memory management. - 1 file changed, 21 insertions(+), 7 deletions(-) - -antirez in commit 11c53f8c: - Modules: dictionary API work in progress #5: rename API for consistency. - 1 file changed, 25 insertions(+), 25 deletions(-) - -antirez in commit 0bd7091b: - Modules: change RedisModuleString API to allow NULL context. - 1 file changed, 33 insertions(+), 12 deletions(-) - -antirez in commit 5fc16f17: - Modules: dictionary API work in progress #4: reseek API. - 1 file changed, 25 insertions(+), 6 deletions(-) - -antirez in commit 45b7f779: - Modules: dictionary API work in progress #3: Iterator creation. - 1 file changed, 41 insertions(+), 1 deletion(-) - -antirez in commit 8576b0ae: - Modules: dictionary API work in progress #2: Del API. - 1 file changed, 17 insertions(+), 2 deletions(-) - -antirez in commit 4b0fa7a7: - Modules: dictionary API work in progress #1. - 2 files changed, 95 insertions(+), 1 deletion(-) - -antirez in commit 28210760: - Module cluster flags: use RM_SetClusterFlags() in the example. - 2 files changed, 11 insertions(+) - -antirez in commit 18c5ab93: - Module cluster flags: add RM_SetClusterFlags() API. - 3 files changed, 33 insertions(+) - -antirez in commit 4ce6bff2: - Module cluster flags: add hooks for NO_FAILOVER flag. - 1 file changed, 4 insertions(+), 2 deletions(-) - -antirez in commit 2ba52889: - Module cluster flags: add hooks for NO_REDIRECTION flag. - 3 files changed, 14 insertions(+), 4 deletions(-) - -antirez in commit 6a39ece6: - Module cluster flags: initial vars / defines added. - 5 files changed, 20 insertions(+) - -antirez in commit 0ff35370: - Modules: rename the reused static client to something more general. - 1 file changed, 10 insertions(+), 8 deletions(-) - -antirez in commit 2d11ee95: - Modules: associate a fake client to timer context callback. - 1 file changed, 2 insertions(+) - -antirez in commit 851b2ed3: - Modules: associate a fake client to cluster message context callback. - 1 file changed, 2 insertions(+) - -artix in commit 148e4911: - Cluster Manager: clusterManagerFixOpenSlot now counts node's keys in slot if node is neither migrating nor importing. - 1 file changed, 20 insertions(+), 1 deletion(-) - -Guy Korland in commit 8afca145: - No need to return "OK" - 1 file changed, 1 insertion(+), 1 deletion(-) - -Guy Korland in commit 9a278db2: - typo fix - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 26479036: - Revert "fix repeat argument issue and reduce unnessary loop times for redis-cli." - 1 file changed, 7 insertions(+), 12 deletions(-) - -Guy Korland in commit 27b7fb5a: - Fix few typos - 1 file changed, 10 insertions(+), 10 deletions(-) - -Guy Korland in commit 233aa2d3: - RedisModule_HashSet call must end with NULL - 1 file changed, 3 insertions(+), 1 deletion(-) - -antirez in commit a8494072: - Sentinel: document how to undo a renamed command. - 1 file changed, 6 insertions(+), 1 deletion(-) - -antirez in commit 6c8a8f2e: - LOLWUT: split the command from version-specific implementations. - 3 files changed, 297 insertions(+), 241 deletions(-) - -antirez in commit 5c758406: - Slave removal: add a few forgotten aliases for CONFIG SET. - 1 file changed, 10 insertions(+) - -antirez in commit 2da823c4: - LOLWUT: add Redis version in the output. - 1 file changed, 3 insertions(+), 1 deletion(-) - -antirez in commit bfcba420: - LOLWUT: Ness -> Nees. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit efed898a: - LOLWUT: Limit maximum CPU effort. - 1 file changed, 5 insertions(+) - -antirez in commit eb0fbd71: - LOLWUT: change padding conditional to a more direct one. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Slobodan Mišković in commit ed08feb7: - Fix spelling descrive -> describe - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 2ffb4413: - LOLWUT: fix crash when col < 2. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 55dae693: - LOLWUT: fix structure typo in comment. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 9b3098b9: - LOLWUT: Fix license copyright year. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 263dbadc: - LOLWUT: increase the translation factor. - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit a622f6c0: - LOLWUT: change default size to fit a normal terminal better. - 1 file changed, 6 insertions(+), 6 deletions(-) - -antirez in commit 38b0d25a: - LOLWUT: wrap it into a proper command. - 4 files changed, 40 insertions(+), 15 deletions(-) - -antirez in commit 34ebd898: - LOLWUT: draw Schotter by Georg Nees. - 1 file changed, 47 insertions(+), 3 deletions(-) - -antirez in commit 46286e64: - LOLWUT: draw rotated squares using trivial trigonometry. - 1 file changed, 44 insertions(+) - -antirez in commit 2d4143fd: - LOLWUT: draw lines using Bresenham algorithm. - 1 file changed, 26 insertions(+), 2 deletions(-) - -antirez in commit 3546d9ce: - LOLWUT: Rendering of the virtual canvas to a string. - 1 file changed, 78 insertions(+), 7 deletions(-) - -antirez in commit b404a6ce: - LOLWUT: show the output verbatim in redis-cli. - 1 file changed, 1 insertion(+) - -antirez in commit e30ba94f: - LOLWUT: canvas structure and BSD license on top. - 1 file changed, 46 insertions(+) - -antirez in commit 9c771145: - LOLWUT: Emit Braille unicode according to pixel pattern. - 1 file changed, 23 insertions(+) +Upgrade urgency: SECURITY if you use 32bit build of redis (see bellow), MODERATE +if you used earlier versions of Redis 6.2, LOW otherwise. + +Integer overflow on 32-bit systems (CVE-2021-21309): +Redis 4.0 or newer uses a configurable limit for the maximum supported bulk +input size. By default, it is 512MB which is a safe value for all platforms. +If the limit is significantly increased, receiving a large request from a client +may trigger several integer overflow scenarios, which would result with buffer +overflow and heap corruption. + +Here is a comprehensive list of changes in this release compared to 6.2 RC3, +each one includes the PR number that added it, so you can get more details +at https://github.com/redis/redis/pull/ + +Bug fixes: +* Avoid 32-bit overflows when proto-max-bulk-len is set high (#8522) +* Fix broken protocol in client tracking tracking-redir-broken message (#8456) +* Avoid unsafe field name characters in INFO commandstats, errorstats, modules (#8492) +* XINFO able to access expired keys during CLIENT PAUSE WRITE (#8436) +* Fix allowed length for REPLCONF ip-address, needed due to Sentinel's support for hostnames (#8517) +* Fix broken protocol in redis-benchmark when used with -a or --dbnum (#8486) +* XADD counts deleted records too when considering switching to a new listpack (#8390) + +Bug fixes that are only applicable to previous releases of Redis 6.2: +* Fixes in GEOSEARCH bybox (accuracy and mismatch between width and height) (#8445) +* Fix risk of OOM panic in HRANDFIELD, ZRANDMEMBER commands with huge negative count (#8429) +* Fix duplicate replicas issue in Sentinel, needed due to hostname support (#8481) +* Fix Sentinel configuration rewrite, an improvement of #8271 (#8480) + +Command behavior changes: +* SRANDMEMBER uses RESP3 array type instead of set type (#8504) +* EXPIRE, EXPIREAT, SETEX, GETEX: Return error when provided expire time overflows (#8287) + +Other behavior changes: +* Remove ACL subcommand validation if fully added command exists. (#8483) -Jakub Vrana in commit 4a1d6c7d: - Slave removal: capitalize Replica - 2 files changed, 5 insertions(+), 5 deletions(-) - -antirez in commit 72e0368a: - Slave removal: remove slave from integration tests descriptions. - 8 files changed, 36 insertions(+), 36 deletions(-) - -antirez in commit c7841c2b: - Slave removal: remove slave from top-level tests descriptions. - 3 files changed, 12 insertions(+), 12 deletions(-) - -antirez in commit 1b9b19ba: - Slave removal: remove slave from object.c. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 7da266e6: - Slave removal: remove slave from the README. - 1 file changed, 7 insertions(+), 7 deletions(-) - -antirez in commit 93d803c9: - Slave removal: server.c logs fixed. - 1 file changed, 5 insertions(+), 5 deletions(-) - -antirez in commit 89434032: - Slave removal: remove slave from sentinel.conf when possible. - 1 file changed, 18 insertions(+), 18 deletions(-) - -antirez in commit 7673d88d: - Slave removal: replace very few things in Sentinel. - 1 file changed, 12 insertions(+), 8 deletions(-) - -antirez in commit f1de29b3: - Slave removal: scripting.c logs and other stuff fixed. - 1 file changed, 6 insertions(+), 2 deletions(-) - -antirez in commit 53fe558e: - Slave removal: replication.c logs fixed. - 1 file changed, 35 insertions(+), 35 deletions(-) - -antirez in commit c92b02dd: - Slave removal: networking.c logs fixed. - 1 file changed, 5 insertions(+), 5 deletions(-) - -antirez in commit be76ed0c: - Slave removal: blocked.c logs fixed. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 3fd73151: - Slave removal: Make obvious in redis.conf what a replica is. - 1 file changed, 5 insertions(+) - -antirez in commit a22168e4: - Slave removal: slave mode -> replica mode text in redis-cli. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 0e222fbe: - Slave removal: fix typo of replicaof. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 34a5615e: - Slave removal: slave -> replica in redis.conf and output buffer option. - 3 files changed, 132 insertions(+), 129 deletions(-) - -antirez in commit 1d2fcf6f: - Slave removal: Convert cluster.c log messages and command names. - 1 file changed, 12 insertions(+), 11 deletions(-) - -antirez in commit 2546158d: - Slave removal: config.c converted + config rewriting hacks. - 1 file changed, 117 insertions(+), 38 deletions(-) - -antirez in commit c0952c0d: - Slave removal: redis-cli --slave -> --replica. - 1 file changed, 3 insertions(+), 1 deletion(-) - -antirez in commit 1f37f1dd: - Slave removal: SLAVEOF -> REPLICAOF. SLAVEOF is now an alias. - 3 files changed, 4 insertions(+), 3 deletions(-) - -Amin Mesbah in commit 7928f578: - Use geohash limit defines in constraint check - 1 file changed, 2 insertions(+), 2 deletions(-) - -Jeffrey Lovitz in commit bb2bed78: - CLI Help text loop verifies arg count - 1 file changed, 1 insertion(+), 1 deletion(-) - -youjiali1995 in commit 246980d0: - sentinel: fix randomized sentinelTimer. - 1 file changed, 1 insertion(+), 3 deletions(-) +Improvements: +* Optimize sorting in GEORADIUS / GEOSEARCH with COUNT (#8326) +* Optimize HRANDFIELD and ZRANDMEMBER case 4 when ziplist encoded (#8444) +* Optimize in-place replacement of elements in HSET, HINCRBY, LSET (#8493) +* Remove redundant list to store pubsub patterns (#8472) +* Add --insecure option to command line tools (#8416) -youjiali1995 in commit fa7de8c4: - bio: fix bioWaitStepOfType. - 1 file changed, 3 insertions(+), 3 deletions(-) +Info fields and introspection changes: +* Add INFO fields to track progress of BGSAVE, AOFRW, replication (#8414) -Weiliang Li in commit 7642f9d5: - fix usage typo in redis-cli - 1 file changed, 1 insertion(+), 1 deletion(-) +Modules: +* RM_ZsetRem: Delete key if empty, the bug could leave empty zset keys (#8453) +* RM_HashSet: Add COUNT_ALL flag and set errno (#8446) ================================================================================ -Redis 5.0 RC5 Released Thu Sep 06 12:54:29 CEST 2018 +Redis 6.2 RC3 Released Tue Feb 1 14:00:00 IST 2021 ================================================================================ -Upgrade urgency HIGH: Several imporant bugs fixed. - -Hi all, - -This is the release candidate number five, and has a lot of bug fixes inside, -together with a few big changes to the Redis behavior from the point of view -of replication of scripts and handling of the maxmemory directive in slaves. -Make sure to read the whole list! - -* BREAKING BEHAVIOR: Slaves now ignore maxmemory by default. -* BREAKING BEHAVIOR: Now scripts are always replicated for their effects, and - never sending the script itself to slaves/AOF. -* Improvement: Big pipelining performances improved significantly. -* Fix: Rewrite BRPOPLPUSH as RPOPLPUSH to propagate. -* Fix: False positives in tests. -* Fix: Certain command flags were modified because not correct. -* Fix: Fix blocking XREAD for streams that are empty. -* Improvement: Allow scripts to timeout on slaves as well. -* Fix: Different corner cases due to CLIENT PAUSE are now fixed. -* Improvement: Optimize parsing large bulk greater than 32k. -* Fix: Propagate read-only scripts as SCRIPT LOAD, not as EVAL. - -The following is the list of commits, so that you can read the details and -check the credits of the commits. - -antirez in commit 1d1bf7f0: - Document that effects replication is Redis 5 default. - 1 file changed, 8 insertions(+) - -antirez in commit cfd969c7: - Fix scripting tests now that we default to commands repl. - 1 file changed, 8 insertions(+), 1 deletion(-) - -antirez in commit 3e1fb5ff: - Use commands (effects) replication by default in scripts. - 3 files changed, 8 insertions(+), 1 deletion(-) - -antirez in commit c6c71abe: - Safer script stop condition on OOM. - 1 file changed, 5 insertions(+), 2 deletions(-) - -antirez in commit dfbce91a: - Propagate read-only scripts as SCRIPT LOAD. - 1 file changed, 16 insertions(+), 3 deletions(-) - -antirez in commit 1705e42e: - Don't perform eviction when re-entering the event loop. - 1 file changed, 7 insertions(+), 2 deletions(-) - -antirez in commit a0dd6f82: - Clarify why remaining may be zero in readQueryFromClient(). - 1 file changed, 2 insertions(+) - -zhaozhao.zz in commit 2eed31a5: - networking: fix unexpected negative or zero readlen - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 37fb606c: - Merge branch '5.0' of github.com:/antirez/redis into 5.0 -zhaozhao.zz in commit 1898e6ce: - networking: optimize parsing large bulk greater than 32k - 1 file changed, 13 insertions(+), 10 deletions(-) - -antirez in commit 82fc63d1: - Unblocked clients API refactoring. See #4418. - 4 files changed, 33 insertions(+), 15 deletions(-) - -zhaozhao.zz in commit 839bb52c: - if master is already unblocked, do not unblock it twice - 1 file changed, 1 insertion(+), 1 deletion(-) - -zhaozhao.zz in commit 2e1cd82d: - fix multiple unblock for clientsArePaused() - 1 file changed, 3 insertions(+), 3 deletions(-) - -antirez in commit 17233080: - Make pending buffer processing safe for CLIENT_MASTER client. - 3 files changed, 22 insertions(+), 13 deletions(-) - -antirez in commit 42bce87a: - Test: processing of master stream in slave -BUSY state. - 1 file changed, 44 insertions(+) - -antirez in commit 8bf42f60: - After slave Lua script leaves busy state, re-process the master buffer. - 2 files changed, 5 insertions(+), 2 deletions(-) - -antirez in commit c2b104c7: - While the slave is busy, just accumulate master input. - 2 files changed, 6 insertions(+), 1 deletion(-) - -antirez in commit 7b75f4ae: - Allow scripts to timeout even if from the master instance. - 1 file changed, 6 insertions(+), 11 deletions(-) - -antirez in commit adc4e031: - Allow scripts to timeout on slaves as well. - 2 files changed, 10 insertions(+), 3 deletions(-) - -dejun.xdj in commit 20ec1f0c: - Revise the comments of latency command. - 1 file changed, 2 insertions(+), 1 deletion(-) - -Chris Lamb in commit 8e5423eb: - Correct "did not received" -> "did not receive" typos/grammar. - 6 files changed, 10 insertions(+), 10 deletions(-) - -zhaozhao.zz in commit 395063d7: - remove duplicate bind in sentinel.conf - 1 file changed, 10 deletions(-) - -Salvatore Sanfilippo in commit b221ca41: - Merge pull request #5300 from SaschaRoland/xread-block-5299 -Sascha Roland in commit eea0d3c5: - #5299 Fix blocking XREAD for streams that ran dry - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit 4cb9ee11: - Add maxmemory slave behavior change in the change log. - 1 file changed, 8 insertions(+) - -zhaozhao.zz in commit 5ad888ba: - Supplement to PR #4835, just take info/memory/command as random commands - 1 file changed, 3 insertions(+), 3 deletions(-) +Upgrade urgency LOW: This is the third Release Candidate of Redis 6.2. -zhaozhao.zz in commit d928487f: - some commands' flags should be set correctly, issue #4834 - 1 file changed, 14 insertions(+), 14 deletions(-) +Here is a comprehensive list of changes in this release compared to 6.2 RC2, +each one includes the PR number that added it, so you can get more details +at https://github.com/redis/redis/pull/ -Oran Agra in commit af675f0a: - Fix unstable tests on slow machines. - 3 files changed, 23 insertions(+), 17 deletions(-) +New commands / args: +* Add HRANDFIELD and ZRANDMEMBER commands (#8297) +* Add FAILOVER command (#8315) +* Add GETEX, GETDEL commands (#8327) +* Add PXAT/EXAT arguments to SET command (#8327) +* Add SYNC arg to FLUSHALL and FLUSHDB, and ASYNC/SYNC arg to SCRIPT FLUSH (#8258) -antirez in commit f2cd16be: - Document slave-ignore-maxmemory in redis.conf. - 1 file changed, 20 insertions(+) +Sentinel: +* Add hostname support to Sentinel (#8282) +* Prevent file descriptors from leaking into Sentinel scripts (#8242) +* Fix config file line order dependency and config rewrite sequence (#8271) -antirez in commit 02d729b4: - Make slave-ignore-maxmemory configurable. - 1 file changed, 9 insertions(+) +New configuration options: +* Add set-proc-title config option to disable changes to the process title (#3623) +* Add proc-title-template option to control what's shown in the process title (#8397) +* Add lazyfree-lazy-user-flush config option to control FLUSHALL, FLUSHDB and SCRIPT FLUSH (#8258) -antirez in commit 447da44d: - Introduce repl_slave_ignore_maxmemory flag internally. - 3 files changed, 7 insertions(+) +Bug fixes: +* AOF: recover from last write error by turning on/off appendonly config (#8030) +* Exit on fsync error when the AOF fsync policy is 'always' (#8347) +* Avoid assertions (on older kernels) when testing arm64 CoW bug (#8405) +* CONFIG REWRITE should honor umask settings (#8371) +* Fix firstkey,lastkey,step in COMMAND command for some commands (#8367) -antirez in commit 868b2925: - Better variable meaning in processCommand(). - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit 319f2ee6: - Re-apply rebased #2358. - 1 file changed, 1 insertion(+), 1 deletion(-) - -zhaozhao.zz in commit 22c166da: - block: format code - 1 file changed, 2 insertions(+), 2 deletions(-) - -zhaozhao.zz in commit c03c5913: - block: rewrite BRPOPLPUSH as RPOPLPUSH to propagate - 3 files changed, 5 insertions(+), 1 deletion(-) - -zhaozhao.zz in commit fcd5ef16: - networking: make setProtocolError simple and clear - 1 file changed, 11 insertions(+), 13 deletions(-) - -zhaozhao.zz in commit 656e4b2f: - networking: just move qb_pos instead of sdsrange in processInlineBuffer - 1 file changed, 2 insertions(+), 3 deletions(-) - -zhaozhao.zz in commit 2c7972ce: - networking: just return C_OK if multibulk processing saw a <= 0 length. - 1 file changed, 2 insertions(+), 5 deletions(-) - -zhaozhao.zz in commit 1203a04f: - adjust qbuf to 26 in test case for client list - 1 file changed, 1 insertion(+), 1 deletion(-) - -zhaozhao.zz in commit aff86fa1: - pipeline: do not sdsrange querybuf unless all commands processed - 2 files changed, 48 insertions(+), 40 deletions(-) - -Chris Lamb in commit 45a6c5be: - Use SOURCE_DATE_EPOCH over unreproducible uname + date calls. - 1 file changed, 3 insertions(+) - -Chris Lamb in commit 186df148: - Make some defaults explicit in the sentinel.conf for package maintainers - 1 file changed, 25 insertions(+) - -dejun.xdj in commit b59f04a0: - Streams: ID of xclaim command starts from the sixth argument. - 1 file changed, 1 insertion(+), 1 deletion(-) - -shenlongxing in commit a3f2437b: - Fix stream command paras - 2 files changed, 7 insertions(+), 7 deletions(-) - -antirez in commit df911235: - Fix AOF comment to report the current behavior. - 1 file changed, 3 insertions(+), 1 deletion(-) - - - -================================================================================ -Redis 5.0 RC4 Released Fri Aug 03 13:51:02 CEST 2018 -================================================================================ - -Upgrade urgency - - HIGH: Many non critical but important issues fixed. - CRITICAL for Stream users: Many important bugs fixed. - -Hi all, welcome to Redis 5.0 RC4. - -This release is a huge step forward in Redis 5 maturity and fixes a number -of issues. It also provides interesting improvements. Here I'll summarize -the biggest one, but laster you can find the full list of commits: - -Fixes: - -* A number of fixes related to Streams: stability and correctnes. -* Fix dbRandomKey() potential infinite loop. -* Improve eviction LFU/LRU when keys are created by INCR commands family. -* Active defragmentation is now working on Redis 5. -* Fix corner case in Redis CLuster / Sentinel failover, by resetting the - disconnection time with master in a more appropriate place. -* Use a private version of localtime() to avoid potential deadlocks. -* Different redis-cli non critical fixes. -* Fix rare replication stream corruption with disk-based replication. +Special considerations: +* Fix misleading description of the save configuration directive (#8337) Improvements: +* A way to get RDB file via replication without excessive replication buffers (#8303) +* Optimize performance of clusterGenNodesDescription for large clusters (#8182) -* Sentinel: add an option to deny online script reconfiguration. -* Improved RESTORE command. -* Sentinel command renaming: allows to use Sentinel with Redis instances - that have non standard command names. -* CLIENT ID and CLIENT UNBLOCK. -* CLIENT LIST now supports a TYPE option. -* redis-cli --cluster now supports authentication. -* redis-trib is now deprecated (use redis-cli --cluster). -* Better slaves output buffers efficiency. -* Faster INFO when there are many clients connected. -* Dynamic HZ feature. -* Improvements in what the MEMORY command is able to report. -* Add year in log. (WARNING: may be incompatible with log scraping tools) -* Lazy freeing now works even when values are overwritten (for instance SET). -* Faster ZADD when elements scores are updated. -* Improvements to the test suite, including many new options. - -antirez in commit a4d1201e: - Test suite: add --loop option. - 1 file changed, 12 insertions(+), 5 deletions(-) - -antirez in commit 273d8191: - Test suite: new --stop option. - 1 file changed, 13 insertions(+), 4 deletions(-) - -antirez in commit fbbcc6a6: - Streams IDs parsing refactoring. - 1 file changed, 32 insertions(+), 17 deletions(-) - -antirez in commit 70c4bcb7: - Test: new sorted set skiplist order consistency. - 1 file changed, 26 insertions(+) - -antirez in commit 63addc5c: - Fix zslUpdateScore() edge case. - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit 724740cc: - More commenting of zslUpdateScore(). - 1 file changed, 2 insertions(+) - -antirez in commit ddc87eef: - Explain what's the point of zslUpdateScore() in top comment. - 1 file changed, 5 insertions(+) - -antirez in commit 741f29ea: - Remove old commented zslUpdateScore() from source. - 1 file changed, 13 deletions(-) - -antirez in commit 20116836: - Optimize zslUpdateScore() as asked in #5179. - 1 file changed, 44 insertions(+) - -antirez in commit 8c297e8b: - zsetAdd() refactored adding zslUpdateScore(). - 1 file changed, 18 insertions(+), 7 deletions(-) - -dejun.xdj in commit bd2f3f6b: - Streams: rearrange the usage of '-' and '+' IDs in stream commands. - 1 file changed, 13 insertions(+), 13 deletions(-) - -dejun.xdj in commit c0c06b84: - Streams: add mmid_supp argument in streamParseIDOrReply(). - 1 file changed, 6 insertions(+), 2 deletions(-) - -antirez in commit ab237a8e: - Minor improvements to PR #5187. - 2 files changed, 13 insertions(+), 6 deletions(-) - -Oran Agra in commit 1ce3cf7a: - test suite conveniency improvements - 3 files changed, 79 insertions(+), 3 deletions(-) - -Oran Agra in commit 36622899: - add DEBUG LOG, to to assist test suite debugging - 1 file changed, 4 insertions(+) - -antirez in commit 83d4311a: - Cluster cron announce IP minor refactoring. - 1 file changed, 6 insertions(+), 3 deletions(-) - -shenlongxing in commit a633f8e1: - Fix cluster-announce-ip memory leak - 1 file changed, 3 insertions(+), 2 deletions(-) - -antirez in commit 24c45538: - Tranfer -> transfer typo fixed. - 1 file changed, 1 insertion(+), 1 deletion(-) - -zhaozhao.zz in commit c609f240: - refactor dbOverwrite to make lazyfree work - 4 files changed, 27 insertions(+), 12 deletions(-) - -antirez in commit 9e971739: - Refactoring: replace low-level checks with writeCommandsDeniedByDiskError(). - 2 files changed, 6 insertions(+), 13 deletions(-) - -antirez in commit 0e77cef0: - Fix writeCommandsDeniedByDiskError() inverted return value. - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit acfe9d13: - Better top comment for writeCommandsDeniedByDiskError(). - 1 file changed, 8 insertions(+), 1 deletion(-) - -antirez in commit 4e933e00: - Introduce writeCommandsDeniedByDiskError(). - 2 files changed, 24 insertions(+) - -WuYunlong in commit 41607dfd: - Consider aof write error as well as rdb in lua script. - 1 file changed, 14 insertions(+), 4 deletions(-) - -Salvatore Sanfilippo in commit 1d073a64: - Merge pull request #5168 from rpv-tomsk/issue-5033 -Guy Korland in commit 2db31fd4: - Few typo fixes - 1 file changed, 13 insertions(+), 13 deletions(-) - -antirez in commit 64242757: - Add year in log. - 1 file changed, 1 insertion(+), 1 deletion(-) +Info fields and introspection changes: +* SLOWLOG and LATENCY monitor include unblocking time of blocked commands (#7491) -antirez in commit 50be4a1f: - Document dynamic-hz in the example redis.conf. - 1 file changed, 16 insertions(+) - -antirez in commit 9a76472d: - Make dynamic hz actually configurable. - 1 file changed, 9 insertions(+) - -antirez in commit a330d06c: - Control dynamic HZ via server configuration. - 2 files changed, 13 insertions(+), 6 deletions(-) - -antirez in commit d42602ff: - Dynamic HZ: adapt cron frequency to number of clients. - 2 files changed, 17 insertions(+), 5 deletions(-) - -antirez in commit 7b5f0223: - Dynamic HZ: separate hz from the configured hz. - 3 files changed, 15 insertions(+), 9 deletions(-) - -antirez in commit 037b00de: - Remove useless conditional from emptyDb(). - 1 file changed, 1 deletion(-) - -antirez in commit 0e97ae79: - Make emptyDb() change introduced in #4852 simpler to read. - 1 file changed, 8 insertions(+), 3 deletions(-) - -zhaozhao.zz in commit f7740faf: - optimize flushdb, avoid useless loops - 1 file changed, 5 insertions(+), 2 deletions(-) - -zhaozhao.zz in commit 0c008376: - Streams: fix xdel memory leak - 1 file changed, 1 insertion(+) - -antirez in commit dc600a25: - Example the magic +1 in migrateCommand(). - 1 file changed, 4 insertions(+) - -antirez in commit d6827ab6: - Make changes of PR #5154 hopefully simpler. - 1 file changed, 10 insertions(+), 5 deletions(-) - -WuYunlong in commit 89ec1453: - Do not migrate already expired keys. - 1 file changed, 6 insertions(+), 2 deletions(-) - -Pavel Rochnyack in commit cd25ed17: - INFO CPU: higher precision of reported values - 1 file changed, 8 insertions(+), 8 deletions(-) - -antirez in commit 6bfb4745: - Streams: refactoring of next entry seek in the iterator. - 1 file changed, 11 insertions(+), 7 deletions(-) - -zhaozhao.zz in commit 4724548e: - Streams: skip master fileds only when we are going forward in streamIteratorGetID - 1 file changed, 8 insertions(+), 5 deletions(-) - -Oran Agra in commit 4b79fdf1: - fix slave buffer test suite false positives - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit a1e081f7: - string2ll(): better commenting. - 1 file changed, 6 insertions(+) - -dsomeshwar in commit 8b4fe752: - removing redundant check - 1 file changed, 3 deletions(-) - -antirez in commit 9e5bf047: - Restore string2ll() to original version. - 1 file changed, 7 insertions(+), 2 deletions(-) - -Oran Agra in commit c2ecdcde: - fix recursion typo in zmalloc_usable - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 4f742bd6: - string2ll(): remove duplicated check for special case. - 1 file changed, 1 insertion(+), 6 deletions(-) - -antirez in commit a4efac00: - string2ll(): test for NULL pointer in all the cases. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 2c07c107: - Change 42 to 1000 as warning level for cached scripts. - 1 file changed, 3 insertions(+), 3 deletions(-) - -Itamar Haber in commit 270903d6: - Adds Lua overheads to MEMORY STATS, smartens the MEMORY DOCTOR - 3 files changed, 30 insertions(+), 4 deletions(-) - -Itamar Haber in commit faf3dbfc: - Adds memory information about the script's cache to INFO - 3 files changed, 12 insertions(+) - -antirez in commit 49841a54: - Fix merge errors. - 2 files changed, 7 deletions(-) - -antirez in commit 77a7ec72: - Merge branch 'unstable' into 5.0 branch -antirez in commit 4ff47a0b: - Top comment clientsCron(). - 1 file changed, 19 insertions(+), 4 deletions(-) - -antirez in commit aba68552: - Clarify that clientsCronTrackExpansiveClients() indexes may jump ahead. - 1 file changed, 9 insertions(+), 1 deletion(-) - -antirez in commit be88c0b1: - Rename INFO CLIENT max buffers field names for correctness. - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit 0cf3794e: - Fix wrong array index variable in getExpansiveClientsInfo(). - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit ea3a20c5: - Change INFO CLIENTS sections to report pre-computed max/min client buffers. - 1 file changed, 5 insertions(+), 5 deletions(-) - -antirez in commit 8f7e496b: - Rename var in clientsCronTrackExpansiveClients() for clarity. - 1 file changed, 3 insertions(+), 3 deletions(-) - -antirez in commit 8d617596: - Implement a function to retrieve the expansive clients mem usage. - 1 file changed, 12 insertions(+) - -antirez in commit 85a1b4f8: - clientsCronTrackExpansiveClients() actual implementation. - 1 file changed, 14 insertions(+), 1 deletion(-) - -antirez in commit d4c5fc57: - clientsCronTrackExpansiveClients() skeleton and ideas. - 1 file changed, 23 insertions(+) - -antirez in commit 1c95c075: - Make vars used only by INFO CLIENTS local to the block. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Salvatore Sanfilippo in commit 16b8d364: - Merge pull request #4727 from kingpeterpaule/redis-fix-info-cli -antirez in commit 0aca977c: - Merge branch 'unstable' of github.com:/antirez/redis into unstable -antirez in commit 313b2240: - In addReplyErrorLength() only panic when replying to slave. - 1 file changed, 4 insertions(+), 3 deletions(-) - -antirez in commit 6183f059: - Refine comment in addReplyErrorLength() about replying to masters/slaves. - 1 file changed, 11 insertions(+) - -Salvatore Sanfilippo in commit 22e9321c: - Merge pull request #5138 from oranagra/improve_defrag_test -Oran Agra in commit f89c93c8: - make active defrag test more stable - 2 files changed, 6 insertions(+), 5 deletions(-) - -Salvatore Sanfilippo in commit 8213f64d: - Merge pull request #5122 from trevor211/allowWritesWhenAofDisabled -Salvatore Sanfilippo in commit 46fd9278: - Merge pull request #4237 from aspirewit/update-comment -antirez in commit 6201f7b4: - Streams: better error when $ is given with XREADGROUP. - 1 file changed, 5 insertions(+), 2 deletions(-) - -Salvatore Sanfilippo in commit 4bff45c7: - Merge pull request #5136 from 0xtonyxia/fix-xread-id-parse -antirez in commit afc7e08a: - Panic when we are sending an error to our master/slave. - 1 file changed, 5 insertions(+) - -Salvatore Sanfilippo in commit e03358c0: - Merge pull request #5135 from oranagra/rare_repl_corruption -dejun.xdj in commit 846cf12a: - Streams: remove meaningless if condition. - 1 file changed, 1 insertion(+), 1 deletion(-) - -dejun.xdj in commit 6501b6bb: - Streams: return an error message if using xreadgroup with '$' ID. - 1 file changed, 5 insertions(+) - -Oran Agra in commit d5559898: - fix rare replication stream corruption with disk-based replication - 3 files changed, 18 insertions(+), 9 deletions(-) - -antirez in commit cefe21d2: - dict.c: remove a few trailing spaces. - 1 file changed, 2 insertions(+), 2 deletions(-) - -Salvatore Sanfilippo in commit 4fc20992: - Merge pull request #5128 from kingpeterpaule/remove-one-loop-in-freeMemoryIfNeeded -Salvatore Sanfilippo in commit 9fbd49bb: - Merge pull request #5113 from 0xtonyxia/using-compare-func-instead -Salvatore Sanfilippo in commit cab39676: - Merge pull request #5127 from oranagra/sds_req_type -antirez in commit f9c84d6d: - Hopefully improve commenting of #5126. - 2 files changed, 22 insertions(+), 10 deletions(-) - -Salvatore Sanfilippo in commit e22a1218: - Merge pull request #5126 from oranagra/slave_buf_memory_2 -Salvatore Sanfilippo in commit 28dd8dd1: - Merge pull request #5132 from soloestoy/propagate-xdel-correctly -Oran Agra in commit bf680b6f: - slave buffers were wasteful and incorrectly counted causing eviction - 10 files changed, 182 insertions(+), 50 deletions(-) - -zhaozhao.zz in commit 73306c6f: - Streams: correctly propagate xdel if needed - 1 file changed, 7 insertions(+), 3 deletions(-) - -antirez in commit 103c5a1a: - Add a few comments to streamIteratorRemoveEntry(). - 1 file changed, 4 insertions(+) - -Salvatore Sanfilippo in commit a317f55d: - Merge pull request #5131 from soloestoy/optimize-xdel -antirez in commit 185e0d9c: - Modify XINFO field from last-id to last-generated-id. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Salvatore Sanfilippo in commit 4215e74b: - Merge pull request #5129 from soloestoy/xinfo-show-last-id -zhaozhao.zz in commit c9324f81: - Streams: free lp if all elements are deleted - 1 file changed, 9 insertions(+), 4 deletions(-) - -paule in commit b6ce7d5d: - Update dict.c - 1 file changed, 4 insertions(+), 2 deletions(-) - -zhaozhao.zz in commit b4ba5ac8: - Streams: show last id for streams and groups - 1 file changed, 6 insertions(+), 2 deletions(-) - -peterpaule in commit 816fc6cb: - remove one ineffective loop in dictGetSomeKeys. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit b05a22e2: - bugfix in sdsReqType creating 64bit sds headers on 32bit systems - 1 file changed, 3 insertions(+), 1 deletion(-) - -dejun.xdj in commit 491682a6: - Streams: using streamCompareID() instead of direct compare in block.c. - 1 file changed, 1 insertion(+), 4 deletions(-) - -dejun.xdj in commit a2177cd2: - Streams: add streamCompareID() declaration in stream.h. - 1 file changed, 1 insertion(+) - -dejun.xdj in commit 0484dbcf: - Streams: using streamCompareID() instead of direct compare. - 1 file changed, 2 insertions(+), 6 deletions(-) - -WuYunlong in commit 2d4366c5: - Accept write commands if persisting is disabled, event if we do have problems persisting on disk previously. - 1 file changed, 2 insertions(+), 1 deletion(-) - -Salvatore Sanfilippo in commit ab33bcd3: - Merge pull request #5120 from andrewsensus/leap-year-comment-patch-1 -antirez in commit 2352a519: - Test: XDEL fuzz testing. Remove and check stage. - 1 file changed, 15 insertions(+) - -antirez in commit 3d7d20b7: - Test: fix lshuffle by providing the "K" combinator. - 1 file changed, 2 insertions(+) - -antirez in commit 967ad364: - Test: add lshuffle in the Tcl utility functions set. - 1 file changed, 14 insertions(+) - -antirez in commit d6efd5fc: - Test: XDEL fuzz testing, stream creation. - 1 file changed, 20 insertions(+) - -andrewsensus in commit 8dc08ae2: - update leap year comment - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 69997153: - Merge branch 'unstable' of github.com:/antirez/redis into unstable -antirez in commit a93f8f09: - Test: XDEL basic test. - 1 file changed, 12 insertions(+) - -Salvatore Sanfilippo in commit a44a5934: - Merge pull request #5119 from trevor211/fixSlowlogConfig -WuYunlong in commit d6ba4fd5: - Fix config set slowlog-log-slower-than and condition in createLatencyReport - 2 files changed, 2 insertions(+), 2 deletions(-) - -WuYunlong in commit b3660be8: - Add test in slowlog.tcl - 1 file changed, 10 insertions(+) - -artix in commit d4182a0a: - Cluster Manager: more checks on --cluster-weight option. - 1 file changed, 12 insertions(+), 2 deletions(-) - -artix in commit d222eda9: - Redis-trib deprecated: it no longer works and it outputs a warning to the user. - 1 file changed, 103 insertions(+), 1804 deletions(-) - -artix in commit 513eb572: - Cluster Manager: auth support (-a argument). - 1 file changed, 41 insertions(+), 19 deletions(-) - -Salvatore Sanfilippo in commit f3980bb9: - Merge pull request #5115 from shenlongxing/patch-1 -Shen Longxing in commit c2a85fb3: - Delete unused role checking. - 1 file changed, 2 insertions(+), 6 deletions(-) - -Salvatore Sanfilippo in commit 4cb5bd4e: - Merge pull request #4820 from charpty/wip-serverc-simplify -antirez in commit 8d6b7861: - Add regression test for #5111. - 1 file changed, 15 insertions(+) - -antirez in commit b6260a02: - Streams: when re-delivering because of SETID, reset deliveries counter. - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit a7c180e5: - Simplify duplicated NACK #5112 fix. - 1 file changed, 18 insertions(+), 21 deletions(-) - -Salvatore Sanfilippo in commit bf4def0f: - Merge pull request #5112 from 0xtonyxia/fix-xreadgroup-crash-after-setid -Salvatore Sanfilippo in commit 16770551: - Merge pull request #5114 from oranagra/defrag_32 -Oran Agra in commit 920158ec: - Active defrag fixes for 32bit builds (again) - 1 file changed, 2 insertions(+), 2 deletions(-) - -Salvatore Sanfilippo in commit f45e7901: - Merge pull request #4967 from JingchengLi/unstable -tengfeng in commit 9505dd20: - fix repeat argument issue and reduce unnessary loop times for redis-cli. - 1 file changed, 12 insertions(+), 7 deletions(-) - -antirez in commit 0420c327: - Merge branch 'unstable' of github.com:/antirez/redis into unstable -antirez in commit 28e95c7c: - Streams: fix typo "consumer". - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit a8c1bb31: - Streams: fix new XREADGROUP sync logic. - 1 file changed, 13 insertions(+), 3 deletions(-) - -antirez in commit 1a02b5f6: - Streams: make blocking for > a truly special case. - 1 file changed, 29 insertions(+), 4 deletions(-) - -antirez in commit a71e8148: - Streams: send an error to consumers blocked on non-existing group. - 1 file changed, 5 insertions(+), 1 deletion(-) - -antirez in commit 09327f11: - Streams: fix unblocking logic into a consumer group. - 1 file changed, 24 insertions(+), 14 deletions(-) - -dejun.xdj in commit 3f8a3efe: - Streams: fix xreadgroup crash after xgroup SETID is sent. - 1 file changed, 20 insertions(+), 15 deletions(-) - -Salvatore Sanfilippo in commit 7239e9ca: - Merge pull request #5095 from 0xtonyxia/fix-indentation -dejun.xdj in commit 61f12973: - Bugfix: PEL is incorrect when consumer is blocked using xreadgroup with NOACK option. - 4 files changed, 6 insertions(+), 1 deletion(-) - -antirez in commit b67f0276: - redis-cli: fix #4990 additional argument in help. - 1 file changed, 1 insertion(+) - -antirez in commit 18d65849: - redis-cli: fix #5096 double error message. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 84620327: - redis-cli: cliConnect() flags CC_FORCE and CC_QUIET. - 1 file changed, 23 insertions(+), 13 deletions(-) - -Amit Dey in commit a3a5a25f: - fixing broken link in CONTRIBUTING - 1 file changed, 1 insertion(+), 1 deletion(-) - -dejun.xdj in commit 289d8d9c: - CLIENT UNBLOCK: fix client unblock help message. - 1 file changed, 1 insertion(+), 1 deletion(-) - -minkikim89 in commit 62a4a8c1: - fix whitespace in redis-cli.c - 1 file changed, 362 insertions(+), 362 deletions(-) - -WuYunlong in commit 0a5805d7: - fix compile warning in addReplySubcommandSyntaxError - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit cb78c842: - Use nolocks_localtime() for safer logging. - 2 files changed, 8 insertions(+), 2 deletions(-) - -antirez in commit 81778d91: - Cache timezone and daylight active flag for safer logging. - 2 files changed, 14 insertions(+), 1 deletion(-) - -antirez in commit 18d8205b: - Localtime: clarify is_leap_year() working with comments. - 1 file changed, 4 insertions(+), 4 deletions(-) - -antirez in commit 29644144: - Localtime: fix comment about leap year. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 0ea39aa4: - Localtime: fix daylight saving adjustment. Use * not +. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 6614d305: - Localtime: fix daylight time documentation and computation. - 1 file changed, 14 insertions(+), 4 deletions(-) - -dejun.xdj in commit 46f5a2ca: - Fix indentation. - 2 files changed, 14 insertions(+), 14 deletions(-) - -antirez in commit 243c5a7a: - Localtime: add a test main() function to check the output. - 1 file changed, 15 insertions(+), 1 deletion(-) - -antirez in commit c25ee35a: - Localtime: day of month is 1 based. Convert from 0 based "days". - 1 file changed, 2 insertions(+), 2 deletions(-) - -antirez in commit b9f33830: - Localtime: fix timezone adjustment. - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit 0c12cbed: - Localtime: compute year, month and day of the month. - 1 file changed, 26 insertions(+) - -antirez in commit 06ca400f: - Localtime: basics initial calculations. Year missing. - 1 file changed, 30 insertions(+), 2 deletions(-) - -antirez in commit 6a529067: - Localtime function skeleton and file added. - 1 file changed, 44 insertions(+) - -Jack Drogon in commit 93238575: - Fix typo - 40 files changed, 99 insertions(+), 99 deletions(-) - -antirez in commit 94b3ee61: - Clarify the pending_querybuf field of clients. - 1 file changed, 4 insertions(+), 3 deletions(-) - -antirez in commit 549b8b99: - Improve style of PR #5084. - 1 file changed, 8 insertions(+), 2 deletions(-) - -Salvatore Sanfilippo in commit 526b30a7: - Merge pull request #5084 from chendq8/pending-querybuf -antirez in commit 677d10b2: - Set repl_down_since to zero on state change. - 1 file changed, 2 insertions(+), 1 deletion(-) - -Salvatore Sanfilippo in commit 02e38516: - Merge pull request #5081 from trevor211/fixClusterFailover -chendianqiang in commit cbb2ac07: - Merge branch 'unstable' into pending-querybuf -antirez in commit 2edcafb3: - addReplySubSyntaxError() renamed to addReplySubcommandSyntaxError(). - 12 files changed, 14 insertions(+), 14 deletions(-) - -Salvatore Sanfilippo in commit bc6a0045: - Merge pull request #4998 from itamarhaber/module_command_help -Salvatore Sanfilippo in commit ee09b5ed: - Merge pull request #5071 from akshaynagpal/patch-2 -Salvatore Sanfilippo in commit f03ad962: - Merge pull request #5068 from shenlongxing/fix-rename-command -Salvatore Sanfilippo in commit e4881cd0: - Merge pull request #5090 from trevor211/test_helper_tcl -WuYunlong in commit 2833cfbe: - fix tests/test_helper.tcl with --wait-server option. Issue #5063 added --wait-server option, but can not work. - 1 file changed, 1 deletion(-) - -chendianqiang in commit 7de1ada0: - limit the size of pending-querybuf in masterclient - 4 files changed, 48 insertions(+) - -WuYunlong in commit 2e167f7d: - fix server.repl_down_since resetting, so that slaves could failover automatically as expected. - 1 file changed, 1 insertion(+), 1 deletion(-) - -WuYunlong in commit aeb7bc3e: - cluster.tcl: Add master consecutively down test. - 1 file changed, 77 insertions(+) - -antirez in commit d751d98b: - Change CLIENT LIST TYPE help string. - 1 file changed, 2 insertions(+), 2 deletions(-) - -Salvatore Sanfilippo in commit a0b05a04: - Merge pull request #5075 from soloestoy/client-list-types -Salvatore Sanfilippo in commit aa2c390e: - Merge pull request #5074 from soloestoy/fix-compile-warning -Salvatore Sanfilippo in commit a4ef94d2: - Merge pull request #5076 from 0xtonyxia/add-no-auth-warning-option -dejun.xdj in commit 9f185626: - Check if password is used on command line interface. - 1 file changed, 1 insertion(+), 1 deletion(-) - -dejun.xdj in commit 1139070a: - Fix trailing white space. - 1 file changed, 1 insertion(+), 1 deletion(-) - -dejun.xdj in commit bbd0ca95: - Fix code format issue. - 1 file changed, 4 insertions(+), 4 deletions(-) - -dejun.xdj in commit 7becf54e: - Don't output password warning message when --no-auth-warning is used. - 1 file changed, 10 insertions(+), 1 deletion(-) - -dejun.xdj in commit bde05e9c: - Avoid -Woverlength-strings compile warning. - 1 file changed, 5 insertions(+), 3 deletions(-) - -antirez in commit 5baf50d8: - Rax library updated (node callback). - 2 files changed, 19 insertions(+), 5 deletions(-) - -dejun.xdj in commit 0b74fd67: - Add --no-auth-warning help message. - 1 file changed, 2 insertions(+) - -zhaozhao.zz in commit b9cbd04b: - clients: add type option for client list - 4 files changed, 20 insertions(+), 6 deletions(-) - -zhaozhao.zz in commit f5538642: - clients: show pubsub flag in client list - 1 file changed, 1 insertion(+) - -zhaozhao.zz in commit 1fcf2737: - fix some compile warnings - 2 files changed, 2 insertions(+), 2 deletions(-) - -Akshay Nagpal in commit 007e3cbd: - Added link to Google Group - 1 file changed, 3 insertions(+), 1 deletion(-) - -antirez in commit ab55f9da: - Make CLIENT HELP output nicer to the eyes. - 1 file changed, 11 insertions(+), 11 deletions(-) - -antirez in commit 75f1a7bd: - Merge branch 'unstable' of github.com:/antirez/redis into unstable -antirez in commit 4a70ff74: - Add unblock in CLIENT HELP. - 1 file changed, 1 insertion(+) - -shenlongxing in commit 3c27db1c: - fix empty string for sentinel rename-command - 1 file changed, 5 insertions(+) - -Salvatore Sanfilippo in commit f7b21bc7: - Merge pull request #5066 from oranagra/defrag_jemalloc5_fix -Salvatore Sanfilippo in commit 730a4cfa: - Merge pull request #5067 from mpaltun/mpaltun-doc-fix -antirez in commit 2214043b: - CLIENT UNBLOCK: support unblocking by error. - 1 file changed, 22 insertions(+), 3 deletions(-) - -Mustafa Paltun in commit 010dc172: - Update t_stream.c - 1 file changed, 2 insertions(+), 2 deletions(-) - -Mustafa Paltun in commit 6d0acb33: - Update help.h - 1 file changed, 1 insertion(+), 1 deletion(-) - -Oran Agra in commit de495ee7: - minor fix in creating a stream NACK for rdb and defrag tests - 2 files changed, 2 insertions(+), 2 deletions(-) - -antirez in commit 71295ee3: - CLIENT UNBLOCK implemented. - 1 file changed, 22 insertions(+) - -antirez in commit fb39bfd7: - Take clients in a ID -> Client handle dictionary. - 3 files changed, 6 insertions(+) - -antirez in commit ed65d734: - CLIENT ID implemented. - 1 file changed, 4 insertions(+) - -Salvatore Sanfilippo in commit 345b4809: - Merge pull request #5063 from oranagra/test_suite_improvements -Salvatore Sanfilippo in commit 35c5f3fa: - Merge pull request #5065 from oranagra/defrag_jemalloc5 -Oran Agra in commit 5616d4c6: - add active defrag support for streams - 6 files changed, 230 insertions(+), 25 deletions(-) - -Oran Agra in commit e8099cab: - add defrag hint support into jemalloc 5 - 3 files changed, 43 insertions(+) - -Oran Agra in commit 751eea24: - test suite infra improvements and fix - 2 files changed, 19 insertions(+) - -Salvatore Sanfilippo in commit bb666d44: - Merge pull request #5027 from tigertv/unstable -antirez in commit b9058c73: - Merge branch 'unstable' of github.com:/antirez/redis into unstable -antirez in commit 43831779: - Sentinel: test command renaming feature. - 1 file changed, 10 insertions(+) - -Salvatore Sanfilippo in commit eb052ba9: - Merge pull request #5059 from guybe7/fix_restore_warning -antirez in commit 27178a3f: - Fix type of argslen in sendSynchronousCommand(). - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 1f1e724f: - Remove black space. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Salvatore Sanfilippo in commit aa5eaad4: - Merge pull request #5037 from madolson/repl-auth-fix -antirez in commit 3cf8dd2c: - Sentinel: fix SENTINEL SET error reporting. - 1 file changed, 18 insertions(+), 9 deletions(-) - -Madelyn Olson in commit 45731edc: - Addressed comments - 1 file changed, 1 insertion(+), 1 deletion(-) - -Madelyn Olson in commit e8d68b6b: - Fixed replication authentication with whitespace in password - 1 file changed, 12 insertions(+), 5 deletions(-) - -antirez in commit fc0c9c80: - Sentinel: drop the renamed-command entry in a more natural way. - 1 file changed, 4 insertions(+), 7 deletions(-) - -antirez in commit 8ba670f5: - Sentinel command renaming: document it into sentinel.conf. - 1 file changed, 19 insertions(+) - -antirez in commit a8a76bda: - Merge branch 'unstable' of github.com:/antirez/redis into unstable -antirez in commit 2358de68: - Sentinel command renaming: use case sensitive hashing for the dict. - 1 file changed, 2 insertions(+), 1 deletion(-) - -antirez in commit a9c50088: - Sentinel command renaming: fix CONFIG SET event logging. - 1 file changed, 17 insertions(+), 1 deletion(-) - -antirez in commit b72cecd7: - Sentinel command renaming: fix CONFIG SET after refactoring. - 1 file changed, 5 insertions(+), 5 deletions(-) - -antirez in commit 91a384a5: - Sentinel command renaming: implement SENTINEL SET. - 1 file changed, 19 insertions(+) - -antirez in commit 903582dd: - Sentinel: make SENTINEL SET able to handle different arities. - 1 file changed, 19 insertions(+), 12 deletions(-) - -antirez in commit c303e768: - Sentinel command renaming: config rewriting. - 1 file changed, 12 insertions(+) - -antirez in commit 60df7dbe: - Sentinel command renaming: rename-command option parsing. - 1 file changed, 11 insertions(+) - -antirez in commit 72e8a33b: - Sentinel command renaming: base machanism implemented. - 1 file changed, 64 insertions(+), 15 deletions(-) - -Guy Benoish in commit dfcc20f4: - Fix compiler warning in restoreCommand - 1 file changed, 1 insertion(+), 1 deletion(-) - -Salvatore Sanfilippo in commit cf7fcdbe: - Merge pull request #4634 from soloestoy/special-auth -Salvatore Sanfilippo in commit 70b7fa2c: - Merge pull request #5049 from youjiali1995/fix-load-rdb -Salvatore Sanfilippo in commit 54d66d39: - Merge pull request #5053 from michael-grunder/zpopminmax-keypos -Salvatore Sanfilippo in commit 199e704a: - Merge pull request #5050 from shenlongxing/fix-typo -michael-grunder in commit db6b99f9: - Update ZPOPMIN/ZPOPMAX command declaration - 1 file changed, 2 insertions(+), 2 deletions(-) - -Salvatore Sanfilippo in commit a16aa03a: - Merge pull request #5051 from oranagra/streams_mem_estimate -Oran Agra in commit 20e10dc7: - fix streams memory estimation, missing raxSeek - 1 file changed, 2 insertions(+), 1 deletion(-) - -shenlongxing in commit ec55df11: - fix typo - 4 files changed, 4 insertions(+), 4 deletions(-) - -youjiali1995 in commit df6644fe: - Fix rdbLoadIntegerObject() to create shared objects when needed. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Salvatore Sanfilippo in commit 1527bcad: - Merge pull request #5036 from bepahol/unstable -Salvatore Sanfilippo in commit c1e82405: - Merge pull request #5039 from oranagra/rdb_dbsize_hint -Salvatore Sanfilippo in commit 79f55eed: - Merge pull request #5040 from oranagra/memrev64ifbe_fix -Salvatore Sanfilippo in commit c6f4118c: - Merge pull request #5045 from guybe7/restore_fix -Guy Benoish in commit b5197f1f: - Enhance RESTORE with RDBv9 new features - 5 files changed, 100 insertions(+), 22 deletions(-) - -Salvatore Sanfilippo in commit c6fdebf5: - Merge pull request #5042 from oranagra/malloc_usable_size_libc -Oran Agra in commit 482785ac: - add malloc_usable_size for libc malloc - 2 files changed, 8 insertions(+), 3 deletions(-) - -Salvatore Sanfilippo in commit 4da29630: - Merge pull request #5023 from FX-HAO/unstable -antirez in commit e7219025: - Test RDB stream encoding saving/loading. - 1 file changed, 17 insertions(+) - -Salvatore Sanfilippo in commit 5f5e1199: - Merge pull request #5041 from oranagra/redis-rdb-check_rdbLoadMillisecondTime -antirez in commit 4848fbec: - Modules: convert hash to hash table for big objects. - 1 file changed, 3 insertions(+) - -Oran Agra in commit f31b0405: - fix redis-rdb-check to provide proper arguments to rdbLoadMillisecondTime - 2 files changed, 3 insertions(+), 2 deletions(-) - -antirez in commit 333c98c4: - AOF: remove no longer used variable "now". - 1 file changed, 1 deletion(-) - -antirez in commit e94b2053: - Modify clusterRedirectClient() to handle ZPOP and XREAD. - 1 file changed, 5 insertions(+), 1 deletion(-) - -Oran Agra in commit 26229aa6: - use safe macro (non empty) in memrev64ifbe to eliminate empty if warning - 1 file changed, 3 insertions(+), 3 deletions(-) - -Oran Agra in commit 5cd3c952: - 64 bit RDB_OPCODE_RESIZEDB in rdb saving - 1 file changed, 3 insertions(+), 7 deletions(-) - -antirez in commit ba92b517: - Remove AOF optimization to skip expired keys. - 1 file changed, 3 deletions(-) - -Benjamin Holst in commit 36524060: - Update README.md - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 44571088: - Merge branch 'unstable' of github.com:/antirez/redis into unstable -antirez in commit 6967d0bd: - Revert fix #4976 just leaving the flush() part. - 1 file changed, 1 insertion(+), 5 deletions(-) - -antirez in commit 0ed0dc3c: - Fix incrDecrCommand() to create shared objects when needed. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit bd92389c: - Refactor createObjectFromLongLong() to be suitable for value objects. - 2 files changed, 33 insertions(+), 2 deletions(-) - -Salvatore Sanfilippo in commit 3518bb66: - Merge pull request #5020 from shenlongxing/fix-config -antirez in commit 20766608: - Streams: fix xreadGetKeys() for correctness. - 1 file changed, 19 insertions(+), 5 deletions(-) - -Salvatore Sanfilippo in commit e670ccff: - Merge pull request #4857 from youjiali1995/fix-command-getkeys -antirez in commit a0b27dae: - Streams: fix xreadGetKeys() buffer overflow. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit 62f9ac6f: - Streams: Change XADD MAXLEN handling of values <= 0. - 1 file changed, 3 insertions(+), 3 deletions(-) - -Max Vetrov in commit d4c4f20a: - Update sort.c - 1 file changed, 1 insertion(+), 3 deletions(-) - -antirez in commit 79a1c19a: - XADD MAXLEN should return an error for values < 0. - 1 file changed, 5 insertions(+) - -Salvatore Sanfilippo in commit 2e0ab4a8: - Merge pull request #4976 from trevor211/fixDebugLoadaof -Salvatore Sanfilippo in commit 94658303: - Merge pull request #4758 from soloestoy/rdb-save-incremental-fsync -antirez in commit 6a66b93b: - Sentinel: add an option to deny online script reconfiguration. - 2 files changed, 41 insertions(+) - -antirez in commit d353023a: - Merge branch 'unstable' of github.com:/antirez/redis into unstable -antirez in commit d6e8fe77: - Fix infinite loop in dbRandomKey(). - 1 file changed, 13 insertions(+) - -Salvatore Sanfilippo in commit 40d5df65: - Merge pull request #5008 from zwkno1/unstable -Salvatore Sanfilippo in commit 8bc3ffcb: - Merge pull request #5021 from soloestoy/fix-exists -Salvatore Sanfilippo in commit 6c7847a1: - Merge pull request #5018 from soloestoy/optimize-reply -antirez in commit 1e92fde3: - Fix SCAN bug regression test, avoiding empty SREM call. - 1 file changed, 1 insertion(+), 1 deletion(-) - -Fuxin Hao in commit a4f658b2: - Fix update_zmalloc_stat_alloc in zrealloc - 1 file changed, 1 insertion(+), 1 deletion(-) +Modules: +* Add modules API for streams (#8288) +* Add event for fork child birth and termination (#8289) +* Add RM_BlockedClientMeasureTime* etc, to track background processing in commandstats (#7491) +* Fix bug in v6.2, wrong value passed to the new unlink callback (#8381) +* Fix bug in v6.2, modules blocked on keys unblock on commands like LPUSH (#8356) ================================================================================ -Redis 5.0 RC3 Released Wed Jun 14 9:51:44 CEST 2018 +Redis 6.2 RC2 Released Tue Jan 12 16:17:20 IST 2021 ================================================================================ -Upgrade urgency LOW: - -This release fixes compilation of Redis RC2. For an error the commit from unstable -updating the Rax library was not merged into the 5.0 branch. +Upgrade urgency LOW: This is the second Release Candidate of Redis 6.2. + +IMPORTANT: If you're running Redis on ARM64 or a big-endian system, upgrade may +have significant implications. Please be sure to read the notes below. + +Here is a comprehensive list of changes in this release compared to 6.2 RC1, +each one includes the PR number that added it, so you can get more details +at https://github.com/redis/redis/pull/ + +New commands / args: +* Add the REV, BYLEX and BYSCORE arguments to ZRANGE, and the ZRANGESTORE command (#7844) +* Add the XAUTOCLAIM command (#7973) +* Add the MINID trimming strategy and the LIMIT argument to XADD and XTRIM (#8169) +* Add the ANY argument to GEOSEARCH and GEORADIUS (#8259) +* Add the CH, NX, XX arguments to GEOADD (#8227) +* Add the COUNT argument to LPOP and RPOP (#8179) +* Add the WRITE argument to CLIENT PAUSE for pausing write commands exclusively (#8170) +* Change the proto-ver argument of HELLO to optional (#7377) +* Add the CLIENT TRACKINGINFO subcommand (#7309) + +Command behavior changes: +* CLIENT TRACKING yields an error when given overlapping BCAST prefixes (#8176) +* SWAPDB invalidates WATCHed keys (#8239) +* SORT command behaves differently when used on a writable replica (#8283) + +Other behavior changes: +* Avoid propagating MULTI/EXEC for read-only transactions (#8216) +* Remove the read-only flag from TIME, ECHO, ROLE, LASTSAVE (#8216) +* Fix the command flags of PFDEBUG (#8222) +* Tracking clients will no longer receive unnecessary key invalidation messages after FLUSHDB (#8039) +* Sentinel: Fix missing updates to the config file after SENTINEL SET command (#8229) + +Bug fixes with compatibility implications (bugs introduced in Redis 6.0): +* Fix RDB CRC64 checksum on big-endian systems (#8270) + If you're using big-endian please consider the compatibility implications with + RESTORE, replication and persistence. +* Fix wrong order of key/value in Lua's map response (#8266) + If your scripts use redis.setresp() or return a map (new in Redis 6.0), please + consider the implications. + +Bug fixes that are only applicable to previous releases of Redis 6.2: +* Resolve rare assertions in active defragmentation while loading (#8284, #8281) + +Bug fixes: +* Fix the selection of a random element from large hash tables (#8133) +* Fix an issue where a forked process deletes the parent's pidfile (#8231) +* Fix crashes when enabling io-threads-do-reads (#8230) +* Fix a crash in redis-cli after executing cluster backup (#8267) +* Fix redis-benchmark to use an IP address for the first cluster node (#8154) +* Fix saving of strings larger than 2GB into RDB files (#8306) + +Additional improvements: +* Improve replication handshake time (#8214) +* Release client tracking table memory asynchronously in cases where the DB is also freed asynchronously (#8039) +* Avoid wasteful transient memory allocation in certain cases (#8286, #5954) +* Handle binary string values by the 'requirepass' and 'masterauth' configs (#8200) + +Platform and deployment-related changes: +* Install redis-check-rdb and redis-check-aof as symlinks to redis-server (#5745) +* Add a check for an ARM64 Linux kernel bug (#8224) + Due to the potential severity of this issue, Redis will refuse to run on + affected platforms by default. + +Info fields and introspection changes: +* Add the errorstats section to the INFO command (#8217) +* Add the failed_calls and rejected_calls fields INFO's commandstats section (#8217) +* Report child copy-on-write metrics continuously (#8264) + +Module API changes: +* Add the RedisModule_SendChildCOWInfo API (#8264) +* Add the may-replicate command flag (#8170) ================================================================================ -Redis 5.0 RC2 Released Wed Jun 13 12:49:13 CEST 2018 +Redis 6.2 RC1 Released Mon Dec 14 11:50:00 IST 2020 ================================================================================ -Upgrade urgency CRITICAL: This release fixes important security issues. - HIGH: This release fixes a SCAN commands family bug. - MODERATE: This release fixes a PSYNC2 edge case with expires. - MODERATE: Sentinel related fixes. - LOW: All the other issues - -NOTE: This release breaks API compatibility with certain APIs that were -introduced in Redis 5.0 RC1. Notably ZPOP* now returns score/element in reverse -order. XINFO special form was removed, now XINFO STREAM must be -used to obtain general information about the stream. - -Redis 5.0 RC2 ixes a number of important issues: - -* Important security issues related to the Lua scripting engine. - Please check https://github.com/antirez/redis/issues/5017 - for more information. - -* A bug with SCAN, SSCAN, HSCAN and ZSCAN, that may not return all the elements. - We also add a regression test that can trigger the issue often when present, and - may in theory be able to find unrelated regressions. - -* A PSYNC2 bug is fixed: Redis should not expire keys when saving RDB files - because otherwise it is no longer possible to use such RDB file as a base - for partial resynchronization. It no longer represents the right state. - -* Compatibility of AOF with RDB preamble when the RDB checksum is disabled. - -* Sentinel bug that in some cases prevented Sentinel to detect that the master - was down immediately. A delay was added to the detection. - -* XREADGROUP would crash when the master had attacked slaves. - -* Replication and events generation of several streams command were fixed. - -* XREVRANGE fixed, in some cases it could not return elements, or crash the - server, or in general not behave correctly. - -* ZPOP can now unblock multiple clients in a sane way. - -* Other minor issues. - -Moreover this release adds new features: - -* XGROUP DESTROY and XGROUP SETID. - -* RDB loading speedup. - -* Configurable stream macro node limits (number of elements / bytes). - -* More smaller improvements. - -The following is the list of commits composing the release, please check -the commit messages and authors for credits. - -antirez in commit 9fdcc159: - Security: fix redis-cli buffer overflow. - 1 file changed, 16 insertions(+), 11 deletions(-) - -antirez in commit cf760071: - Security: fix Lua struct package offset handling. - 1 file changed, 6 insertions(+), 2 deletions(-) - -antirez in commit a57595ca: - Security: more cmsgpack fixes by @soloestoy. - 1 file changed, 7 insertions(+) - -antirez in commit 8783fb94: - Security: update Lua struct package for security. - 1 file changed, 23 insertions(+), 23 deletions(-) - -antirez in commit 8cb9344b: - Security: fix Lua cmsgpack library stack overflow. - 1 file changed, 3 insertions(+) +Upgrade urgency LOW: This is the first Release Candidate of Redis 6.2. + +Introduction to the Redis 6.2 release +===================================== + +This release is the first significant Redis release managed by the core team +under the new project governance model. + +Redis 6.2 includes many new commands and improvements, but no big features. It +mainly makes Redis more complete and addresses issues that have been requested +by many users frequently or for a long time. + +Many of these changes were not eligible for 6.0.x for several reasons: + +1. They are not backward compatible, which is always the case with new or + extended commands (that cannot be replicated to an older replica). +2. They require a longer release-candidate test cycle. + + +Here is a comprehensive list of changes in this release compared to 6.0.9, +each one includes the PR number that added it, so you can get more details +at https://github.com/redis/redis/pull/ + +New commands / args: +* Add SMISMEMBER command that checks multiple members (#7615) +* Add ZMSCORE command that returns an array of scores (#7593) +* Add LMOVE and BLMOVE commands that pop and push arbitrarily (#6929) +* Add RESET command that resets client connection state (#7982) +* Add COPY command that copies keys (#7953) +* Add ZDIFF and ZDIFFSTORE commands (#7961) +* Add ZINTER and ZUNION commands (#7794) +* Add GEOSEARCH/GEOSEARCHSTORE commands for bounding box spatial queries (#8094) +* Add GET parameter to SET command, for more powerful GETSET (#7852) +* Add exclusive range query to XPENDING (#8130) +* Add exclusive range query to X[REV]RANGE (#8072) +* Add GT and LT options to ZADD for conditional score updates (#7818) +* Add CLIENT INFO and CLIENT LIST for specific ids (#8113) +* Add IDLE argument to XPENDING command (#7972) +* Add local address to CLIENT LIST, and a CLIENT KILL filter. (#7913) +* Add NOMKSTREAM option to XADD command (#7910) +* Add command introspection to Sentinel (#7940) +* Add SENTINEL MYID subcommand (#7858) + +New features: +* Dump payload sanitization: prevent corrupt payload causing crashes (#7807) + Has flags to enable full O(N) validation (disabled by default). +* ACL patterns for Pub/Sub channels (#7993) +* Support ACL for Sentinel mode (#7888) +* Support getting configuration from both stdin and file at the same time (#7893) + Lets you avoid storing secrets on the disk. + +New features in CLI tools: +* redis-cli RESP3 push support (#7609) +* redis-cli cluster import support source and target that require auth (#7994) +* redis-cli URIs able to provide user name in addition to password (#8048) +* redis-cli/redis-benchmark allow specifying the prefered ciphers/ciphersuites (#8005) +* redis-cli add -e option to exit with code when command execution fails (#8136) + +Command behavior changes: +* EXISTS should not alter LRU (#8016) + In Redis 5.0 and 6.0 it would have touched the LRU/LFU of the key. +* OBJECT should not reveal logically expired keys (#8016) + Will now behave the same TYPE or any other non-DEBUG command. +* Improve db id range check for SELECT and MOVE (#8085) + Changes the error message text on a wrong db index. +* Modify AUTH / HELLO error message (#7648) + Changes the error message text when the user isn't found or is disabled. +* BITOPS length limited to proto_max_bulk_len rather than 512MB (#8096) + The limit is now configurable like in SETRANGE, and APPEND. +* GEORADIUS[BYMEMBER] can fail with -OOM if Redis is over the memory limit (#8107) + +Other behavior changes: +* Optionally (default) fail to start if requested bind address is not available (#7936) + If you rely on Redis starting successfully even if one of the bind addresses + is not available, you'll need to tune the new config. +* Limit the main db dictionaries expansion to prevent key eviction (#7954) + In the past big dictionary rehashing could result in massive data eviction. + Now this rehashing is delayed (up to a limit), which can result in performance + loss due to hash collisions. +* CONFIG REWRITE is atomic and safer, but requires write access to the config file's folder (#7824, #8051) + This change was already present in 6.0.9, but was missing from the release + notes. +* A new incremental eviction mechanism that reduces latency on eviction spikes (#7653) + In pathological cases this can cause memory to grow uncontrolled and may require + specific tuning. +* Not resetting "save" config when Redis is started with command line arguments. (#7092) + In case you provide command line arguments without "save" and count on it + being disabled, Now the defaults "save" config will kick in. +* Update memory metrics for INFO during loading (#7690) +* When "supervised" config is enabled, it takes precedence over "daemonize". (#8036) +* Assertion and panic, print crash log without generating SIGSEGV (#7585) +* Added crash log report on SIGABRT, instead of silently exiting (#8004) +* Disable THP (Transparent Huge Pages) if enabled (#7381) + If you deliberately enabled it, you'll need to config Redis to keep it. + +Bug fixes: +* Handle output buffer limits for module blocked clients (#8141) + Could result in a module sending reply to a blocked client to go beyond the + limit. +* Fix setproctitle related crashes. (#8150, #8088) + Caused various crashes on startup, mainly on Apple M1 chips or under + instrumentation. +* A module doing RM_Call could cause replicas to get nested MULTI (#8097). +* Backup/restore cluster mode keys to slots map for repl-diskless-load=swapdb (#8108) + In cluster mode with repl-diskless-load, when loading failed, slot map + wouldn't have been restored. +* Fix oom-score-adj-values range, and bug when used in config file (#8046) + Enabling setting this in the config file in a line after enabling it, would + have been buggy. +* Reset average ttl when empty databases (#8106) + Just causing misleading metric in INFO +* Disable rehash when Redis has child process (#8007) + This could have caused excessive CoW during BGSAVE, replication or AOFRW. +* Further improved ACL algorithm for picking categories (#7966) + Output of ACL GETUSER is now more similar to the one provided by ACL SETUSER. +* Fix bug with module GIL being released prematurely (#8061) + Could in theory (and rarely) cause multi-threaded modules to corrupt memory. +* Fix cluster redirect for module command with no firstkey. (#7539) +* Reduce effect of client tracking causing feedback loop in key eviction (#8100) +* Kill disk-based fork child when all replicas drop and 'save' is not enabled (#7819) +* Rewritten commands (modified for propagation) are logged as their original command (#8006) +* Fix cluster access to unaligned memory (SIGBUS on old ARM) #7958 +* If diskless repl child is killed, make sure to reap the child pid (#7742) +* Broadcast a PONG message when slot's migration is over, may reduce MOVED responses (#7571) + +Other improvements: +* TLS Support in redis-benchmark (#7959) +* Accelerate diskless master connections, and general re-connections (#6271) +* Run active defrag while blocked / loading (#7726) +* Performance and memory reporting improvement - sds take control of its internal fragmentation (#7875) +* Speedup cluster failover. (#7948) + +Platform / toolchain support related improvements: +* Optionally (not by default) use H/W Monotonic clock for faster time sampling (#7644) +* Remove the requirements for C11 and _Atomic supporting compiler (#7707) + This would allow to more easily build and use Redis on older systems and + compilers again. +* Fix crash log registers output on ARM. (#8020) +* Raspberry build fix. (#8095) +* Setting process title support for Haiku. (#8060) +* DragonFlyBSD RSS memory sampling support. (#8023) + +New configuration options: +* Enable configuring OpenSSL using the standard openssl.cnf (#8143) +* oom-score-adj-values config can now take absolute values (besides relative ones) (#8046) +* TLS: Add different client cert support. (#8076) +* Note that a few other changes listed above added their config options. + +Info fields and introspection changes: +* Add INFO fields to track diskless and disk-based replication progress (#7981) +* Add INFO field for main thread cpu time, and scrape system time. (#8132) +* Add total_forks to INFO STATS (#8155) +* Add maxclients and cluster_connections to INFO CLIENTS (#7979) +* Add tracking bcast flag and client redirection in client list (#7995) +* Fixed INFO client_recent_max_input_buffer includes argv array (#8065, see #7874) +* Note that a few other changes listed above added their info fields. + +Module API changes: +* Add CTX_FLAGS_DENY_BLOCKING as a unified the way to know if blocking is allowed (#8025) +* Add data type callbacks for lazy free effort, and unlink (#7912) +* Add data type callback for COPY command (#8112) +* Add callbacks for defrag support. (#8149) +* Add module event for repl-diskless-load swapdb (#8153) + +Module related fixes: +* Moved RMAPI_FUNC_SUPPORTED so that it's usable (#8037) +* Improve timer accuracy (#7987) +* Allow '\0' inside of result of RM_CreateStringPrintf (#6260) -赵磊 in commit 59080f60: - Fix dictScan(): It can't scan all buckets when dict is shrinking. - 1 file changed, 14 insertions(+), 11 deletions(-) - -dejun.xdj in commit ac2a824a: - Fix redis-cli memory leak when sending set preference command. - 1 file changed, 2 insertions(+) - -dejun.xdj in commit c7197ff5: - Check if the repeat value is positive in while loop of cliSendCommand(). - 1 file changed, 1 insertion(+), 1 deletion(-) - -dejun.xdj in commit 3f77777f: - Change the type of repeat argument to long for function cliSendCommand. - 1 file changed, 1 insertion(+), 1 deletion(-) - -dejun.xdj in commit 7a565d72: - Fix negtive repeat command value issue. - 1 file changed, 11 insertions(+), 3 deletions(-) - -dejun.xdj in commit 64bf60fb: - Detect and stop saving history for auth command with repeat option. - 1 file changed, 17 insertions(+), 10 deletions(-) - -dejun.xdj in commit 5bed12aa: - Change the warning message a little bit to avoid trademark issuses. - 1 file changed, 1 insertion(+), 1 deletion(-) - -dejun.xdj in commit d71c4961: - Stop saving auth command in redis-cli history. - 1 file changed, 4 insertions(+), 2 deletions(-) - -dejun.xdj in commit fca99e41: - Add warning message when using password on command line - 1 file changed, 1 insertion(+) - -antirez in commit 01407a3a: - Don't expire keys while loading RDB from AOF preamble. - 3 files changed, 5 insertions(+), 5 deletions(-) - -WuYunlong in commit fb5408cf: - Fix rdb save by allowing dumping of expire keys, so that when we add a new slave, and do a failover, eighter by manual or not, other local slaves will delete the expired keys properly. - 2 files changed, 3 insertions(+), 7 deletions(-) - -antirez in commit 0b8b6df4: - Backport hiredis issue 525 fix to compile on FreeBSD. - 1 file changed, 1 insertion(+), 1 deletion(-) - -antirez in commit e98627c5: - Add INIT INFO to the provided init script. - 1 file changed, 8 insertions(+) - -antirez in commit 17f5de89: - Fix ae.c when a timer finalizerProc adds an event. - 2 files changed, 10 insertions(+), 6 deletions(-) - -antirez in commit 266e6423: - Sentinel: fix delay in detecting ODOWN. - 1 file changed, 9 insertions(+), 5 deletions(-) - -zhaozhao.zz in commit eafaf172: - AOF & RDB: be compatible with rdbchecksum no - 1 file changed, 9 insertions(+), 7 deletions(-) - -huijing.whj in commit 4630da37: - fix int overflow problem in freeMemoryIfNeeded - 1 file changed, 1 insertion(+), 1 deletion(-) - -================================================================================ -Redis 5.0 RC1 Released Tue May 29 14:14:11 CEST 2018 -================================================================================ - -Upgrade urgency LOW: This is the first RC of Redis 5. - -Introduction to the Redis 5 release -=================================== - -Redis 5 is a release focused on a few important features. While Redis 4 -was very very focused on operations, Redis 5 changes are mostly user-facing, -with the implementation of new data types and operations on top of existing -types. The following are the major features of this release: - -1. The new Stream data type. https://redis.io/topics/streams-intro -2. New Redis modules APIs: Timers, Cluster and Dictionary APIs. -3. RDB now store LFU and LRU information. -4. The cluster manager was ported from Ruby (redis-trib.rb) to C code - inside redis-cli. Check `redis-cli --cluster help` for more info. -5. New sorted set commands: ZPOPMIN/MAX and blocking variants. -6. Active defragmentation version 2. -7. Improvemenets in HyperLogLog implementations. -8. Better memory reporting capabilities. -9. Many commands with sub-commands now have an HELP subcommand. -10. Better performances when clients connect and disconnect often. -11. Many bug fixes and other random improvements. -12. Jemalloc was upgraded to version 5.1 -13. CLIENT UNBLOCK and CLIENT ID. -14. The LOLWUT command was added. http://antirez.com/news/123 -15. We no longer use the "slave" word if not for API backward compatibility. -16. Differnet optimizations in the networking layer. -17. Lua improvements: - - Better propagation of Lua scripts to slaves / AOF. - - Lua scripts can now timeout and get in -BUSY state in the slave as well. -18. Dynamic HZ to balance idle CPU usage with responsiveness. -19. The Redis core was refactored and improved in many ways. Thanks to all the users and developers who made this release possible. We'll follow up with more RC releases, until the code looks production ready and we don't get reports of serious issues for a while. -A special thank you for the amount of work put into this release -(in decreasing number of commits) by: - -Fabio Nicotra, -Soloestoy -Itamar Haber -Oran Agra -Dvir Volk -dejun.xdj -Guy Benoish -Charsyam -Otmar Ertl -Jan-Erik Rediger -Spinlock - -Migrating from 4.0 to 5.0 +A special thank you for the amount of work put into this release by: +- Oran Agra +- Yossi Gottlieb +- Viktor Söderqvist +- Yang Bodong +- Filipe Oliveira +- Guy Benoish +- Itamar Haber +- Madelyn Olson +- Wang Yuan +- Felipe Machado +- Wen Hui +- Tatsuya Arisawa +- Jonah H. Harris +- Raghav Muddur +- Jim Brunner +- Yaacov Hazan +- Allen Farris +- Chen Yang +- Nitai Caro +- sundb +- Meir Shpilraien +- maohuazhu +- Valentino Geron +- Zhao Zhao +- Qu Chen +- George Prekas +- Tyson Andre +- Uri Yagelnik +- Michael Grunder +- Huang Zw +- alexronke-channeladvisor +- Andy Pan +- Wu Yunlong +- Wei Kukey +- Yoav Steinberg +- Greg Femec +- Uri Shachar +- Nykolas Laurentino de Lima +- xhe +- zhenwei pi +- David CARLIER + +Migrating from 6.0 to 6.2 ========================= -Redis 4.0 is mostly a strict subset of 5.0, you should not have any problem -upgrading your application from 4.0 to 5.0. However this is a list of small -non-backward compatible changes introduced in the 5.0 release: - -* redis-cli now implements the cluster management tool. We still ship the - old redis-trib, but new fixes will be implemented only in redis-cli. - See `redis-cli --cluster help` for more info. - -* The RDB format changed. Redis 5.0 is still able to read 4.0 (and all the - past versions) files, but not the other way around. - -* Certain log formats and sentences are different in Redis 5.0. - -* Now by default maxmemory is ignored by slaves, and used only once a slave - is promoted to master. It means that in setups where you want slaves to - enforce maxmemory in an independent way from the master (that will anyway - stream the key eviction DEL commands), you should active this feature manually - and make sure you understand that it breaks consistency if writes are not - always idempotent. TLDR: the new behavior is much better for 99.999% of use - cases, revert it if you really know what you are doing. - -* Scripts are only replicated by their *effects* and not by sending EVAL/EVALSHA - to slaves or the AOF log itself. This is much better in the general case - and in the future we want to totally remove the other possiblity of - propagating scripts the old way (as EVAL). However you can still turn this - back to the default via the non-documented (if not here) Redis configuration - directive "lua-replicate-commands yes" or - "DEBUG lua-always-replicate-commands 0". However note that Redis 6 may - completely remove such feature. - -* Because of the above change related to scripts replication, certain Redis - commands that in Redis 4 had their result ordered lexicographically before - being passed to Lua via the return value of redis.call(), now have a behavior - more similar to calling the commands directly from a normal client. For - instance the ordering of elements returned by SMEMBERS or SDIFF is now - undetermined in Lua, exactly as it is by default when calling the commands - from a non-scripting context. - --------------------------------------------------------------------------------- +Redis 6.2 is mostly a strict superset of 6.0, you should not have any problem +upgrading your application from 6.0 to 6.2. However there are some small changes +of behavior listed above, please make sure you are not badly affected by any of +them. -Credits: For each release, a list of changes with the relative author is -provided. Where not specified the implementation and design is done by -Salvatore Sanfilippo. Thanks to Redis Labs for making all this possible. -Also many thanks to all the other contributors and the amazing community -we have. +Specifically these sections: +* Command behavior changes +* Other behavior changes -Commit messages may contain additional credits. +-------------------------------------------------------------------------------- -Enjoy, -Salvatore +Cheers, +The Redis team diff --git a/redis.submodule/BUGS b/redis.submodule/BUGS index a8e9368..7af2593 100644 --- a/redis.submodule/BUGS +++ b/redis.submodule/BUGS @@ -1 +1 @@ -Please check https://github.com/antirez/redis/issues +Please check https://github.com/redis/redis/issues diff --git a/redis.submodule/CONDUCT b/redis.submodule/CONDUCT new file mode 100644 index 0000000..e0e15e2 --- /dev/null +++ b/redis.submodule/CONDUCT @@ -0,0 +1,96 @@ +Contributor Covenant Code of Conduct +Our Pledge +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. +Our Standards +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, +and learning from the experience +* Focusing on what is best not just for us as individuals, but for the +overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or +advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others’ private information, such as a physical or email +address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a +professional setting + +Enforcement Responsibilities +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. +Scope +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. +Enforcement +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +this email address: redis@redis.io. +All complaints will be reviewed and investigated promptly and fairly. +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. +Enforcement Guidelines +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: +1. Correction +Community Impact: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. +Consequence: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. +2. Warning +Community Impact: A violation through a single incident or series +of actions. +Consequence: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. +3. Temporary Ban +Community Impact: A serious violation of community standards, including +sustained inappropriate behavior. +Consequence: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. +4. Permanent Ban +Community Impact: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. +Consequence: A permanent ban from any sort of public interaction within +the community. +Attribution +This Code of Conduct is adapted from the Contributor Covenant, +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. +Community Impact Guidelines were inspired by Mozilla’s code of conduct +enforcement ladder. +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. \ No newline at end of file diff --git a/redis.submodule/CONTRIBUTING b/redis.submodule/CONTRIBUTING index 000edbe..50d5678 100644 --- a/redis.submodule/CONTRIBUTING +++ b/redis.submodule/CONTRIBUTING @@ -7,18 +7,42 @@ each source file that you contribute. # IMPORTANT: HOW TO USE REDIS GITHUB ISSUES -* Github issues SHOULD ONLY BE USED to report bugs, and for DETAILED feature - requests. Everything else belongs to the Redis Google Group: +Github issues SHOULD ONLY BE USED to report bugs, and for DETAILED feature +requests. Everything else belongs to the Redis Google Group: - https://groups.google.com/forum/m/#!forum/Redis-db + https://groups.google.com/forum/m/#!forum/Redis-db - PLEASE DO NOT POST GENERAL QUESTIONS that are not about bugs or suspected - bugs in the Github issues system. We'll be very happy to help you and provide - all the support in the mailing list. +PLEASE DO NOT POST GENERAL QUESTIONS that are not about bugs or suspected +bugs in the Github issues system. We'll be very happy to help you and provide +all the support in the mailing list. - There is also an active community of Redis users at Stack Overflow: +There is also an active community of Redis users at Stack Overflow: - http://stackoverflow.com/questions/tagged/redis + http://stackoverflow.com/questions/tagged/redis + +# Reporting Security Bugs + +*If you are reporting a security bug*, please contact the core team privately +by emailing redis@redis.io. Your report will be acknowledged by a core team +member and once the report has been reviewed you will receive a more detailed +response including next steps. + +If you do not receive a reply you can escalate to the Redis Google Group, +linked above. Because this group is a public space please do not disclose the +issue in detail, only say that you are trying to reach the core team for a +security issue. + +Redis follows a responsible disclosure process: + +1. Reports are reviewed and analyzed privately +2. Patches are prepared for supported versions of Redis +3. Vendor lists are notified with an embargo date to reduce the public impact +4. We push a fix release and your bug can be posted publicly with credit in + release notes and the version history (and our thanks!) + + Issues and pull requests for documentation belong on the redis-doc repo: + + https://github.com/redis/redis-doc # How to provide a patch for a new feature diff --git a/redis.submodule/COPYING b/redis.submodule/COPYING index ac68e01..a381681 100644 --- a/redis.submodule/COPYING +++ b/redis.submodule/COPYING @@ -1,4 +1,4 @@ -Copyright (c) 2006-2015, Salvatore Sanfilippo +Copyright (c) 2006-2020, Salvatore Sanfilippo All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/redis.submodule/README.md b/redis.submodule/README.md index 4b1a983..87225b3 100644 --- a/redis.submodule/README.md +++ b/redis.submodule/README.md @@ -3,22 +3,22 @@ This README is just a fast *quick start* document. You can find more detailed do What is Redis? -------------- -Redis is often referred as a *data structures* server. What this means is that Redis provides access to mutable data structures via a set of commands, which are sent using a *server-client* model with TCP sockets and a simple protocol. So different processes can query and modify the same data structures in a shared way. +Redis is often referred to as a *data structures* server. What this means is that Redis provides access to mutable data structures via a set of commands, which are sent using a *server-client* model with TCP sockets and a simple protocol. So different processes can query and modify the same data structures in a shared way. Data structures implemented into Redis have a few special properties: -* Redis cares to store them on disk, even if they are always served and modified into the server memory. This means that Redis is fast, but that is also non-volatile. -* Implementation of data structures stress on memory efficiency, so data structures inside Redis will likely use less memory compared to the same data structure modeled using an high level programming language. -* Redis offers a number of features that are natural to find in a database, like replication, tunable levels of durability, cluster, high availability. +* Redis cares to store them on disk, even if they are always served and modified into the server memory. This means that Redis is fast, but that it is also non-volatile. +* The implementation of data structures emphasizes memory efficiency, so data structures inside Redis will likely use less memory compared to the same data structure modelled using a high-level programming language. +* Redis offers a number of features that are natural to find in a database, like replication, tunable levels of durability, clustering, and high availability. -Another good example is to think of Redis as a more complex version of memcached, where the operations are not just SETs and GETs, but operations to work with complex data types like Lists, Sets, ordered data structures, and so forth. +Another good example is to think of Redis as a more complex version of memcached, where the operations are not just SETs and GETs, but operations that work with complex data types like Lists, Sets, ordered data structures, and so forth. If you want to know more, this is a list of selected starting points: -* Introduction to Redis data types. http://redis.io/topics/data-types-intro +* Introduction to Redis data types. https://redis.io/topics/data-types-intro * Try Redis directly inside your browser. http://try.redis.io -* The full list of Redis commands. http://redis.io/commands -* There is much more inside the Redis official documentation. http://redis.io/documentation +* The full list of Redis commands. https://redis.io/commands +* There is much more inside the official Redis documentation. https://redis.io/documentation Building Redis -------------- @@ -29,13 +29,27 @@ and 64 bit systems. It may compile on Solaris derived systems (for instance SmartOS) but our support for this platform is *best effort* and Redis is not guaranteed to -work as well as in Linux, OSX, and \*BSD there. +work as well as in Linux, OSX, and \*BSD. It is as simple as: % make -You can run a 32 bit Redis binary using: +To build with TLS support, you'll need OpenSSL development libraries (e.g. +libssl-dev on Debian/Ubuntu) and run: + + % make BUILD_TLS=yes + +To build with systemd support, you'll need systemd development libraries (such +as libsystemd-dev on Debian/Ubuntu or systemd-devel on CentOS) and run: + + % make USE_SYSTEMD=yes + +To append a suffix to Redis program names, use: + + % make PROG_SUFFIX="-alt" + +You can build a 32 bit Redis binary using: % make 32bit @@ -43,10 +57,17 @@ After building Redis, it is a good idea to test it using: % make test +If TLS is built, running the tests with TLS enabled (you will need `tcl-tls` +installed): + + % ./utils/gen-test-certs.sh + % ./runtest --tls + + Fixing build problems with dependencies or cached build options --------- -Redis has some dependencies which are included into the `deps` directory. +Redis has some dependencies which are included in the `deps` directory. `make` does not automatically rebuild dependencies even if something in the source code of dependencies changes. @@ -73,7 +94,7 @@ with a 64 bit target, or the other way around, you need to perform a In case of build errors when trying to build a 32 bit binary of Redis, try the following steps: -* Install the packages libc6-dev-i386 (also try g++-multilib). +* Install the package libc6-dev-i386 (also try g++-multilib). * Try using the following command line instead of `make 32bit`: `make CFLAGS="-m32 -march=native" LDFLAGS="-m32"` @@ -94,18 +115,30 @@ To compile against jemalloc on Mac OS X systems, use: % make MALLOC=jemalloc +Monotonic clock +--------------- + +By default, Redis will build using the POSIX clock_gettime function as the +monotonic clock source. On most modern systems, the internal processor clock +can be used to improve performance. Cautions can be found here: + http://oliveryang.net/2015/09/pitfalls-of-TSC-usage/ + +To build with support for the processor's internal instruction clock, use: + + % make CFLAGS="-DUSE_PROCESSOR_CLOCK" + Verbose build ------------- -Redis will build with a user friendly colorized output by default. -If you want to see a more verbose output use the following: +Redis will build with a user-friendly colorized output by default. +If you want to see a more verbose output, use the following: % make V=1 Running Redis ------------- -To run Redis with the default configuration just type: +To run Redis with the default configuration, just type: % cd src % ./redis-server @@ -125,6 +158,12 @@ as options using the command line. Examples: All the options in redis.conf are also supported as options using the command line, with exactly the same name. +Running Redis with TLS: +------------------ + +Please consult the [TLS.md](TLS.md) file for more information on +how to use Redis with TLS. + Playing with Redis ------------------ @@ -145,12 +184,12 @@ then in another terminal try the following: (integer) 2 redis> -You can find the list of all the available commands at http://redis.io/commands. +You can find the list of all the available commands at https://redis.io/commands. Installing Redis ----------------- -In order to install Redis binaries into /usr/local/bin just use: +In order to install Redis binaries into /usr/local/bin, just use: % make install @@ -159,13 +198,15 @@ different destination. Make install will just install binaries in your system, but will not configure init scripts and configuration files in the appropriate place. This is not -needed if you want just to play a bit with Redis, but if you are installing -it the proper way for a production system, we have a script doing this +needed if you just want to play a bit with Redis, but if you are installing +it the proper way for a production system, we have a script that does this for Ubuntu and Debian systems: % cd utils % ./install_server.sh +_Note_: `install_server.sh` will not work on Mac OSX; it is built for Linux only. + The script will ask you a few questions and will setup everything you need to run Redis properly as a background daemon that will start again on system reboots. @@ -176,17 +217,17 @@ You'll be able to stop and start Redis using the script named Code contributions ----------------- -Note: by contributing code to the Redis project in any form, including sending +Note: By contributing code to the Redis project in any form, including sending a pull request via Github, a code fragment or patch via private email or public discussion groups, you agree to release your code under the terms of the BSD license that you can find in the [COPYING][1] file included in the Redis source distribution. Please see the [CONTRIBUTING][2] file in this source distribution for more -information. +information, including details on our process for security bugs/vulnerabilities. -[1]: https://github.com/antirez/redis/blob/unstable/COPYING -[2]: https://github.com/antirez/redis/blob/unstable/CONTRIBUTING +[1]: https://github.com/redis/redis/blob/unstable/COPYING +[2]: https://github.com/redis/redis/blob/unstable/CONTRIBUTING Redis internals === @@ -216,7 +257,7 @@ Inside the root are the following important directories: * `src`: contains the Redis implementation, written in C. * `tests`: contains the unit tests, implemented in Tcl. -* `deps`: contains libraries Redis uses. Everything needed to compile Redis is inside this directory; your system just needs to provide `libc`, a POSIX compatible interface and a C compiler. Notably `deps` contains a copy of `jemalloc`, which is the default allocator of Redis under Linux. Note that under `deps` there are also things which started with the Redis project, but for which the main repository is not `antirez/redis`. An exception to this rule is `deps/geohash-int` which is the low level geocoding library used by Redis: it originated from a different project, but at this point it diverged so much that it is developed as a separated entity directly inside the Redis repository. +* `deps`: contains libraries Redis uses. Everything needed to compile Redis is inside this directory; your system just needs to provide `libc`, a POSIX compatible interface and a C compiler. Notably `deps` contains a copy of `jemalloc`, which is the default allocator of Redis under Linux. Note that under `deps` there are also things which started with the Redis project, but for which the main repository is not `redis/redis`. There are a few more directories but they are not very important for our goals here. We'll focus mostly on `src`, where the Redis implementation is contained, @@ -226,7 +267,7 @@ of complexity incrementally. Note: lately Redis was refactored quite a bit. Function names and file names have been changed, so you may find that this documentation reflects the -`unstable` branch more closely. For instance in Redis 3.0 the `server.c` +`unstable` branch more closely. For instance, in Redis 3.0 the `server.c` and `server.h` files were named `redis.c` and `redis.h`. However the overall structure is the same. Keep in mind that all the new developments and pull requests should be performed against the `unstable` branch. @@ -253,25 +294,25 @@ the structure definition. Another important Redis data structure is the one defining a client. In the past it was called `redisClient`, now just `client`. The structure has many fields, here we'll just show the main ones: - - struct client { - int fd; - sds querybuf; - int argc; - robj **argv; - redisDb *db; - int flags; - list *reply; - char buf[PROTO_REPLY_CHUNK_BYTES]; - ... many other fields ... - } - +```c +struct client { + int fd; + sds querybuf; + int argc; + robj **argv; + redisDb *db; + int flags; + list *reply; + char buf[PROTO_REPLY_CHUNK_BYTES]; + // ... many other fields ... +} +``` The client structure defines a *connected client*: * The `fd` field is the client socket file descriptor. * `argc` and `argv` are populated with the command the client is executing, so that functions implementing a given Redis command can read the arguments. * `querybuf` accumulates the requests from the client, which are parsed by the Redis server according to the Redis protocol and executed by calling the implementations of the commands the client is executing. -* `reply` and `buf` are dynamic and static buffers that accumulate the replies the server sends to the client. These buffers are incrementally written to the socket as soon as the file descriptor is writable. +* `reply` and `buf` are dynamic and static buffers that accumulate the replies the server sends to the client. These buffers are incrementally written to the socket as soon as the file descriptor is writeable. As you can see in the client structure above, arguments in a command are described as `robj` structures. The following is the full `robj` @@ -304,20 +345,20 @@ This is the entry point of the Redis server, where the `main()` function is defined. The following are the most important steps in order to startup the Redis server. -* `initServerConfig()` setups the default values of the `server` structure. +* `initServerConfig()` sets up the default values of the `server` structure. * `initServer()` allocates the data structures needed to operate, setup the listening socket, and so forth. * `aeMain()` starts the event loop which listens for new connections. There are two special functions called periodically by the event loop: -1. `serverCron()` is called periodically (according to `server.hz` frequency), and performs tasks that must be performed from time to time, like checking for timedout clients. +1. `serverCron()` is called periodically (according to `server.hz` frequency), and performs tasks that must be performed from time to time, like checking for timed out clients. 2. `beforeSleep()` is called every time the event loop fired, Redis served a few requests, and is returning back into the event loop. Inside server.c you can find code that handles other vital things of the Redis server: * `call()` is used in order to call a given command in the context of a given client. -* `activeExpireCycle()` handles eviciton of keys with a time to live set via the `EXPIRE` command. -* `freeMemoryIfNeeded()` is called when a new write command should be performed but Redis is out of memory according to the `maxmemory` directive. +* `activeExpireCycle()` handles eviction of keys with a time to live set via the `EXPIRE` command. +* `performEvictions()` is called when a new write command should be performed but Redis is out of memory according to the `maxmemory` directive. * The global variable `redisCommandTable` defines all the Redis commands, specifying the name of the command, the function implementing the command, the number of arguments required, and other properties of each command. networking.c @@ -327,16 +368,16 @@ This file defines all the I/O functions with clients, masters and replicas (which in Redis are just special clients): * `createClient()` allocates and initializes a new client. -* the `addReply*()` family of functions are used by commands implementations in order to append data to the client structure, that will be transmitted to the client as a reply for a given command executed. +* the `addReply*()` family of functions are used by command implementations in order to append data to the client structure, that will be transmitted to the client as a reply for a given command executed. * `writeToClient()` transmits the data pending in the output buffers to the client and is called by the *writable event handler* `sendReplyToClient()`. -* `readQueryFromClient()` is the *readable event handler* and accumulates data from read from the client into the query buffer. +* `readQueryFromClient()` is the *readable event handler* and accumulates data read from the client into the query buffer. * `processInputBuffer()` is the entry point in order to parse the client query buffer according to the Redis protocol. Once commands are ready to be processed, it calls `processCommand()` which is defined inside `server.c` in order to actually execute the command. * `freeClient()` deallocates, disconnects and removes a client. aof.c and rdb.c --- -As you can guess from the names these files implement the RDB and AOF +As you can guess from the names, these files implement the RDB and AOF persistence for Redis. Redis uses a persistence model based on the `fork()` system call in order to create a thread with the same (shared) memory content of the main Redis thread. This secondary thread dumps the content @@ -348,13 +389,13 @@ The implementation inside `aof.c` has additional functions in order to implement an API that allows commands to append new commands into the AOF file as clients execute them. -The `call()` function defined inside `server.c` is responsible to call +The `call()` function defined inside `server.c` is responsible for calling the functions that in turn will write the commands into the AOF. db.c --- -Certain Redis commands operate on specific data types, others are general. +Certain Redis commands operate on specific data types; others are general. Examples of generic commands are `DEL` and `EXPIRE`. They operate on keys and not on their values specifically. All those generic commands are defined inside `db.c`. @@ -362,7 +403,7 @@ defined inside `db.c`. Moreover `db.c` implements an API in order to perform certain operations on the Redis dataset without directly accessing the internal data structures. -The most important functions inside `db.c` which are used in many commands +The most important functions inside `db.c` which are used in many command implementations are the following: * `lookupKeyRead()` and `lookupKeyWrite()` are used in order to get a pointer to the value associated to a given key, or `NULL` if the key does not exist. @@ -380,7 +421,7 @@ The `robj` structure defining Redis objects was already described. Inside a basic level, like functions to allocate new objects, handle the reference counting and so forth. Notable functions inside this file: -* `incrRefcount()` and `decrRefCount()` are used in order to increment or decrement an object reference count. When it drops to 0 the object is finally freed. +* `incrRefCount()` and `decrRefCount()` are used in order to increment or decrement an object reference count. When it drops to 0 the object is finally freed. * `createObject()` allocates a new object. There are also specialized functions to allocate string objects having a specific content, like `createStringObjectFromLongLong()` and similar functions. This file also implements the `OBJECT` command. @@ -404,15 +445,15 @@ replicas, or to continue the replication after a disconnection. Other C files --- -* `t_hash.c`, `t_list.c`, `t_set.c`, `t_string.c` and `t_zset.c` contains the implementation of the Redis data types. They implement both an API to access a given data type, and the client commands implementations for these data types. +* `t_hash.c`, `t_list.c`, `t_set.c`, `t_string.c`, `t_zset.c` and `t_stream.c` contains the implementation of the Redis data types. They implement both an API to access a given data type, and the client command implementations for these data types. * `ae.c` implements the Redis event loop, it's a self contained library which is simple to read and understand. * `sds.c` is the Redis string library, check http://github.com/antirez/sds for more information. * `anet.c` is a library to use POSIX networking in a simpler way compared to the raw interface exposed by the kernel. * `dict.c` is an implementation of a non-blocking hash table which rehashes incrementally. -* `scripting.c` implements Lua scripting. It is completely self contained from the rest of the Redis implementation and is simple enough to understand if you are familar with the Lua API. +* `scripting.c` implements Lua scripting. It is completely self-contained and isolated from the rest of the Redis implementation and is simple enough to understand if you are familiar with the Lua API. * `cluster.c` implements the Redis Cluster. Probably a good read only after being very familiar with the rest of the Redis code base. If you want to read `cluster.c` make sure to read the [Redis Cluster specification][3]. -[3]: http://redis.io/topics/cluster-spec +[3]: https://redis.io/topics/cluster-spec Anatomy of a Redis command --- @@ -435,12 +476,12 @@ top comment inside `server.c`. After the command operates in some way, it returns a reply to the client, usually using `addReply()` or a similar function defined inside `networking.c`. -There are tons of commands implementations inside the Redis source code -that can serve as examples of actual commands implementations. To write -a few toy commands can be a good exercise to familiarize with the code base. +There are tons of command implementations inside the Redis source code +that can serve as examples of actual commands implementations. Writing +a few toy commands can be a good exercise to get familiar with the code base. There are also many other files not described here, but it is useless to -cover everything. We want to just help you with the first steps. +cover everything. We just want to help you with the first steps. Eventually you'll find your way inside the Redis code base :-) Enjoy! diff --git a/redis.submodule/TLS.md b/redis.submodule/TLS.md new file mode 100644 index 0000000..2d020d0 --- /dev/null +++ b/redis.submodule/TLS.md @@ -0,0 +1,87 @@ +TLS Support +=========== + +Getting Started +--------------- + +### Building + +To build with TLS support you'll need OpenSSL development libraries (e.g. +libssl-dev on Debian/Ubuntu). + +Run `make BUILD_TLS=yes`. + +### Tests + +To run Redis test suite with TLS, you'll need TLS support for TCL (i.e. +`tcl-tls` package on Debian/Ubuntu). + +1. Run `./utils/gen-test-certs.sh` to generate a root CA and a server + certificate. + +2. Run `./runtest --tls` or `./runtest-cluster --tls` to run Redis and Redis + Cluster tests in TLS mode. + +### Running manually + +To manually run a Redis server with TLS mode (assuming `gen-test-certs.sh` was +invoked so sample certificates/keys are available): + + ./src/redis-server --tls-port 6379 --port 0 \ + --tls-cert-file ./tests/tls/redis.crt \ + --tls-key-file ./tests/tls/redis.key \ + --tls-ca-cert-file ./tests/tls/ca.crt + +To connect to this Redis server with `redis-cli`: + + ./src/redis-cli --tls \ + --cert ./tests/tls/redis.crt \ + --key ./tests/tls/redis.key \ + --cacert ./tests/tls/ca.crt + +This will disable TCP and enable TLS on port 6379. It's also possible to have +both TCP and TLS available, but you'll need to assign different ports. + +To make a Replica connect to the master using TLS, use `--tls-replication yes`, +and to make Redis Cluster use TLS across nodes use `--tls-cluster yes`. + +Connections +----------- + +All socket operations now go through a connection abstraction layer that hides +I/O and read/write event handling from the caller. + +**Multi-threading I/O is not currently supported for TLS**, as a TLS connection +needs to do its own manipulation of AE events which is not thread safe. The +solution is probably to manage independent AE loops for I/O threads and longer +term association of connections with threads. This may potentially improve +overall performance as well. + +Sync IO for TLS is currently implemented in a hackish way, i.e. making the +socket blocking and configuring socket-level timeout. This means the timeout +value may not be so accurate, and there would be a lot of syscall overhead. +However I believe that getting rid of syncio completely in favor of pure async +work is probably a better move than trying to fix that. For replication it would +probably not be so hard. For cluster keys migration it might be more difficult, +but there are probably other good reasons to improve that part anyway. + +To-Do List +---------- + +- [ ] redis-benchmark support. The current implementation is a mix of using + hiredis for parsing and basic networking (establishing connections), but + directly manipulating sockets for most actions. This will need to be cleaned + up for proper TLS support. The best approach is probably to migrate to hiredis + async mode. +- [ ] redis-cli `--slave` and `--rdb` support. + +Multi-port +---------- + +Consider the implications of allowing TLS to be configured on a separate port, +making Redis listening on multiple ports: + +1. Startup banner port notification +2. Proctitle +3. How slaves announce themselves +4. Cluster bus port calculation diff --git a/redis.submodule/deps/Makefile b/redis.submodule/deps/Makefile index eb35c1e..ff16ee9 100644 --- a/redis.submodule/deps/Makefile +++ b/redis.submodule/deps/Makefile @@ -37,13 +37,18 @@ distclean: -(cd linenoise && $(MAKE) clean) > /dev/null || true -(cd lua && $(MAKE) clean) > /dev/null || true -(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true + -(cd hdr_histogram && $(MAKE) clean) > /dev/null || true -(rm -f .make-*) .PHONY: distclean +ifeq ($(BUILD_TLS),yes) + HIREDIS_MAKE_FLAGS = USE_SSL=1 +endif + hiredis: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) - cd hiredis && $(MAKE) static + cd hiredis && $(MAKE) static $(HIREDIS_MAKE_FLAGS) .PHONY: hiredis @@ -53,18 +58,24 @@ linenoise: .make-prerequisites .PHONY: linenoise +hdr_histogram: .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) + cd hdr_histogram && $(MAKE) + +.PHONY: hdr_histogram + ifeq ($(uname_S),SunOS) # Make isinf() available LUA_CFLAGS= -D__C99FEATURES__=1 endif -LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' $(CFLAGS) +LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' -DLUA_USE_MKSTEMP $(CFLAGS) LUA_LDFLAGS+= $(LDFLAGS) # lua's Makefile defines AR="ar rcu", which is unusual, and makes it more # challenging to cross-compile lua (and redis). These defines make it easier # to fit redis into cross-compilation environments, which typically set AR. AR=ar -ARFLAGS=rcu +ARFLAGS=rc lua: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) @@ -77,7 +88,7 @@ JEMALLOC_LDFLAGS= $(LDFLAGS) jemalloc: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) - cd jemalloc && ./configure --with-version=5.1.0-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ --enable-cc-silence CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" + cd jemalloc && ./configure --with-version=5.1.0-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" cd jemalloc && $(MAKE) CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" lib/libjemalloc.a .PHONY: jemalloc diff --git a/redis.submodule/deps/README.md b/redis.submodule/deps/README.md index 367ee16..c3ec424 100644 --- a/redis.submodule/deps/README.md +++ b/redis.submodule/deps/README.md @@ -2,7 +2,6 @@ This directory contains all Redis dependencies, except for the libc that should be provided by the operating system. * **Jemalloc** is our memory allocator, used as replacement for libc malloc on Linux by default. It has good performances and excellent fragmentation behavior. This component is upgraded from time to time. -* **geohash-int** is inside the dependencies directory but is actually part of the Redis project, since it is our private fork (heavily modified) of a library initially developed for Ardb, which is in turn a fork of Redis. * **hiredis** is the official C client library for Redis. It is used by redis-cli, redis-benchmark and Redis Sentinel. It is part of the Redis official ecosystem but is developed externally from the Redis repository, so we just upgrade it as needed. * **linenoise** is a readline replacement. It is developed by the same authors of Redis but is managed as a separated project and updated as needed. * **lua** is Lua 5.1 with minor changes for security and additional libraries. @@ -18,11 +17,11 @@ active defragmentation logic. However this feature of Redis is not mandatory and Redis is able to understand if the Jemalloc version it is compiled against supports such Redis-specific modifications. So in theory, if you are not interested in the active defragmentation, you can replace Jemalloc -just following tose steps: +just following these steps: 1. Remove the jemalloc directory. 2. Substitute it with the new jemalloc source tree. -3. Edit the Makefile localted in the same directory as the README you are +3. Edit the Makefile located in the same directory as the README you are reading, and change the --with-version in the Jemalloc configure script options with the version you are using. This is required because otherwise Jemalloc configuration script is broken and will not work nested in another @@ -34,7 +33,7 @@ If you want to upgrade Jemalloc while also providing support for active defragmentation, in addition to the above steps you need to perform the following additional steps: -5. In Jemalloc three, file `include/jemalloc/jemalloc_macros.h.in`, make sure +5. In Jemalloc tree, file `include/jemalloc/jemalloc_macros.h.in`, make sure to add `#define JEMALLOC_FRAG_HINT`. 6. Implement the function `je_get_defrag_hint()` inside `src/jemalloc.c`. You can see how it is implemented in the current Jemalloc source tree shipped @@ -42,20 +41,15 @@ the following additional steps: changed, otherwise you could just copy the old implementation if you are upgrading just to a similar version of Jemalloc. -Geohash ---- - -This is never upgraded since it's part of the Redis project. If there are changes to merge from Ardb there is the need to manually check differences, but at this point the source code is pretty different. - Hiredis --- Hiredis uses the SDS string library, that must be the same version used inside Redis itself. Hiredis is also very critical for Sentinel. Historically Redis often used forked versions of hiredis in a way or the other. In order to upgrade it is advised to take a lot of care: 1. Check with diff if hiredis API changed and what impact it could have in Redis. -2. Make sure thet the SDS library inside Hiredis and inside Redis are compatible. +2. Make sure that the SDS library inside Hiredis and inside Redis are compatible. 3. After the upgrade, run the Redis Sentinel test. -4. Check manually that redis-cli and redis-benchmark behave as expecteed, since we have no tests for CLI utilities currently. +4. Check manually that redis-cli and redis-benchmark behave as expected, since we have no tests for CLI utilities currently. Linenoise --- @@ -83,6 +77,6 @@ and our version: 1. Makefile is modified to allow a different compiler than GCC. 2. We have the implementation source code, and directly link to the following external libraries: `lua_cjson.o`, `lua_struct.o`, `lua_cmsgpack.o` and `lua_bit.o`. -3. There is a security fix in `ldo.c`, line 498: The check for `LUA_SIGNATURE[0]` is removed in order toa void direct bytecode execution. +3. There is a security fix in `ldo.c`, line 498: The check for `LUA_SIGNATURE[0]` is removed in order to avoid direct bytecode execution. diff --git a/redis.submodule/deps/hdr_histogram/COPYING.txt b/redis.submodule/deps/hdr_histogram/COPYING.txt new file mode 100644 index 0000000..0e259d4 --- /dev/null +++ b/redis.submodule/deps/hdr_histogram/COPYING.txt @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/redis.submodule/deps/hdr_histogram/LICENSE.txt b/redis.submodule/deps/hdr_histogram/LICENSE.txt new file mode 100644 index 0000000..9b4e66e --- /dev/null +++ b/redis.submodule/deps/hdr_histogram/LICENSE.txt @@ -0,0 +1,41 @@ +The code in this repository code was Written by Gil Tene, Michael Barker, +and Matt Warren, and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/ + +For users of this code who wish to consume it under the "BSD" license +rather than under the public domain or CC0 contribution text mentioned +above, the code found under this directory is *also* provided under the +following license (commonly referred to as the BSD 2-Clause License). This +license does not detract from the above stated release of the code into +the public domain, and simply represents an additional license granted by +the Author. + +----------------------------------------------------------------------------- +** Beginning of "BSD 2-Clause License" text. ** + + Copyright (c) 2012, 2013, 2014 Gil Tene + Copyright (c) 2014 Michael Barker + Copyright (c) 2014 Matt Warren + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. diff --git a/redis.submodule/deps/hdr_histogram/Makefile b/redis.submodule/deps/hdr_histogram/Makefile new file mode 100644 index 0000000..83d3760 --- /dev/null +++ b/redis.submodule/deps/hdr_histogram/Makefile @@ -0,0 +1,20 @@ +STD= +WARN= -Wall +OPT= -Os + +R_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) +R_LDFLAGS= $(LDFLAGS) +DEBUG= -g + +R_CC=$(CC) $(R_CFLAGS) +R_LD=$(CC) $(R_LDFLAGS) + +hdr_histogram.o: hdr_histogram.h hdr_histogram.c + +.c.o: + $(R_CC) -c $< + +clean: + rm -f *.o + + diff --git a/redis.submodule/deps/hdr_histogram/README.md b/redis.submodule/deps/hdr_histogram/README.md new file mode 100644 index 0000000..5f62c23 --- /dev/null +++ b/redis.submodule/deps/hdr_histogram/README.md @@ -0,0 +1,10 @@ +HdrHistogram_c v0.11.0 + +---------------------------------------------- + +This port contains a subset of the 'C' version of High Dynamic Range (HDR) Histogram available at [github.com/HdrHistogram/HdrHistogram_c](https://github.com/HdrHistogram/HdrHistogram_c). + + +The code present on `hdr_histogram.c`, `hdr_histogram.h`, and `hdr_atomic.c` was Written by Gil Tene, Michael Barker, +and Matt Warren, and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/. \ No newline at end of file diff --git a/redis.submodule/deps/hdr_histogram/hdr_atomic.h b/redis.submodule/deps/hdr_histogram/hdr_atomic.h new file mode 100644 index 0000000..ae1056a --- /dev/null +++ b/redis.submodule/deps/hdr_histogram/hdr_atomic.h @@ -0,0 +1,146 @@ +/** + * hdr_atomic.h + * Written by Philip Orwig and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#ifndef HDR_ATOMIC_H__ +#define HDR_ATOMIC_H__ + + +#if defined(_MSC_VER) + +#include +#include +#include + +static void __inline * hdr_atomic_load_pointer(void** pointer) +{ + _ReadBarrier(); + return *pointer; +} + +static void hdr_atomic_store_pointer(void** pointer, void* value) +{ + _WriteBarrier(); + *pointer = value; +} + +static int64_t __inline hdr_atomic_load_64(int64_t* field) +{ + _ReadBarrier(); + return *field; +} + +static void __inline hdr_atomic_store_64(int64_t* field, int64_t value) +{ + _WriteBarrier(); + *field = value; +} + +static int64_t __inline hdr_atomic_exchange_64(volatile int64_t* field, int64_t value) +{ +#if defined(_WIN64) + return _InterlockedExchange64(field, value); +#else + int64_t comparand; + int64_t initial_value = *field; + do + { + comparand = initial_value; + initial_value = _InterlockedCompareExchange64(field, value, comparand); + } + while (comparand != initial_value); + + return initial_value; +#endif +} + +static int64_t __inline hdr_atomic_add_fetch_64(volatile int64_t* field, int64_t value) +{ +#if defined(_WIN64) + return _InterlockedExchangeAdd64(field, value) + value; +#else + int64_t comparand; + int64_t initial_value = *field; + do + { + comparand = initial_value; + initial_value = _InterlockedCompareExchange64(field, comparand + value, comparand); + } + while (comparand != initial_value); + + return initial_value + value; +#endif +} + +static bool __inline hdr_atomic_compare_exchange_64(volatile int64_t* field, int64_t* expected, int64_t desired) +{ + return *expected == _InterlockedCompareExchange64(field, desired, *expected); +} + +#elif defined(__ATOMIC_SEQ_CST) + +#define hdr_atomic_load_pointer(x) __atomic_load_n(x, __ATOMIC_SEQ_CST) +#define hdr_atomic_store_pointer(f,v) __atomic_store_n(f,v, __ATOMIC_SEQ_CST) +#define hdr_atomic_load_64(x) __atomic_load_n(x, __ATOMIC_SEQ_CST) +#define hdr_atomic_store_64(f,v) __atomic_store_n(f,v, __ATOMIC_SEQ_CST) +#define hdr_atomic_exchange_64(f,i) __atomic_exchange_n(f,i, __ATOMIC_SEQ_CST) +#define hdr_atomic_add_fetch_64(field, value) __atomic_add_fetch(field, value, __ATOMIC_SEQ_CST) +#define hdr_atomic_compare_exchange_64(field, expected, desired) __atomic_compare_exchange_n(field, expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) + +#elif defined(__x86_64__) + +#include +#include + +static inline void* hdr_atomic_load_pointer(void** pointer) +{ + void* p = *pointer; + asm volatile ("" ::: "memory"); + return p; +} + +static inline void hdr_atomic_store_pointer(void** pointer, void* value) +{ + asm volatile ("lock; xchgq %0, %1" : "+q" (value), "+m" (*pointer)); +} + +static inline int64_t hdr_atomic_load_64(int64_t* field) +{ + int64_t i = *field; + asm volatile ("" ::: "memory"); + return i; +} + +static inline void hdr_atomic_store_64(int64_t* field, int64_t value) +{ + asm volatile ("lock; xchgq %0, %1" : "+q" (value), "+m" (*field)); +} + +static inline int64_t hdr_atomic_exchange_64(volatile int64_t* field, int64_t value) +{ + int64_t result = 0; + asm volatile ("lock; xchgq %1, %2" : "=r" (result), "+q" (value), "+m" (*field)); + return result; +} + +static inline int64_t hdr_atomic_add_fetch_64(volatile int64_t* field, int64_t value) +{ + return __sync_add_and_fetch(field, value); +} + +static inline bool hdr_atomic_compare_exchange_64(volatile int64_t* field, int64_t* expected, int64_t desired) +{ + int64_t original; + asm volatile( "lock; cmpxchgq %2, %1" : "=a"(original), "+m"(*field) : "q"(desired), "0"(*expected)); + return original == *expected; +} + +#else + +#error "Unable to determine atomic operations for your platform" + +#endif + +#endif /* HDR_ATOMIC_H__ */ diff --git a/redis.submodule/deps/hdr_histogram/hdr_histogram.c b/redis.submodule/deps/hdr_histogram/hdr_histogram.c new file mode 100644 index 0000000..5f5286f --- /dev/null +++ b/redis.submodule/deps/hdr_histogram/hdr_histogram.c @@ -0,0 +1,1155 @@ +/** + * hdr_histogram.c + * Written by Michael Barker and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hdr_histogram.h" +#include "hdr_atomic.h" + +/* ###### ####### ## ## ## ## ######## ###### */ +/* ## ## ## ## ## ## ### ## ## ## ## */ +/* ## ## ## ## ## #### ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ###### */ +/* ## ## ## ## ## ## #### ## ## */ +/* ## ## ## ## ## ## ## ### ## ## ## */ +/* ###### ####### ####### ## ## ## ###### */ + +static int32_t normalize_index(const struct hdr_histogram* h, int32_t index) +{ + int32_t normalized_index; + int32_t adjustment = 0; + if (h->normalizing_index_offset == 0) + { + return index; + } + + normalized_index = index - h->normalizing_index_offset; + + if (normalized_index < 0) + { + adjustment = h->counts_len; + } + else if (normalized_index >= h->counts_len) + { + adjustment = -h->counts_len; + } + + return normalized_index + adjustment; +} + +static int64_t counts_get_direct(const struct hdr_histogram* h, int32_t index) +{ + return h->counts[index]; +} + +static int64_t counts_get_normalised(const struct hdr_histogram* h, int32_t index) +{ + return counts_get_direct(h, normalize_index(h, index)); +} + +static void counts_inc_normalised( + struct hdr_histogram* h, int32_t index, int64_t value) +{ + int32_t normalised_index = normalize_index(h, index); + h->counts[normalised_index] += value; + h->total_count += value; +} + +static void counts_inc_normalised_atomic( + struct hdr_histogram* h, int32_t index, int64_t value) +{ + int32_t normalised_index = normalize_index(h, index); + + hdr_atomic_add_fetch_64(&h->counts[normalised_index], value); + hdr_atomic_add_fetch_64(&h->total_count, value); +} + +static void update_min_max(struct hdr_histogram* h, int64_t value) +{ + h->min_value = (value < h->min_value && value != 0) ? value : h->min_value; + h->max_value = (value > h->max_value) ? value : h->max_value; +} + +static void update_min_max_atomic(struct hdr_histogram* h, int64_t value) +{ + int64_t current_min_value; + int64_t current_max_value; + do + { + current_min_value = hdr_atomic_load_64(&h->min_value); + + if (0 == value || current_min_value <= value) + { + break; + } + } + while (!hdr_atomic_compare_exchange_64(&h->min_value, ¤t_min_value, value)); + + do + { + current_max_value = hdr_atomic_load_64(&h->max_value); + + if (value <= current_max_value) + { + break; + } + } + while (!hdr_atomic_compare_exchange_64(&h->max_value, ¤t_max_value, value)); +} + + +/* ## ## ######## #### ## #### ######## ## ## */ +/* ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## #### */ +/* ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## */ +/* ####### ## #### ######## #### ## ## */ + +static int64_t power(int64_t base, int64_t exp) +{ + int64_t result = 1; + while(exp) + { + result *= base; exp--; + } + return result; +} + +#if defined(_MSC_VER) +# if defined(_WIN64) +# pragma intrinsic(_BitScanReverse64) +# else +# pragma intrinsic(_BitScanReverse) +# endif +#endif + +static int32_t count_leading_zeros_64(int64_t value) +{ +#if defined(_MSC_VER) + uint32_t leading_zero = 0; +#if defined(_WIN64) + _BitScanReverse64(&leading_zero, value); +#else + uint32_t high = value >> 32; + if (_BitScanReverse(&leading_zero, high)) + { + leading_zero += 32; + } + else + { + uint32_t low = value & 0x00000000FFFFFFFF; + _BitScanReverse(&leading_zero, low); + } +#endif + return 63 - leading_zero; /* smallest power of 2 containing value */ +#else + return __builtin_clzll(value); /* smallest power of 2 containing value */ +#endif +} + +static int32_t get_bucket_index(const struct hdr_histogram* h, int64_t value) +{ + int32_t pow2ceiling = 64 - count_leading_zeros_64(value | h->sub_bucket_mask); /* smallest power of 2 containing value */ + return pow2ceiling - h->unit_magnitude - (h->sub_bucket_half_count_magnitude + 1); +} + +static int32_t get_sub_bucket_index(int64_t value, int32_t bucket_index, int32_t unit_magnitude) +{ + return (int32_t)(value >> (bucket_index + unit_magnitude)); +} + +static int32_t counts_index(const struct hdr_histogram* h, int32_t bucket_index, int32_t sub_bucket_index) +{ + /* Calculate the index for the first entry in the bucket: */ + /* (The following is the equivalent of ((bucket_index + 1) * subBucketHalfCount) ): */ + int32_t bucket_base_index = (bucket_index + 1) << h->sub_bucket_half_count_magnitude; + /* Calculate the offset in the bucket: */ + int32_t offset_in_bucket = sub_bucket_index - h->sub_bucket_half_count; + /* The following is the equivalent of ((sub_bucket_index - subBucketHalfCount) + bucketBaseIndex; */ + return bucket_base_index + offset_in_bucket; +} + +static int64_t value_from_index(int32_t bucket_index, int32_t sub_bucket_index, int32_t unit_magnitude) +{ + return ((int64_t) sub_bucket_index) << (bucket_index + unit_magnitude); +} + +int32_t counts_index_for(const struct hdr_histogram* h, int64_t value) +{ + int32_t bucket_index = get_bucket_index(h, value); + int32_t sub_bucket_index = get_sub_bucket_index(value, bucket_index, h->unit_magnitude); + + return counts_index(h, bucket_index, sub_bucket_index); +} + +int64_t hdr_value_at_index(const struct hdr_histogram *h, int32_t index) +{ + int32_t bucket_index = (index >> h->sub_bucket_half_count_magnitude) - 1; + int32_t sub_bucket_index = (index & (h->sub_bucket_half_count - 1)) + h->sub_bucket_half_count; + + if (bucket_index < 0) + { + sub_bucket_index -= h->sub_bucket_half_count; + bucket_index = 0; + } + + return value_from_index(bucket_index, sub_bucket_index, h->unit_magnitude); +} + +int64_t hdr_size_of_equivalent_value_range(const struct hdr_histogram* h, int64_t value) +{ + int32_t bucket_index = get_bucket_index(h, value); + int32_t sub_bucket_index = get_sub_bucket_index(value, bucket_index, h->unit_magnitude); + int32_t adjusted_bucket = (sub_bucket_index >= h->sub_bucket_count) ? (bucket_index + 1) : bucket_index; + return INT64_C(1) << (h->unit_magnitude + adjusted_bucket); +} + +static int64_t lowest_equivalent_value(const struct hdr_histogram* h, int64_t value) +{ + int32_t bucket_index = get_bucket_index(h, value); + int32_t sub_bucket_index = get_sub_bucket_index(value, bucket_index, h->unit_magnitude); + return value_from_index(bucket_index, sub_bucket_index, h->unit_magnitude); +} + +int64_t hdr_next_non_equivalent_value(const struct hdr_histogram *h, int64_t value) +{ + return lowest_equivalent_value(h, value) + hdr_size_of_equivalent_value_range(h, value); +} + +static int64_t highest_equivalent_value(const struct hdr_histogram* h, int64_t value) +{ + return hdr_next_non_equivalent_value(h, value) - 1; +} + +int64_t hdr_median_equivalent_value(const struct hdr_histogram *h, int64_t value) +{ + return lowest_equivalent_value(h, value) + (hdr_size_of_equivalent_value_range(h, value) >> 1); +} + +static int64_t non_zero_min(const struct hdr_histogram* h) +{ + if (INT64_MAX == h->min_value) + { + return INT64_MAX; + } + + return lowest_equivalent_value(h, h->min_value); +} + +void hdr_reset_internal_counters(struct hdr_histogram* h) +{ + int min_non_zero_index = -1; + int max_index = -1; + int64_t observed_total_count = 0; + int i; + + for (i = 0; i < h->counts_len; i++) + { + int64_t count_at_index; + + if ((count_at_index = counts_get_direct(h, i)) > 0) + { + observed_total_count += count_at_index; + max_index = i; + if (min_non_zero_index == -1 && i != 0) + { + min_non_zero_index = i; + } + } + } + + if (max_index == -1) + { + h->max_value = 0; + } + else + { + int64_t max_value = hdr_value_at_index(h, max_index); + h->max_value = highest_equivalent_value(h, max_value); + } + + if (min_non_zero_index == -1) + { + h->min_value = INT64_MAX; + } + else + { + h->min_value = hdr_value_at_index(h, min_non_zero_index); + } + + h->total_count = observed_total_count; +} + +static int32_t buckets_needed_to_cover_value(int64_t value, int32_t sub_bucket_count, int32_t unit_magnitude) +{ + int64_t smallest_untrackable_value = ((int64_t) sub_bucket_count) << unit_magnitude; + int32_t buckets_needed = 1; + while (smallest_untrackable_value <= value) + { + if (smallest_untrackable_value > INT64_MAX / 2) + { + return buckets_needed + 1; + } + smallest_untrackable_value <<= 1; + buckets_needed++; + } + + return buckets_needed; +} + +/* ## ## ######## ## ## ####### ######## ## ## */ +/* ### ### ## ### ### ## ## ## ## ## ## */ +/* #### #### ## #### #### ## ## ## ## #### */ +/* ## ### ## ###### ## ### ## ## ## ######## ## */ +/* ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ######## ## ## ####### ## ## ## */ + +int hdr_calculate_bucket_config( + int64_t lowest_trackable_value, + int64_t highest_trackable_value, + int significant_figures, + struct hdr_histogram_bucket_config* cfg) +{ + int32_t sub_bucket_count_magnitude; + int64_t largest_value_with_single_unit_resolution; + + if (lowest_trackable_value < 1 || + significant_figures < 1 || 5 < significant_figures || + lowest_trackable_value * 2 > highest_trackable_value) + { + return EINVAL; + } + + cfg->lowest_trackable_value = lowest_trackable_value; + cfg->significant_figures = significant_figures; + cfg->highest_trackable_value = highest_trackable_value; + + largest_value_with_single_unit_resolution = 2 * power(10, significant_figures); + sub_bucket_count_magnitude = (int32_t) ceil(log((double)largest_value_with_single_unit_resolution) / log(2)); + cfg->sub_bucket_half_count_magnitude = ((sub_bucket_count_magnitude > 1) ? sub_bucket_count_magnitude : 1) - 1; + + cfg->unit_magnitude = (int32_t) floor(log((double)lowest_trackable_value) / log(2)); + + cfg->sub_bucket_count = (int32_t) pow(2, (cfg->sub_bucket_half_count_magnitude + 1)); + cfg->sub_bucket_half_count = cfg->sub_bucket_count / 2; + cfg->sub_bucket_mask = ((int64_t) cfg->sub_bucket_count - 1) << cfg->unit_magnitude; + + if (cfg->unit_magnitude + cfg->sub_bucket_half_count_magnitude > 61) + { + return EINVAL; + } + + cfg->bucket_count = buckets_needed_to_cover_value(highest_trackable_value, cfg->sub_bucket_count, (int32_t)cfg->unit_magnitude); + cfg->counts_len = (cfg->bucket_count + 1) * (cfg->sub_bucket_count / 2); + + return 0; +} + +void hdr_init_preallocated(struct hdr_histogram* h, struct hdr_histogram_bucket_config* cfg) +{ + h->lowest_trackable_value = cfg->lowest_trackable_value; + h->highest_trackable_value = cfg->highest_trackable_value; + h->unit_magnitude = (int32_t)cfg->unit_magnitude; + h->significant_figures = (int32_t)cfg->significant_figures; + h->sub_bucket_half_count_magnitude = cfg->sub_bucket_half_count_magnitude; + h->sub_bucket_half_count = cfg->sub_bucket_half_count; + h->sub_bucket_mask = cfg->sub_bucket_mask; + h->sub_bucket_count = cfg->sub_bucket_count; + h->min_value = INT64_MAX; + h->max_value = 0; + h->normalizing_index_offset = 0; + h->conversion_ratio = 1.0; + h->bucket_count = cfg->bucket_count; + h->counts_len = cfg->counts_len; + h->total_count = 0; +} + +int hdr_init( + int64_t lowest_trackable_value, + int64_t highest_trackable_value, + int significant_figures, + struct hdr_histogram** result) +{ + int64_t* counts; + struct hdr_histogram_bucket_config cfg; + struct hdr_histogram* histogram; + + int r = hdr_calculate_bucket_config(lowest_trackable_value, highest_trackable_value, significant_figures, &cfg); + if (r) + { + return r; + } + + counts = (int64_t*) calloc((size_t) cfg.counts_len, sizeof(int64_t)); + if (!counts) + { + return ENOMEM; + } + + histogram = (struct hdr_histogram*) calloc(1, sizeof(struct hdr_histogram)); + if (!histogram) + { + free(counts); + return ENOMEM; + } + + histogram->counts = counts; + + hdr_init_preallocated(histogram, &cfg); + *result = histogram; + + return 0; +} + +void hdr_close(struct hdr_histogram* h) +{ + if (h) { + free(h->counts); + free(h); + } +} + +int hdr_alloc(int64_t highest_trackable_value, int significant_figures, struct hdr_histogram** result) +{ + return hdr_init(1, highest_trackable_value, significant_figures, result); +} + +/* reset a histogram to zero. */ +void hdr_reset(struct hdr_histogram *h) +{ + h->total_count=0; + h->min_value = INT64_MAX; + h->max_value = 0; + memset(h->counts, 0, (sizeof(int64_t) * h->counts_len)); +} + +size_t hdr_get_memory_size(struct hdr_histogram *h) +{ + return sizeof(struct hdr_histogram) + h->counts_len * sizeof(int64_t); +} + +/* ## ## ######## ######## ### ######## ######## ###### */ +/* ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ######## ## ## ## ## ## ###### ###### */ +/* ## ## ## ## ## ######### ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## */ +/* ####### ## ######## ## ## ## ######## ###### */ + + +bool hdr_record_value(struct hdr_histogram* h, int64_t value) +{ + return hdr_record_values(h, value, 1); +} + +bool hdr_record_value_atomic(struct hdr_histogram* h, int64_t value) +{ + return hdr_record_values_atomic(h, value, 1); +} + +bool hdr_record_values(struct hdr_histogram* h, int64_t value, int64_t count) +{ + int32_t counts_index; + + if (value < 0) + { + return false; + } + + counts_index = counts_index_for(h, value); + + if (counts_index < 0 || h->counts_len <= counts_index) + { + return false; + } + + counts_inc_normalised(h, counts_index, count); + update_min_max(h, value); + + return true; +} + +bool hdr_record_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count) +{ + int32_t counts_index; + + if (value < 0) + { + return false; + } + + counts_index = counts_index_for(h, value); + + if (counts_index < 0 || h->counts_len <= counts_index) + { + return false; + } + + counts_inc_normalised_atomic(h, counts_index, count); + update_min_max_atomic(h, value); + + return true; +} + +bool hdr_record_corrected_value(struct hdr_histogram* h, int64_t value, int64_t expected_interval) +{ + return hdr_record_corrected_values(h, value, 1, expected_interval); +} + +bool hdr_record_corrected_value_atomic(struct hdr_histogram* h, int64_t value, int64_t expected_interval) +{ + return hdr_record_corrected_values_atomic(h, value, 1, expected_interval); +} + +bool hdr_record_corrected_values(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval) +{ + int64_t missing_value; + + if (!hdr_record_values(h, value, count)) + { + return false; + } + + if (expected_interval <= 0 || value <= expected_interval) + { + return true; + } + + missing_value = value - expected_interval; + for (; missing_value >= expected_interval; missing_value -= expected_interval) + { + if (!hdr_record_values(h, missing_value, count)) + { + return false; + } + } + + return true; +} + +bool hdr_record_corrected_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval) +{ + int64_t missing_value; + + if (!hdr_record_values_atomic(h, value, count)) + { + return false; + } + + if (expected_interval <= 0 || value <= expected_interval) + { + return true; + } + + missing_value = value - expected_interval; + for (; missing_value >= expected_interval; missing_value -= expected_interval) + { + if (!hdr_record_values_atomic(h, missing_value, count)) + { + return false; + } + } + + return true; +} + +int64_t hdr_add(struct hdr_histogram* h, const struct hdr_histogram* from) +{ + struct hdr_iter iter; + int64_t dropped = 0; + hdr_iter_recorded_init(&iter, from); + + while (hdr_iter_next(&iter)) + { + int64_t value = iter.value; + int64_t count = iter.count; + + if (!hdr_record_values(h, value, count)) + { + dropped += count; + } + } + + return dropped; +} + +int64_t hdr_add_while_correcting_for_coordinated_omission( + struct hdr_histogram* h, struct hdr_histogram* from, int64_t expected_interval) +{ + struct hdr_iter iter; + int64_t dropped = 0; + hdr_iter_recorded_init(&iter, from); + + while (hdr_iter_next(&iter)) + { + int64_t value = iter.value; + int64_t count = iter.count; + + if (!hdr_record_corrected_values(h, value, count, expected_interval)) + { + dropped += count; + } + } + + return dropped; +} + + + +/* ## ## ### ## ## ## ######## ###### */ +/* ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ###### ###### */ +/* ## ## ######### ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## */ +/* ### ## ## ######## ####### ######## ###### */ + + +int64_t hdr_max(const struct hdr_histogram* h) +{ + if (0 == h->max_value) + { + return 0; + } + + return highest_equivalent_value(h, h->max_value); +} + +int64_t hdr_min(const struct hdr_histogram* h) +{ + if (0 < hdr_count_at_index(h, 0)) + { + return 0; + } + + return non_zero_min(h); +} + +int64_t hdr_value_at_percentile(const struct hdr_histogram* h, double percentile) +{ + struct hdr_iter iter; + int64_t total = 0; + double requested_percentile = percentile < 100.0 ? percentile : 100.0; + int64_t count_at_percentile = + (int64_t) (((requested_percentile / 100) * h->total_count) + 0.5); + count_at_percentile = count_at_percentile > 1 ? count_at_percentile : 1; + + hdr_iter_init(&iter, h); + + while (hdr_iter_next(&iter)) + { + total += iter.count; + + if (total >= count_at_percentile) + { + int64_t value_from_index = iter.value; + return highest_equivalent_value(h, value_from_index); + } + } + + return 0; +} + +double hdr_mean(const struct hdr_histogram* h) +{ + struct hdr_iter iter; + int64_t total = 0; + + hdr_iter_init(&iter, h); + + while (hdr_iter_next(&iter)) + { + if (0 != iter.count) + { + total += iter.count * hdr_median_equivalent_value(h, iter.value); + } + } + + return (total * 1.0) / h->total_count; +} + +double hdr_stddev(const struct hdr_histogram* h) +{ + double mean = hdr_mean(h); + double geometric_dev_total = 0.0; + + struct hdr_iter iter; + hdr_iter_init(&iter, h); + + while (hdr_iter_next(&iter)) + { + if (0 != iter.count) + { + double dev = (hdr_median_equivalent_value(h, iter.value) * 1.0) - mean; + geometric_dev_total += (dev * dev) * iter.count; + } + } + + return sqrt(geometric_dev_total / h->total_count); +} + +bool hdr_values_are_equivalent(const struct hdr_histogram* h, int64_t a, int64_t b) +{ + return lowest_equivalent_value(h, a) == lowest_equivalent_value(h, b); +} + +int64_t hdr_lowest_equivalent_value(const struct hdr_histogram* h, int64_t value) +{ + return lowest_equivalent_value(h, value); +} + +int64_t hdr_count_at_value(const struct hdr_histogram* h, int64_t value) +{ + return counts_get_normalised(h, counts_index_for(h, value)); +} + +int64_t hdr_count_at_index(const struct hdr_histogram* h, int32_t index) +{ + return counts_get_normalised(h, index); +} + + +/* #### ######## ######## ######## ### ######## ####### ######## ###### */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ###### ######## ## ## ## ## ## ######## ###### */ +/* ## ## ## ## ## ######### ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* #### ## ######## ## ## ## ## ## ####### ## ## ###### */ + + +static bool has_buckets(struct hdr_iter* iter) +{ + return iter->counts_index < iter->h->counts_len; +} + +static bool has_next(struct hdr_iter* iter) +{ + return iter->cumulative_count < iter->total_count; +} + +static bool move_next(struct hdr_iter* iter) +{ + iter->counts_index++; + + if (!has_buckets(iter)) + { + return false; + } + + iter->count = counts_get_normalised(iter->h, iter->counts_index); + iter->cumulative_count += iter->count; + + iter->value = hdr_value_at_index(iter->h, iter->counts_index); + iter->highest_equivalent_value = highest_equivalent_value(iter->h, iter->value); + iter->lowest_equivalent_value = lowest_equivalent_value(iter->h, iter->value); + iter->median_equivalent_value = hdr_median_equivalent_value(iter->h, iter->value); + + return true; +} + +static int64_t peek_next_value_from_index(struct hdr_iter* iter) +{ + return hdr_value_at_index(iter->h, iter->counts_index + 1); +} + +static bool next_value_greater_than_reporting_level_upper_bound( + struct hdr_iter *iter, int64_t reporting_level_upper_bound) +{ + if (iter->counts_index >= iter->h->counts_len) + { + return false; + } + + return peek_next_value_from_index(iter) > reporting_level_upper_bound; +} + +static bool basic_iter_next(struct hdr_iter *iter) +{ + if (!has_next(iter) || iter->counts_index >= iter->h->counts_len) + { + return false; + } + + move_next(iter); + + return true; +} + +static void update_iterated_values(struct hdr_iter* iter, int64_t new_value_iterated_to) +{ + iter->value_iterated_from = iter->value_iterated_to; + iter->value_iterated_to = new_value_iterated_to; +} + +static bool all_values_iter_next(struct hdr_iter* iter) +{ + bool result = move_next(iter); + + if (result) + { + update_iterated_values(iter, iter->value); + } + + return result; +} + +void hdr_iter_init(struct hdr_iter* iter, const struct hdr_histogram* h) +{ + iter->h = h; + + iter->counts_index = -1; + iter->total_count = h->total_count; + iter->count = 0; + iter->cumulative_count = 0; + iter->value = 0; + iter->highest_equivalent_value = 0; + iter->value_iterated_from = 0; + iter->value_iterated_to = 0; + + iter->_next_fp = all_values_iter_next; +} + +bool hdr_iter_next(struct hdr_iter* iter) +{ + return iter->_next_fp(iter); +} + +/* ######## ######## ######## ###### ######## ## ## ######## #### ## ######## ###### */ +/* ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## #### ## ## ## ## ## ## */ +/* ######## ###### ######## ## ###### ## ## ## ## ## ## ###### ###### */ +/* ## ## ## ## ## ## ## #### ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## */ +/* ## ######## ## ## ###### ######## ## ## ## #### ######## ######## ###### */ + +static bool percentile_iter_next(struct hdr_iter* iter) +{ + int64_t temp, half_distance, percentile_reporting_ticks; + + struct hdr_iter_percentiles* percentiles = &iter->specifics.percentiles; + + if (!has_next(iter)) + { + if (percentiles->seen_last_value) + { + return false; + } + + percentiles->seen_last_value = true; + percentiles->percentile = 100.0; + + return true; + } + + if (iter->counts_index == -1 && !basic_iter_next(iter)) + { + return false; + } + + do + { + double current_percentile = (100.0 * (double) iter->cumulative_count) / iter->h->total_count; + if (iter->count != 0 && + percentiles->percentile_to_iterate_to <= current_percentile) + { + update_iterated_values(iter, highest_equivalent_value(iter->h, iter->value)); + + percentiles->percentile = percentiles->percentile_to_iterate_to; + temp = (int64_t)(log(100 / (100.0 - (percentiles->percentile_to_iterate_to))) / log(2)) + 1; + half_distance = (int64_t) pow(2, (double) temp); + percentile_reporting_ticks = percentiles->ticks_per_half_distance * half_distance; + percentiles->percentile_to_iterate_to += 100.0 / percentile_reporting_ticks; + + return true; + } + } + while (basic_iter_next(iter)); + + return true; +} + +void hdr_iter_percentile_init(struct hdr_iter* iter, const struct hdr_histogram* h, int32_t ticks_per_half_distance) +{ + iter->h = h; + + hdr_iter_init(iter, h); + + iter->specifics.percentiles.seen_last_value = false; + iter->specifics.percentiles.ticks_per_half_distance = ticks_per_half_distance; + iter->specifics.percentiles.percentile_to_iterate_to = 0.0; + iter->specifics.percentiles.percentile = 0.0; + + iter->_next_fp = percentile_iter_next; +} + +static void format_line_string(char* str, size_t len, int significant_figures, format_type format) +{ +#if defined(_MSC_VER) +#define snprintf _snprintf +#pragma warning(push) +#pragma warning(disable: 4996) +#endif + const char* format_str = "%s%d%s"; + + switch (format) + { + case CSV: + snprintf(str, len, format_str, "%.", significant_figures, "f,%f,%d,%.2f\n"); + break; + case CLASSIC: + snprintf(str, len, format_str, "%12.", significant_figures, "f %12f %12d %12.2f\n"); + break; + default: + snprintf(str, len, format_str, "%12.", significant_figures, "f %12f %12d %12.2f\n"); + } +#if defined(_MSC_VER) +#undef snprintf +#pragma warning(pop) +#endif +} + + +/* ######## ######## ###### ####### ######## ######## ######## ######## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ######## ###### ## ## ## ######## ## ## ###### ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ######## ###### ####### ## ## ######## ######## ######## */ + + +static bool recorded_iter_next(struct hdr_iter* iter) +{ + while (basic_iter_next(iter)) + { + if (iter->count != 0) + { + update_iterated_values(iter, iter->value); + + iter->specifics.recorded.count_added_in_this_iteration_step = iter->count; + return true; + } + } + + return false; +} + +void hdr_iter_recorded_init(struct hdr_iter* iter, const struct hdr_histogram* h) +{ + hdr_iter_init(iter, h); + + iter->specifics.recorded.count_added_in_this_iteration_step = 0; + + iter->_next_fp = recorded_iter_next; +} + +/* ## #### ## ## ######## ### ######## */ +/* ## ## ### ## ## ## ## ## ## */ +/* ## ## #### ## ## ## ## ## ## */ +/* ## ## ## ## ## ###### ## ## ######## */ +/* ## ## ## #### ## ######### ## ## */ +/* ## ## ## ### ## ## ## ## ## */ +/* ######## #### ## ## ######## ## ## ## ## */ + + +static bool iter_linear_next(struct hdr_iter* iter) +{ + struct hdr_iter_linear* linear = &iter->specifics.linear; + + linear->count_added_in_this_iteration_step = 0; + + if (has_next(iter) || + next_value_greater_than_reporting_level_upper_bound( + iter, linear->next_value_reporting_level_lowest_equivalent)) + { + do + { + if (iter->value >= linear->next_value_reporting_level_lowest_equivalent) + { + update_iterated_values(iter, linear->next_value_reporting_level); + + linear->next_value_reporting_level += linear->value_units_per_bucket; + linear->next_value_reporting_level_lowest_equivalent = + lowest_equivalent_value(iter->h, linear->next_value_reporting_level); + + return true; + } + + if (!move_next(iter)) + { + return true; + } + + linear->count_added_in_this_iteration_step += iter->count; + } + while (true); + } + + return false; +} + + +void hdr_iter_linear_init(struct hdr_iter* iter, const struct hdr_histogram* h, int64_t value_units_per_bucket) +{ + hdr_iter_init(iter, h); + + iter->specifics.linear.count_added_in_this_iteration_step = 0; + iter->specifics.linear.value_units_per_bucket = value_units_per_bucket; + iter->specifics.linear.next_value_reporting_level = value_units_per_bucket; + iter->specifics.linear.next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(h, value_units_per_bucket); + + iter->_next_fp = iter_linear_next; +} + +void hdr_iter_linear_set_value_units_per_bucket(struct hdr_iter* iter, int64_t value_units_per_bucket) +{ + iter->specifics.linear.value_units_per_bucket = value_units_per_bucket; +} + +/* ## ####### ###### ### ######## #### ######## ## ## ## ## #### ###### */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ### ### ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## #### #### ## ## */ +/* ## ## ## ## #### ## ## ######## ## ## ######### ## ### ## ## ## */ +/* ## ## ## ## ## ######### ## ## ## ## ## ## ## ## ## ## */ +/* ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## */ +/* ######## ####### ###### ## ## ## ## #### ## ## ## ## ## #### ###### */ + +static bool log_iter_next(struct hdr_iter *iter) +{ + struct hdr_iter_log* logarithmic = &iter->specifics.log; + + logarithmic->count_added_in_this_iteration_step = 0; + + if (has_next(iter) || + next_value_greater_than_reporting_level_upper_bound( + iter, logarithmic->next_value_reporting_level_lowest_equivalent)) + { + do + { + if (iter->value >= logarithmic->next_value_reporting_level_lowest_equivalent) + { + update_iterated_values(iter, logarithmic->next_value_reporting_level); + + logarithmic->next_value_reporting_level *= (int64_t)logarithmic->log_base; + logarithmic->next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(iter->h, logarithmic->next_value_reporting_level); + + return true; + } + + if (!move_next(iter)) + { + return true; + } + + logarithmic->count_added_in_this_iteration_step += iter->count; + } + while (true); + } + + return false; +} + +void hdr_iter_log_init( + struct hdr_iter* iter, + const struct hdr_histogram* h, + int64_t value_units_first_bucket, + double log_base) +{ + hdr_iter_init(iter, h); + iter->specifics.log.count_added_in_this_iteration_step = 0; + iter->specifics.log.log_base = log_base; + iter->specifics.log.next_value_reporting_level = value_units_first_bucket; + iter->specifics.log.next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(h, value_units_first_bucket); + + iter->_next_fp = log_iter_next; +} + +/* Printing. */ + +static const char* format_head_string(format_type format) +{ + switch (format) + { + case CSV: + return "%s,%s,%s,%s\n"; + case CLASSIC: + default: + return "%12s %12s %12s %12s\n\n"; + } +} + +static const char CLASSIC_FOOTER[] = + "#[Mean = %12.3f, StdDeviation = %12.3f]\n" + "#[Max = %12.3f, Total count = %12" PRIu64 "]\n" + "#[Buckets = %12d, SubBuckets = %12d]\n"; + +int hdr_percentiles_print( + struct hdr_histogram* h, FILE* stream, int32_t ticks_per_half_distance, + double value_scale, format_type format) +{ + char line_format[25]; + const char* head_format; + int rc = 0; + struct hdr_iter iter; + struct hdr_iter_percentiles * percentiles; + + format_line_string(line_format, 25, h->significant_figures, format); + head_format = format_head_string(format); + + hdr_iter_percentile_init(&iter, h, ticks_per_half_distance); + + if (fprintf( + stream, head_format, + "Value", "Percentile", "TotalCount", "1/(1-Percentile)") < 0) + { + rc = EIO; + goto cleanup; + } + + percentiles = &iter.specifics.percentiles; + while (hdr_iter_next(&iter)) + { + double value = iter.highest_equivalent_value / value_scale; + double percentile = percentiles->percentile / 100.0; + int64_t total_count = iter.cumulative_count; + double inverted_percentile = (1.0 / (1.0 - percentile)); + + if (fprintf( + stream, line_format, value, percentile, total_count, inverted_percentile) < 0) + { + rc = EIO; + goto cleanup; + } + } + + if (CLASSIC == format) + { + double mean = hdr_mean(h) / value_scale; + double stddev = hdr_stddev(h) / value_scale; + double max = hdr_max(h) / value_scale; + + if (fprintf( + stream, CLASSIC_FOOTER, mean, stddev, max, + h->total_count, h->bucket_count, h->sub_bucket_count) < 0) + { + rc = EIO; + goto cleanup; + } + } + + cleanup: + return rc; +} diff --git a/redis.submodule/deps/hdr_histogram/hdr_histogram.h b/redis.submodule/deps/hdr_histogram/hdr_histogram.h new file mode 100644 index 0000000..c26c56b --- /dev/null +++ b/redis.submodule/deps/hdr_histogram/hdr_histogram.h @@ -0,0 +1,509 @@ +/** + * hdr_histogram.h + * Written by Michael Barker and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + * + * The source for the hdr_histogram utilises a few C99 constructs, specifically + * the use of stdint/stdbool and inline variable declaration. + */ + +#ifndef HDR_HISTOGRAM_H +#define HDR_HISTOGRAM_H 1 + +#include +#include +#include + +struct hdr_histogram +{ + int64_t lowest_trackable_value; + int64_t highest_trackable_value; + int32_t unit_magnitude; + int32_t significant_figures; + int32_t sub_bucket_half_count_magnitude; + int32_t sub_bucket_half_count; + int64_t sub_bucket_mask; + int32_t sub_bucket_count; + int32_t bucket_count; + int64_t min_value; + int64_t max_value; + int32_t normalizing_index_offset; + double conversion_ratio; + int32_t counts_len; + int64_t total_count; + int64_t* counts; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Allocate the memory and initialise the hdr_histogram. + * + * Due to the size of the histogram being the result of some reasonably + * involved math on the input parameters this function it is tricky to stack allocate. + * The histogram should be released with hdr_close + * + * @param lowest_trackable_value The smallest possible value to be put into the + * histogram. + * @param highest_trackable_value The largest possible value to be put into the + * histogram. + * @param significant_figures The level of precision for this histogram, i.e. the number + * of figures in a decimal number that will be maintained. E.g. a value of 3 will mean + * the results from the histogram will be accurate up to the first three digits. Must + * be a value between 1 and 5 (inclusive). + * @param result Output parameter to capture allocated histogram. + * @return 0 on success, EINVAL if lowest_trackable_value is < 1 or the + * significant_figure value is outside of the allowed range, ENOMEM if malloc + * failed. + */ +int hdr_init( + int64_t lowest_trackable_value, + int64_t highest_trackable_value, + int significant_figures, + struct hdr_histogram** result); + +/** + * Free the memory and close the hdr_histogram. + * + * @param h The histogram you want to close. + */ +void hdr_close(struct hdr_histogram* h); + +/** + * Allocate the memory and initialise the hdr_histogram. This is the equivalent of calling + * hdr_init(1, highest_trackable_value, significant_figures, result); + * + * @deprecated use hdr_init. + */ +int hdr_alloc(int64_t highest_trackable_value, int significant_figures, struct hdr_histogram** result); + + +/** + * Reset a histogram to zero - empty out a histogram and re-initialise it + * + * If you want to re-use an existing histogram, but reset everything back to zero, this + * is the routine to use. + * + * @param h The histogram you want to reset to empty. + * + */ +void hdr_reset(struct hdr_histogram* h); + +/** + * Get the memory size of the hdr_histogram. + * + * @param h "This" pointer + * @return The amount of memory used by the hdr_histogram in bytes + */ +size_t hdr_get_memory_size(struct hdr_histogram* h); + +/** + * Records a value in the histogram, will round this value of to a precision at or better + * than the significant_figure specified at construction time. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_value(struct hdr_histogram* h, int64_t value); + +/** + * Records a value in the histogram, will round this value of to a precision at or better + * than the significant_figure specified at construction time. + * + * Will record this value atomically, however the whole structure may appear inconsistent + * when read concurrently with this update. Do NOT mix calls to this method with calls + * to non-atomic updates. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_value_atomic(struct hdr_histogram* h, int64_t value); + +/** + * Records count values in the histogram, will round this value of to a + * precision at or better than the significant_figure specified at construction + * time. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param count Number of 'value's to add to the histogram + * @return false if any value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_values(struct hdr_histogram* h, int64_t value, int64_t count); + +/** + * Records count values in the histogram, will round this value of to a + * precision at or better than the significant_figure specified at construction + * time. + * + * Will record this value atomically, however the whole structure may appear inconsistent + * when read concurrently with this update. Do NOT mix calls to this method with calls + * to non-atomic updates. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param count Number of 'value's to add to the histogram + * @return false if any value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count); + +/** + * Record a value in the histogram and backfill based on an expected interval. + * + * Records a value in the histogram, will round this value of to a precision at or better + * than the significant_figure specified at contruction time. This is specifically used + * for recording latency. If the value is larger than the expected_interval then the + * latency recording system has experienced co-ordinated omission. This method fills in the + * values that would have occured had the client providing the load not been blocked. + + * @param h "This" pointer + * @param value Value to add to the histogram + * @param expected_interval The delay between recording values. + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_corrected_value(struct hdr_histogram* h, int64_t value, int64_t expexcted_interval); + +/** + * Record a value in the histogram and backfill based on an expected interval. + * + * Records a value in the histogram, will round this value of to a precision at or better + * than the significant_figure specified at contruction time. This is specifically used + * for recording latency. If the value is larger than the expected_interval then the + * latency recording system has experienced co-ordinated omission. This method fills in the + * values that would have occured had the client providing the load not been blocked. + * + * Will record this value atomically, however the whole structure may appear inconsistent + * when read concurrently with this update. Do NOT mix calls to this method with calls + * to non-atomic updates. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param expected_interval The delay between recording values. + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_corrected_value_atomic(struct hdr_histogram* h, int64_t value, int64_t expexcted_interval); + +/** + * Record a value in the histogram 'count' times. Applies the same correcting logic + * as 'hdr_record_corrected_value'. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param count Number of 'value's to add to the histogram + * @param expected_interval The delay between recording values. + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_corrected_values(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval); + +/** + * Record a value in the histogram 'count' times. Applies the same correcting logic + * as 'hdr_record_corrected_value'. + * + * Will record this value atomically, however the whole structure may appear inconsistent + * when read concurrently with this update. Do NOT mix calls to this method with calls + * to non-atomic updates. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param count Number of 'value's to add to the histogram + * @param expected_interval The delay between recording values. + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_corrected_values_atomic(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval); + +/** + * Adds all of the values from 'from' to 'this' histogram. Will return the + * number of values that are dropped when copying. Values will be dropped + * if they around outside of h.lowest_trackable_value and + * h.highest_trackable_value. + * + * @param h "This" pointer + * @param from Histogram to copy values from. + * @return The number of values dropped when copying. + */ +int64_t hdr_add(struct hdr_histogram* h, const struct hdr_histogram* from); + +/** + * Adds all of the values from 'from' to 'this' histogram. Will return the + * number of values that are dropped when copying. Values will be dropped + * if they around outside of h.lowest_trackable_value and + * h.highest_trackable_value. + * + * @param h "This" pointer + * @param from Histogram to copy values from. + * @return The number of values dropped when copying. + */ +int64_t hdr_add_while_correcting_for_coordinated_omission( + struct hdr_histogram* h, struct hdr_histogram* from, int64_t expected_interval); + +/** + * Get minimum value from the histogram. Will return 2^63-1 if the histogram + * is empty. + * + * @param h "This" pointer + */ +int64_t hdr_min(const struct hdr_histogram* h); + +/** + * Get maximum value from the histogram. Will return 0 if the histogram + * is empty. + * + * @param h "This" pointer + */ +int64_t hdr_max(const struct hdr_histogram* h); + +/** + * Get the value at a specific percentile. + * + * @param h "This" pointer. + * @param percentile The percentile to get the value for + */ +int64_t hdr_value_at_percentile(const struct hdr_histogram* h, double percentile); + +/** + * Gets the standard deviation for the values in the histogram. + * + * @param h "This" pointer + * @return The standard deviation + */ +double hdr_stddev(const struct hdr_histogram* h); + +/** + * Gets the mean for the values in the histogram. + * + * @param h "This" pointer + * @return The mean + */ +double hdr_mean(const struct hdr_histogram* h); + +/** + * Determine if two values are equivalent with the histogram's resolution. + * Where "equivalent" means that value samples recorded for any two + * equivalent values are counted in a common total count. + * + * @param h "This" pointer + * @param a first value to compare + * @param b second value to compare + * @return 'true' if values are equivalent with the histogram's resolution. + */ +bool hdr_values_are_equivalent(const struct hdr_histogram* h, int64_t a, int64_t b); + +/** + * Get the lowest value that is equivalent to the given value within the histogram's resolution. + * Where "equivalent" means that value samples recorded for any two + * equivalent values are counted in a common total count. + * + * @param h "This" pointer + * @param value The given value + * @return The lowest value that is equivalent to the given value within the histogram's resolution. + */ +int64_t hdr_lowest_equivalent_value(const struct hdr_histogram* h, int64_t value); + +/** + * Get the count of recorded values at a specific value + * (to within the histogram resolution at the value level). + * + * @param h "This" pointer + * @param value The value for which to provide the recorded count + * @return The total count of values recorded in the histogram within the value range that is + * {@literal >=} lowestEquivalentValue(value) and {@literal <=} highestEquivalentValue(value) + */ +int64_t hdr_count_at_value(const struct hdr_histogram* h, int64_t value); + +int64_t hdr_count_at_index(const struct hdr_histogram* h, int32_t index); + +int64_t hdr_value_at_index(const struct hdr_histogram* h, int32_t index); + +struct hdr_iter_percentiles +{ + bool seen_last_value; + int32_t ticks_per_half_distance; + double percentile_to_iterate_to; + double percentile; +}; + +struct hdr_iter_recorded +{ + int64_t count_added_in_this_iteration_step; +}; + +struct hdr_iter_linear +{ + int64_t value_units_per_bucket; + int64_t count_added_in_this_iteration_step; + int64_t next_value_reporting_level; + int64_t next_value_reporting_level_lowest_equivalent; +}; + +struct hdr_iter_log +{ + double log_base; + int64_t count_added_in_this_iteration_step; + int64_t next_value_reporting_level; + int64_t next_value_reporting_level_lowest_equivalent; +}; + +/** + * The basic iterator. This is a generic structure + * that supports all of the types of iteration. Use + * the appropriate initialiser to get the desired + * iteration. + * + * @ + */ +struct hdr_iter +{ + const struct hdr_histogram* h; + /** raw index into the counts array */ + int32_t counts_index; + /** snapshot of the length at the time the iterator is created */ + int64_t total_count; + /** value directly from array for the current counts_index */ + int64_t count; + /** sum of all of the counts up to and including the count at this index */ + int64_t cumulative_count; + /** The current value based on counts_index */ + int64_t value; + int64_t highest_equivalent_value; + int64_t lowest_equivalent_value; + int64_t median_equivalent_value; + int64_t value_iterated_from; + int64_t value_iterated_to; + + union + { + struct hdr_iter_percentiles percentiles; + struct hdr_iter_recorded recorded; + struct hdr_iter_linear linear; + struct hdr_iter_log log; + } specifics; + + bool (* _next_fp)(struct hdr_iter* iter); + +}; + +/** + * Initalises the basic iterator. + * + * @param itr 'This' pointer + * @param h The histogram to iterate over + */ +void hdr_iter_init(struct hdr_iter* iter, const struct hdr_histogram* h); + +/** + * Initialise the iterator for use with percentiles. + */ +void hdr_iter_percentile_init(struct hdr_iter* iter, const struct hdr_histogram* h, int32_t ticks_per_half_distance); + +/** + * Initialise the iterator for use with recorded values. + */ +void hdr_iter_recorded_init(struct hdr_iter* iter, const struct hdr_histogram* h); + +/** + * Initialise the iterator for use with linear values. + */ +void hdr_iter_linear_init( + struct hdr_iter* iter, + const struct hdr_histogram* h, + int64_t value_units_per_bucket); + +/** + * Update the iterator value units per bucket + */ +void hdr_iter_linear_set_value_units_per_bucket(struct hdr_iter* iter, int64_t value_units_per_bucket); + +/** + * Initialise the iterator for use with logarithmic values + */ +void hdr_iter_log_init( + struct hdr_iter* iter, + const struct hdr_histogram* h, + int64_t value_units_first_bucket, + double log_base); + +/** + * Iterate to the next value for the iterator. If there are no more values + * available return faluse. + * + * @param itr 'This' pointer + * @return 'false' if there are no values remaining for this iterator. + */ +bool hdr_iter_next(struct hdr_iter* iter); + +typedef enum +{ + CLASSIC, + CSV +} format_type; + +/** + * Print out a percentile based histogram to the supplied stream. Note that + * this call will not flush the FILE, this is left up to the user. + * + * @param h 'This' pointer + * @param stream The FILE to write the output to + * @param ticks_per_half_distance The number of iteration steps per half-distance to 100% + * @param value_scale Scale the output values by this amount + * @param format_type Format to use, e.g. CSV. + * @return 0 on success, error code on failure. EIO if an error occurs writing + * the output. + */ +int hdr_percentiles_print( + struct hdr_histogram* h, FILE* stream, int32_t ticks_per_half_distance, + double value_scale, format_type format); + +/** +* Internal allocation methods, used by hdr_dbl_histogram. +*/ +struct hdr_histogram_bucket_config +{ + int64_t lowest_trackable_value; + int64_t highest_trackable_value; + int64_t unit_magnitude; + int64_t significant_figures; + int32_t sub_bucket_half_count_magnitude; + int32_t sub_bucket_half_count; + int64_t sub_bucket_mask; + int32_t sub_bucket_count; + int32_t bucket_count; + int32_t counts_len; +}; + +int hdr_calculate_bucket_config( + int64_t lowest_trackable_value, + int64_t highest_trackable_value, + int significant_figures, + struct hdr_histogram_bucket_config* cfg); + +void hdr_init_preallocated(struct hdr_histogram* h, struct hdr_histogram_bucket_config* cfg); + +int64_t hdr_size_of_equivalent_value_range(const struct hdr_histogram* h, int64_t value); + +int64_t hdr_next_non_equivalent_value(const struct hdr_histogram* h, int64_t value); + +int64_t hdr_median_equivalent_value(const struct hdr_histogram* h, int64_t value); + +/** + * Used to reset counters after importing data manuallying into the histogram, used by the logging code + * and other custom serialisation tools. + */ +void hdr_reset_internal_counters(struct hdr_histogram* h); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/redis.submodule/deps/hiredis/.gitignore b/redis.submodule/deps/hiredis/.gitignore index c44b5c5..056959f 100644 --- a/redis.submodule/deps/hiredis/.gitignore +++ b/redis.submodule/deps/hiredis/.gitignore @@ -5,3 +5,5 @@ /*.dylib /*.a /*.pc +*.dSYM +tags diff --git a/redis.submodule/deps/hiredis/.travis.yml b/redis.submodule/deps/hiredis/.travis.yml index ad08076..f9a9460 100644 --- a/redis.submodule/deps/hiredis/.travis.yml +++ b/redis.submodule/deps/hiredis/.travis.yml @@ -1,5 +1,4 @@ language: c -sudo: false compiler: - gcc - clang @@ -8,11 +7,34 @@ os: - linux - osx +dist: bionic + +branches: + only: + - staging + - trying + - master + - /^release\/.*$/ + +install: + - if [ "$BITS" == "64" ]; then + wget https://github.com/redis/redis/archive/6.0.6.tar.gz; + tar -xzvf 6.0.6.tar.gz; + pushd redis-6.0.6 && BUILD_TLS=yes make && export PATH=$PWD/src:$PATH && popd; + fi + before_script: - - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then brew update; brew install redis; fi + - if [ "$TRAVIS_OS_NAME" == "osx" ]; then + curl -O https://distfiles.macports.org/MacPorts/MacPorts-2.6.2-10.13-HighSierra.pkg; + sudo installer -pkg MacPorts-2.6.2-10.13-HighSierra.pkg -target /; + export PATH=$PATH:/opt/local/bin && sudo port -v selfupdate; + sudo port -N install openssl redis; + fi; addons: apt: + sources: + - sourceline: 'ppa:chris-lea/redis-server' packages: - libc6-dbg - libc6-dev @@ -20,20 +42,90 @@ addons: - libc6-dev-i386 - libc6-dbg:i386 - gcc-multilib + - g++-multilib + - libssl-dev + - libssl-dev:i386 - valgrind + - redis env: - - CFLAGS="-Werror" - - PRE="valgrind --track-origins=yes --leak-check=full" - - TARGET="32bit" TARGET_VARS="32bit-vars" CFLAGS="-Werror" - - TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" + - BITS="32" + - BITS="64" -matrix: - exclude: - - os: osx - env: PRE="valgrind --track-origins=yes --leak-check=full" +script: + - EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON"; + if [ "$BITS" == "64" ]; then + EXTRA_CMAKE_OPTS="$EXTRA_CMAKE_OPTS -DENABLE_SSL_TESTS:BOOL=ON"; + fi; + if [ "$TRAVIS_OS_NAME" == "osx" ]; then + if [ "$BITS" == "32" ]; then + CFLAGS="-m32 -Werror"; + CXXFLAGS="-m32 -Werror"; + LDFLAGS="-m32"; + EXTRA_CMAKE_OPTS=; + else + CFLAGS="-Werror"; + CXXFLAGS="-Werror"; + fi; + else + TEST_PREFIX="valgrind --track-origins=yes --leak-check=full"; + if [ "$BITS" == "32" ]; then + CFLAGS="-m32 -Werror"; + CXXFLAGS="-m32 -Werror"; + LDFLAGS="-m32"; + EXTRA_CMAKE_OPTS=; + else + CFLAGS="-Werror"; + CXXFLAGS="-Werror"; + fi; + fi; + export CFLAGS CXXFLAGS LDFLAGS TEST_PREFIX EXTRA_CMAKE_OPTS + - make && make clean; + if [ "$TRAVIS_OS_NAME" == "osx" ]; then + if [ "$BITS" == "64" ]; then + OPENSSL_PREFIX="$(ls -d /usr/local/Cellar/openssl@1.1/*)" USE_SSL=1 make; + fi; + else + USE_SSL=1 make; + fi; + - mkdir build/ && cd build/ + - cmake .. ${EXTRA_CMAKE_OPTS} + - make VERBOSE=1 + - if [ "$BITS" == "64" ]; then + TEST_SSL=1 SKIPS_AS_FAILS=1 ctest -V; + else + SKIPS_AS_FAILS=1 ctest -V; + fi; - - os: osx - env: TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" +jobs: + include: + # Windows MinGW cross compile on Linux + - os: linux + dist: xenial + compiler: mingw + addons: + apt: + packages: + - ninja-build + - gcc-mingw-w64-x86-64 + - g++-mingw-w64-x86-64 + script: + - mkdir build && cd build + - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_BUILD_WITH_INSTALL_RPATH=on + - ninja -v -script: make $TARGET CFLAGS="$CFLAGS" && make check PRE="$PRE" && make $TARGET_VARS hiredis-example + # Windows MSVC 2017 + - os: windows + compiler: msvc + env: + - MATRIX_EVAL="CC=cl.exe && CXX=cl.exe" + before_install: + - eval "${MATRIX_EVAL}" + install: + - choco install ninja + - choco install -y memurai-developer + script: + - mkdir build && cd build + - cmd.exe //C 'C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat' amd64 '&&' + cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DENABLE_EXAMPLES=ON '&&' ninja -v + - ./hiredis-test.exe diff --git a/redis.submodule/deps/hiredis/CHANGELOG.md b/redis.submodule/deps/hiredis/CHANGELOG.md index f92bcb3..271f1fc 100644 --- a/redis.submodule/deps/hiredis/CHANGELOG.md +++ b/redis.submodule/deps/hiredis/CHANGELOG.md @@ -1,18 +1,215 @@ -### 1.0.0 (unreleased) +## [1.0.0](https://github.com/redis/hiredis/tree/v1.0.0) - (2020-08-03) -**Fixes**: +Announcing Hiredis v1.0.0, which adds support for RESP3, SSL connections, allocator injection, and better Windows support! :tada: -* Catch a buffer overflow when formatting the error message -* Import latest upstream sds. This breaks applications that are linked against the old hiredis v0.13 -* Fix warnings, when compiled with -Wshadow -* Make hiredis compile in Cygwin on Windows, now CI-tested +_A big thanks to everyone who helped with this release. The following list includes everyone who contributed at least five lines, sorted by lines contributed._ :sparkling_heart: + +[Michael Grunder](https://github.com/michael-grunder), [Yossi Gottlieb](https://github.com/yossigo), +[Mark Nunberg](https://github.com/mnunberg), [Marcus Geelnard](https://github.com/mbitsnbites), +[Justin Brewer](https://github.com/justinbrewer), [Valentino Geron](https://github.com/valentinogeron), +[Minun Dragonation](https://github.com/dragonation), [Omri Steiner](https://github.com/OmriSteiner), +[Sangmoon Yi](https://github.com/jman-krafton), [Jinjiazh](https://github.com/jinjiazhang), +[Odin Hultgren Van Der Horst](https://github.com/Miniwoffer), [Muhammad Zahalqa](https://github.com/tryfinally), +[Nick Rivera](https://github.com/heronr), [Qi Yang](https://github.com/movebean), +[kevin1018](https://github.com/kevin1018) + +[Full Changelog](https://github.com/redis/hiredis/compare/v0.14.1...v1.0.0) **BREAKING CHANGES**: +* `redisOptions` now has two timeout fields. One for connecting, and one for commands. If you're presently using `options->timeout` you will need to change it to use `options->connect_timeout`. (See [example](https://github.com/redis/hiredis/commit/38b5ae543f5c99eb4ccabbe277770fc6bc81226f#diff-86ba39d37aa829c8c82624cce4f049fbL36)) + +* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now protocol errors. This is consistent + with the RESP specification. On 32-bit platforms, the upper bound is lowered to `SIZE_MAX`. + +* `redisReplyObjectFunctions.createArray` now takes `size_t` for its length parameter. + +**New features:** +- Support for RESP3 + [\#697](https://github.com/redis/hiredis/pull/697), + [\#805](https://github.com/redis/hiredis/pull/805), + [\#819](https://github.com/redis/hiredis/pull/819), + [\#841](https://github.com/redis/hiredis/pull/841) + ([Yossi Gottlieb](https://github.com/yossigo), [Michael Grunder](https://github.com/michael-grunder)) +- Support for SSL connections + [\#645](https://github.com/redis/hiredis/pull/645), + [\#699](https://github.com/redis/hiredis/pull/699), + [\#702](https://github.com/redis/hiredis/pull/702), + [\#708](https://github.com/redis/hiredis/pull/708), + [\#711](https://github.com/redis/hiredis/pull/711), + [\#821](https://github.com/redis/hiredis/pull/821), + [more](https://github.com/redis/hiredis/pulls?q=is%3Apr+is%3Amerged+SSL) + ([Mark Nunberg](https://github.com/mnunberg), [Yossi Gottlieb](https://github.com/yossigo)) +- Run-time allocator injection + [\#800](https://github.com/redis/hiredis/pull/800) + ([Michael Grunder](https://github.com/michael-grunder)) +- Improved Windows support (including MinGW and Windows CI) + [\#652](https://github.com/redis/hiredis/pull/652), + [\#663](https://github.com/redis/hiredis/pull/663) + ([Marcus Geelnard](https://www.bitsnbites.eu/author/m/)) +- Adds support for distinct connect and command timeouts + [\#839](https://github.com/redis/hiredis/pull/839), + [\#829](https://github.com/redis/hiredis/pull/829) + ([Valentino Geron](https://github.com/valentinogeron)) +- Add generic pointer and destructor to `redisContext` that users can use for context. + [\#855](https://github.com/redis/hiredis/pull/855) + ([Michael Grunder](https://github.com/michael-grunder)) + +**Closed issues (that involved code changes):** + +- Makefile does not install TLS libraries [\#809](https://github.com/redis/hiredis/issues/809) +- redisConnectWithOptions should not set command timeout [\#722](https://github.com/redis/hiredis/issues/722), [\#829](https://github.com/redis/hiredis/pull/829) ([valentinogeron](https://github.com/valentinogeron)) +- Fix integer overflow in `sdsrange` [\#827](https://github.com/redis/hiredis/issues/827) +- INFO & CLUSTER commands failed when using RESP3 [\#802](https://github.com/redis/hiredis/issues/802) +- Windows compatibility patches [\#687](https://github.com/redis/hiredis/issues/687), [\#838](https://github.com/redis/hiredis/issues/838), [\#842](https://github.com/redis/hiredis/issues/842) +- RESP3 PUSH messages incorrectly use pending callback [\#825](https://github.com/redis/hiredis/issues/825) +- Asynchronous PSUBSCRIBE command fails when using RESP3 [\#815](https://github.com/redis/hiredis/issues/815) +- New SSL API [\#804](https://github.com/redis/hiredis/issues/804), [\#813](https://github.com/redis/hiredis/issues/813) +- Hard-coded limit of nested reply depth [\#794](https://github.com/redis/hiredis/issues/794) +- Fix TCP_NODELAY in Windows/OSX [\#679](https://github.com/redis/hiredis/issues/679), [\#690](https://github.com/redis/hiredis/issues/690), [\#779](https://github.com/redis/hiredis/issues/779), [\#785](https://github.com/redis/hiredis/issues/785), +- Added timers to libev adapter. [\#778](https://github.com/redis/hiredis/issues/778), [\#795](https://github.com/redis/hiredis/pull/795) +- Initialization discards const qualifier [\#777](https://github.com/redis/hiredis/issues/777) +- \[BUG\]\[MinGW64\] Error setting socket timeout [\#775](https://github.com/redis/hiredis/issues/775) +- undefined reference to hi_malloc [\#769](https://github.com/redis/hiredis/issues/769) +- hiredis pkg-config file incorrectly ignores multiarch libdir spec'n [\#767](https://github.com/redis/hiredis/issues/767) +- Don't use -G to build shared object on Solaris [\#757](https://github.com/redis/hiredis/issues/757) +- error when make USE\_SSL=1 [\#748](https://github.com/redis/hiredis/issues/748) +- Allow to change SSL Mode [\#646](https://github.com/redis/hiredis/issues/646) +- hiredis/adapters/libevent.h memleak [\#618](https://github.com/redis/hiredis/issues/618) +- redisLibuvPoll crash when server closes the connetion [\#545](https://github.com/redis/hiredis/issues/545) +- about redisAsyncDisconnect question [\#518](https://github.com/redis/hiredis/issues/518) +- hiredis adapters libuv error for help [\#508](https://github.com/redis/hiredis/issues/508) +- API/ABI changes analysis [\#506](https://github.com/redis/hiredis/issues/506) +- Memory leak patch in Redis [\#502](https://github.com/redis/hiredis/issues/502) +- Remove the depth limitation [\#421](https://github.com/redis/hiredis/issues/421) + +**Merged pull requests:** + +- Move SSL management to a distinct private pointer [\#855](https://github.com/redis/hiredis/pull/855) ([michael-grunder](https://github.com/michael-grunder)) +- Move include to sockcompat.h to maintain style [\#850](https://github.com/redis/hiredis/pull/850) ([michael-grunder](https://github.com/michael-grunder)) +- Remove erroneous tag and add license to push example [\#849](https://github.com/redis/hiredis/pull/849) ([michael-grunder](https://github.com/michael-grunder)) +- fix windows compiling with mingw [\#848](https://github.com/redis/hiredis/pull/848) ([rmalizia44](https://github.com/rmalizia44)) +- Some Windows quality of life improvements. [\#846](https://github.com/redis/hiredis/pull/846) ([michael-grunder](https://github.com/michael-grunder)) +- Use \_WIN32 define instead of WIN32 [\#845](https://github.com/redis/hiredis/pull/845) ([michael-grunder](https://github.com/michael-grunder)) +- Non Linux CI fixes [\#844](https://github.com/redis/hiredis/pull/844) ([michael-grunder](https://github.com/michael-grunder)) +- Resp3 oob push support [\#841](https://github.com/redis/hiredis/pull/841) ([michael-grunder](https://github.com/michael-grunder)) +- fix \#785: defer TCP\_NODELAY in async tcp connections [\#836](https://github.com/redis/hiredis/pull/836) ([OmriSteiner](https://github.com/OmriSteiner)) +- sdsrange overflow fix [\#830](https://github.com/redis/hiredis/pull/830) ([michael-grunder](https://github.com/michael-grunder)) +- Use explicit pointer casting for c++ compatibility [\#826](https://github.com/redis/hiredis/pull/826) ([aureus1](https://github.com/aureus1)) +- Document allocator injection and completeness fix in test.c [\#824](https://github.com/redis/hiredis/pull/824) ([michael-grunder](https://github.com/michael-grunder)) +- Use unique names for allocator struct members [\#823](https://github.com/redis/hiredis/pull/823) ([michael-grunder](https://github.com/michael-grunder)) +- New SSL API to replace redisSecureConnection\(\). [\#821](https://github.com/redis/hiredis/pull/821) ([yossigo](https://github.com/yossigo)) +- Add logic to handle RESP3 push messages [\#819](https://github.com/redis/hiredis/pull/819) ([michael-grunder](https://github.com/michael-grunder)) +- Use standrad isxdigit instead of custom helper function. [\#814](https://github.com/redis/hiredis/pull/814) ([tryfinally](https://github.com/tryfinally)) +- Fix missing SSL build/install options. [\#812](https://github.com/redis/hiredis/pull/812) ([yossigo](https://github.com/yossigo)) +- Add link to ABI tracker [\#808](https://github.com/redis/hiredis/pull/808) ([michael-grunder](https://github.com/michael-grunder)) +- Resp3 verbatim string support [\#805](https://github.com/redis/hiredis/pull/805) ([michael-grunder](https://github.com/michael-grunder)) +- Allow users to replace allocator and handle OOM everywhere. [\#800](https://github.com/redis/hiredis/pull/800) ([michael-grunder](https://github.com/michael-grunder)) +- Remove nested depth limitation. [\#797](https://github.com/redis/hiredis/pull/797) ([michael-grunder](https://github.com/michael-grunder)) +- Attempt to fix compilation on Solaris [\#796](https://github.com/redis/hiredis/pull/796) ([michael-grunder](https://github.com/michael-grunder)) +- Support timeouts in libev adapater [\#795](https://github.com/redis/hiredis/pull/795) ([michael-grunder](https://github.com/michael-grunder)) +- Fix pkgconfig when installing to a custom lib dir [\#793](https://github.com/redis/hiredis/pull/793) ([michael-grunder](https://github.com/michael-grunder)) +- Fix USE\_SSL=1 make/cmake on OSX and CMake tests [\#789](https://github.com/redis/hiredis/pull/789) ([michael-grunder](https://github.com/michael-grunder)) +- Use correct libuv call on Windows [\#784](https://github.com/redis/hiredis/pull/784) ([michael-grunder](https://github.com/michael-grunder)) +- Added CMake package config and fixed hiredis\_ssl on Windows [\#783](https://github.com/redis/hiredis/pull/783) ([michael-grunder](https://github.com/michael-grunder)) +- CMake: Set hiredis\_ssl shared object version. [\#780](https://github.com/redis/hiredis/pull/780) ([yossigo](https://github.com/yossigo)) +- Win32 tests and timeout fix [\#776](https://github.com/redis/hiredis/pull/776) ([michael-grunder](https://github.com/michael-grunder)) +- Provides an optional cleanup callback for async data. [\#768](https://github.com/redis/hiredis/pull/768) ([heronr](https://github.com/heronr)) +- Housekeeping fixes [\#764](https://github.com/redis/hiredis/pull/764) ([michael-grunder](https://github.com/michael-grunder)) +- install alloc.h [\#756](https://github.com/redis/hiredis/pull/756) ([ch1aki](https://github.com/ch1aki)) +- fix spelling mistakes [\#746](https://github.com/redis/hiredis/pull/746) ([ShooterIT](https://github.com/ShooterIT)) +- Free the reply in redisGetReply when passed NULL [\#741](https://github.com/redis/hiredis/pull/741) ([michael-grunder](https://github.com/michael-grunder)) +- Fix dead code in sslLogCallback relating to should\_log variable. [\#737](https://github.com/redis/hiredis/pull/737) ([natoscott](https://github.com/natoscott)) +- Fix typo in dict.c. [\#731](https://github.com/redis/hiredis/pull/731) ([Kevin-Xi](https://github.com/Kevin-Xi)) +- Adding an option to DISABLE\_TESTS [\#727](https://github.com/redis/hiredis/pull/727) ([pbotros](https://github.com/pbotros)) +- Update README with SSL support. [\#720](https://github.com/redis/hiredis/pull/720) ([yossigo](https://github.com/yossigo)) +- Fixes leaks in unit tests [\#715](https://github.com/redis/hiredis/pull/715) ([michael-grunder](https://github.com/michael-grunder)) +- SSL Tests [\#711](https://github.com/redis/hiredis/pull/711) ([yossigo](https://github.com/yossigo)) +- SSL Reorganization [\#708](https://github.com/redis/hiredis/pull/708) ([yossigo](https://github.com/yossigo)) +- Fix MSVC build. [\#706](https://github.com/redis/hiredis/pull/706) ([yossigo](https://github.com/yossigo)) +- SSL: Properly report SSL\_connect\(\) errors. [\#702](https://github.com/redis/hiredis/pull/702) ([yossigo](https://github.com/yossigo)) +- Silent SSL trace to stdout by default. [\#699](https://github.com/redis/hiredis/pull/699) ([yossigo](https://github.com/yossigo)) +- Port RESP3 support from Redis. [\#697](https://github.com/redis/hiredis/pull/697) ([yossigo](https://github.com/yossigo)) +- Removed whitespace before newline [\#691](https://github.com/redis/hiredis/pull/691) ([Miniwoffer](https://github.com/Miniwoffer)) +- Add install adapters header files [\#688](https://github.com/redis/hiredis/pull/688) ([kevin1018](https://github.com/kevin1018)) +- Remove unnecessary null check before free [\#684](https://github.com/redis/hiredis/pull/684) ([qlyoung](https://github.com/qlyoung)) +- redisReaderGetReply leak memory [\#671](https://github.com/redis/hiredis/pull/671) ([movebean](https://github.com/movebean)) +- fix timeout code in windows [\#670](https://github.com/redis/hiredis/pull/670) ([jman-krafton](https://github.com/jman-krafton)) +- test: fix errstr matching for musl libc [\#665](https://github.com/redis/hiredis/pull/665) ([ghost](https://github.com/ghost)) +- Windows: MinGW fixes and Windows Travis builders [\#663](https://github.com/redis/hiredis/pull/663) ([mbitsnbites](https://github.com/mbitsnbites)) +- The setsockopt and getsockopt API diffs from BSD socket and WSA one [\#662](https://github.com/redis/hiredis/pull/662) ([dragonation](https://github.com/dragonation)) +- Fix Compile Error On Windows \(Visual Studio\) [\#658](https://github.com/redis/hiredis/pull/658) ([jinjiazhang](https://github.com/jinjiazhang)) +- Fix NXDOMAIN test case [\#653](https://github.com/redis/hiredis/pull/653) ([michael-grunder](https://github.com/michael-grunder)) +- Add MinGW support [\#652](https://github.com/redis/hiredis/pull/652) ([mbitsnbites](https://github.com/mbitsnbites)) +- SSL Support [\#645](https://github.com/redis/hiredis/pull/645) ([mnunberg](https://github.com/mnunberg)) +- Fix Invalid argument after redisAsyncConnectUnix [\#644](https://github.com/redis/hiredis/pull/644) ([codehz](https://github.com/codehz)) +- Makefile: use predefined AR [\#632](https://github.com/redis/hiredis/pull/632) ([Mic92](https://github.com/Mic92)) +- FreeBSD build fix [\#628](https://github.com/redis/hiredis/pull/628) ([devnexen](https://github.com/devnexen)) +- Fix errors not propagating properly with libuv.h. [\#624](https://github.com/redis/hiredis/pull/624) ([yossigo](https://github.com/yossigo)) +- Update README.md [\#621](https://github.com/redis/hiredis/pull/621) ([Crunsher](https://github.com/Crunsher)) +- Fix redisBufferRead documentation [\#620](https://github.com/redis/hiredis/pull/620) ([hacst](https://github.com/hacst)) +- Add CPPFLAGS to REAL\_CFLAGS [\#614](https://github.com/redis/hiredis/pull/614) ([thomaslee](https://github.com/thomaslee)) +- Update createArray to take size\_t [\#597](https://github.com/redis/hiredis/pull/597) ([justinbrewer](https://github.com/justinbrewer)) +- fix common realloc mistake and add null check more [\#580](https://github.com/redis/hiredis/pull/580) ([charsyam](https://github.com/charsyam)) +- Proper error reporting for connect failures [\#578](https://github.com/redis/hiredis/pull/578) ([mnunberg](https://github.com/mnunberg)) + +\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* + +## [1.0.0-rc1](https://github.com/redis/hiredis/tree/v1.0.0-rc1) - (2020-07-29) + +_Note: There were no changes to code between v1.0.0-rc1 and v1.0.0 so see v1.0.0 for changelog_ + +### 0.14.1 (2020-03-13) + +* Adds safe allocation wrappers (CVE-2020-7105, #747, #752) (Michael Grunder) + +### 0.14.0 (2018-09-25) +**BREAKING CHANGES**: + * Change `redisReply.len` to `size_t`, as it denotes the the size of a string -User code should compare this to `size_t` values as well. -If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before. + User code should compare this to `size_t` values as well. + If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before. + +* Make string2ll static to fix conflict with Redis (Tom Lee [c3188b]) +* Use -dynamiclib instead of -shared for OSX (Ryan Schmidt [a65537]) +* Use string2ll from Redis w/added tests (Michael Grunder [7bef04, 60f622]) +* Makefile - OSX compilation fixes (Ryan Schmidt [881fcb, 0e9af8]) +* Remove redundant NULL checks (Justin Brewer [54acc8, 58e6b8]) +* Fix bulk and multi-bulk length truncation (Justin Brewer [109197]) +* Fix SIGSEGV in OpenBSD by checking for NULL before calling freeaddrinfo (Justin Brewer [546d94]) +* Several POSIX compatibility fixes (Justin Brewer [bbeab8, 49bbaa, d1c1b6]) +* Makefile - Compatibility fixes (Dimitri Vorobiev [3238cf, 12a9d1]) +* Makefile - Fix make install on FreeBSD (Zach Shipko [a2ef2b]) +* Makefile - don't assume $(INSTALL) is cp (Igor Gnatenko [725a96]) +* Separate side-effect causing function from assert and small cleanup (amallia [b46413, 3c3234]) +* Don't send negative values to `__redisAsyncCommand` (Frederik Deweerdt [706129]) +* Fix leak if setsockopt fails (Frederik Deweerdt [e21c9c]) +* Fix libevent leak (zfz [515228]) +* Clean up GCC warning (Ichito Nagata [2ec774]) +* Keep track of errno in `__redisSetErrorFromErrno()` as snprintf may use it (Jin Qing [25cd88]) +* Solaris compilation fix (Donald Whyte [41b07d]) +* Reorder linker arguments when building examples (Tustfarm-heart [06eedd]) +* Keep track of subscriptions in case of rapid subscribe/unsubscribe (Hyungjin Kim [073dc8, be76c5, d46999]) +* libuv use after free fix (Paul Scott [cbb956]) +* Properly close socket fd on reconnect attempt (WSL [64d1ec]) +* Skip valgrind in OSX tests (Jan-Erik Rediger [9deb78]) +* Various updates for Travis testing OSX (Ted Nyman [fa3774, 16a459, bc0ea5]) +* Update libevent (Chris Xin [386802]) +* Change sds.h for building in C++ projects (Ali Volkan ATLI [f5b32e]) +* Use proper format specifier in redisFormatSdsCommandArgv (Paulino Huerta, Jan-Erik Rediger [360a06, 8655a6]) +* Better handling of NULL reply in example code (Jan-Erik Rediger [1b8ed3]) +* Prevent overflow when formatting an error (Jan-Erik Rediger [0335cb]) +* Compatibility fix for strerror_r (Tom Lee [bb1747]) +* Properly detect integer parse/overflow errors (Justin Brewer [93421f]) +* Adds CI for Windows and cygwin fixes (owent, [6c53d6, 6c3e40]) +* Catch a buffer overflow when formatting the error message +* Import latest upstream sds. This breaks applications that are linked against the old hiredis v0.13 +* Fix warnings, when compiled with -Wshadow +* Make hiredis compile in Cygwin on Windows, now CI-tested +* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now + protocol errors. This is consistent with the RESP specification. On 32-bit + platforms, the upper bound is lowered to `SIZE_MAX`. * Remove backwards compatibility macro's @@ -94,7 +291,7 @@ The parser, standalone since v0.12.0, can now be compiled on Windows * Add IPv6 support -* Remove possiblity of multiple close on same fd +* Remove possibility of multiple close on same fd * Add ability to bind source address on connect @@ -146,4 +343,3 @@ The parser, standalone since v0.12.0, can now be compiled on Windows ### 0.10.0 * See commit log. - diff --git a/redis.submodule/deps/hiredis/CMakeLists.txt b/redis.submodule/deps/hiredis/CMakeLists.txt new file mode 100644 index 0000000..f86c9b7 --- /dev/null +++ b/redis.submodule/deps/hiredis/CMakeLists.txt @@ -0,0 +1,165 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 3.4.0) +INCLUDE(GNUInstallDirs) +PROJECT(hiredis) + +OPTION(ENABLE_SSL "Build hiredis_ssl for SSL support" OFF) +OPTION(DISABLE_TESTS "If tests should be compiled or not" OFF) +OPTION(ENABLE_SSL_TESTS, "Should we test SSL connections" OFF) + +MACRO(getVersionBit name) + SET(VERSION_REGEX "^#define ${name} (.+)$") + FILE(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/hiredis.h" + VERSION_BIT REGEX ${VERSION_REGEX}) + STRING(REGEX REPLACE ${VERSION_REGEX} "\\1" ${name} "${VERSION_BIT}") +ENDMACRO(getVersionBit) + +getVersionBit(HIREDIS_MAJOR) +getVersionBit(HIREDIS_MINOR) +getVersionBit(HIREDIS_PATCH) +getVersionBit(HIREDIS_SONAME) +SET(VERSION "${HIREDIS_MAJOR}.${HIREDIS_MINOR}.${HIREDIS_PATCH}") +MESSAGE("Detected version: ${VERSION}") + +PROJECT(hiredis VERSION "${VERSION}") + +SET(ENABLE_EXAMPLES OFF CACHE BOOL "Enable building hiredis examples") + +SET(hiredis_sources + alloc.c + async.c + dict.c + hiredis.c + net.c + read.c + sds.c + sockcompat.c) + +SET(hiredis_sources ${hiredis_sources}) + +IF(WIN32) + ADD_COMPILE_DEFINITIONS(_CRT_SECURE_NO_WARNINGS WIN32_LEAN_AND_MEAN) +ENDIF() + +ADD_LIBRARY(hiredis SHARED ${hiredis_sources}) + +SET_TARGET_PROPERTIES(hiredis + PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE + VERSION "${HIREDIS_SONAME}") +IF(WIN32 OR MINGW) + TARGET_LINK_LIBRARIES(hiredis PRIVATE ws2_32) +ENDIF() + +TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC $ $) + +CONFIGURE_FILE(hiredis.pc.in hiredis.pc @ONLY) + +INSTALL(TARGETS hiredis + EXPORT hiredis-targets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) + +INSTALL(FILES hiredis.h read.h sds.h async.h alloc.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + +INSTALL(DIRECTORY adapters + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + +export(EXPORT hiredis-targets + FILE "${CMAKE_CURRENT_BINARY_DIR}/hiredis-targets.cmake" + NAMESPACE hiredis::) + +SET(CMAKE_CONF_INSTALL_DIR share/hiredis) +SET(INCLUDE_INSTALL_DIR include) +include(CMakePackageConfigHelpers) +configure_package_config_file(hiredis-config.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/hiredis-config.cmake + INSTALL_DESTINATION ${CMAKE_CONF_INSTALL_DIR} + PATH_VARS INCLUDE_INSTALL_DIR) + +INSTALL(EXPORT hiredis-targets + FILE hiredis-targets.cmake + NAMESPACE hiredis:: + DESTINATION ${CMAKE_CONF_INSTALL_DIR}) + +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis-config.cmake + DESTINATION ${CMAKE_CONF_INSTALL_DIR}) + + +IF(ENABLE_SSL) + IF (NOT OPENSSL_ROOT_DIR) + IF (APPLE) + SET(OPENSSL_ROOT_DIR "/usr/local/opt/openssl") + ENDIF() + ENDIF() + FIND_PACKAGE(OpenSSL REQUIRED) + SET(hiredis_ssl_sources + ssl.c) + ADD_LIBRARY(hiredis_ssl SHARED + ${hiredis_ssl_sources}) + + IF (APPLE) + SET_PROPERTY(TARGET hiredis_ssl PROPERTY LINK_FLAGS "-Wl,-undefined -Wl,dynamic_lookup") + ENDIF() + + SET_TARGET_PROPERTIES(hiredis_ssl + PROPERTIES + WINDOWS_EXPORT_ALL_SYMBOLS TRUE + VERSION "${HIREDIS_SONAME}") + + TARGET_INCLUDE_DIRECTORIES(hiredis_ssl PRIVATE "${OPENSSL_INCLUDE_DIR}") + TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE ${OPENSSL_LIBRARIES}) + IF (WIN32 OR MINGW) + TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE hiredis) + ENDIF() + CONFIGURE_FILE(hiredis_ssl.pc.in hiredis_ssl.pc @ONLY) + + INSTALL(TARGETS hiredis_ssl + EXPORT hiredis_ssl-targets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) + + INSTALL(FILES hiredis_ssl.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + + export(EXPORT hiredis_ssl-targets + FILE "${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl-targets.cmake" + NAMESPACE hiredis::) + + SET(CMAKE_CONF_INSTALL_DIR share/hiredis_ssl) + configure_package_config_file(hiredis_ssl-config.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl-config.cmake + INSTALL_DESTINATION ${CMAKE_CONF_INSTALL_DIR} + PATH_VARS INCLUDE_INSTALL_DIR) + + INSTALL(EXPORT hiredis_ssl-targets + FILE hiredis_ssl-targets.cmake + NAMESPACE hiredis:: + DESTINATION ${CMAKE_CONF_INSTALL_DIR}) + + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl-config.cmake + DESTINATION ${CMAKE_CONF_INSTALL_DIR}) +ENDIF() + +IF(NOT DISABLE_TESTS) + ENABLE_TESTING() + ADD_EXECUTABLE(hiredis-test test.c) + IF(ENABLE_SSL_TESTS) + ADD_DEFINITIONS(-DHIREDIS_TEST_SSL=1) + TARGET_LINK_LIBRARIES(hiredis-test hiredis hiredis_ssl) + ELSE() + TARGET_LINK_LIBRARIES(hiredis-test hiredis) + ENDIF() + ADD_TEST(NAME hiredis-test + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test.sh) +ENDIF() + +# Add examples +IF(ENABLE_EXAMPLES) + ADD_SUBDIRECTORY(examples) +ENDIF(ENABLE_EXAMPLES) diff --git a/redis.submodule/deps/hiredis/Makefile b/redis.submodule/deps/hiredis/Makefile index 9a4de83..7e41c97 100644 --- a/redis.submodule/deps/hiredis/Makefile +++ b/redis.submodule/deps/hiredis/Makefile @@ -3,11 +3,17 @@ # Copyright (C) 2010-2011 Pieter Noordhuis # This file is released under the BSD license, see the COPYING file -OBJ=net.o hiredis.o sds.o async.o read.o -EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib +OBJ=alloc.o net.o hiredis.o sds.o async.o read.o sockcompat.o +SSL_OBJ=ssl.o +EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib hiredis-example-push +ifeq ($(USE_SSL),1) +EXAMPLES+=hiredis-example-ssl hiredis-example-libevent-ssl +endif TESTS=hiredis-test LIBNAME=libhiredis PKGCONFNAME=hiredis.pc +SSL_LIBNAME=libhiredis_ssl +SSL_PKGCONFNAME=hiredis_ssl.pc HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}') HIREDIS_MINOR=$(shell grep HIREDIS_MINOR hiredis.h | awk '{print $$3}') @@ -36,71 +42,130 @@ endef export REDIS_TEST_CONFIG # Fallback to gcc when $CC is not in $PATH. -CC:=$(shell sh -c 'type $(CC) >/dev/null 2>/dev/null && echo $(CC) || echo gcc') -CXX:=$(shell sh -c 'type $(CXX) >/dev/null 2>/dev/null && echo $(CXX) || echo g++') +CC:=$(shell sh -c 'type $${CC%% *} >/dev/null 2>/dev/null && echo $(CC) || echo gcc') +CXX:=$(shell sh -c 'type $${CXX%% *} >/dev/null 2>/dev/null && echo $(CXX) || echo g++') OPTIMIZATION?=-O3 -WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings +WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings -Wno-missing-field-initializers DEBUG_FLAGS?= -g -ggdb -REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CFLAGS) $(WARNINGS) $(DEBUG_FLAGS) $(ARCH) -REAL_LDFLAGS=$(LDFLAGS) $(ARCH) +REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CPPFLAGS) $(CFLAGS) $(WARNINGS) $(DEBUG_FLAGS) +REAL_LDFLAGS=$(LDFLAGS) DYLIBSUFFIX=so STLIBSUFFIX=a DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME) DYLIB_MAJOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR) DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX) -DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) + +DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) STLIBNAME=$(LIBNAME).$(STLIBSUFFIX) -STLIB_MAKE_CMD=ar rcs $(STLIBNAME) +STLIB_MAKE_CMD=$(AR) rcs + +SSL_DYLIB_MINOR_NAME=$(SSL_LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME) +SSL_DYLIB_MAJOR_NAME=$(SSL_LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR) +SSL_DYLIBNAME=$(SSL_LIBNAME).$(DYLIBSUFFIX) +SSL_STLIBNAME=$(SSL_LIBNAME).$(STLIBSUFFIX) +SSL_DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(SSL_DYLIB_MINOR_NAME) # Platform-specific overrides uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') + +USE_SSL?=0 + +# This is required for test.c only +ifeq ($(USE_SSL),1) + CFLAGS+=-DHIREDIS_TEST_SSL +endif + +ifeq ($(uname_S),Linux) + ifdef OPENSSL_PREFIX + CFLAGS+=-I$(OPENSSL_PREFIX)/include + SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto + else + SSL_LDFLAGS=-lssl -lcrypto + endif +else + OPENSSL_PREFIX?=/usr/local/opt/openssl + CFLAGS+=-I$(OPENSSL_PREFIX)/include + SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto +endif + ifeq ($(uname_S),SunOS) + IS_SUN_CC=$(shell sh -c '$(CC) -V 2>&1 |egrep -i -c "sun|studio"') + ifeq ($(IS_SUN_CC),1) + SUN_SHARED_FLAG=-G + else + SUN_SHARED_FLAG=-shared + endif REAL_LDFLAGS+= -ldl -lnsl -lsocket - DYLIB_MAKE_CMD=$(CC) -G -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS) - INSTALL= cp -r + DYLIB_MAKE_CMD=$(CC) $(SUN_SHARED_FLAG) -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS) + SSL_DYLIB_MAKE_CMD=$(CC) $(SUN_SHARED_FLAG) -o $(SSL_DYLIBNAME) -h $(SSL_DYLIB_MINOR_NAME) $(LDFLAGS) $(SSL_LDFLAGS) endif ifeq ($(uname_S),Darwin) DYLIBSUFFIX=dylib DYLIB_MINOR_NAME=$(LIBNAME).$(HIREDIS_SONAME).$(DYLIBSUFFIX) - DYLIB_MAKE_CMD=$(CC) -shared -Wl,-install_name,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) + DYLIB_MAKE_CMD=$(CC) -dynamiclib -Wl,-install_name,$(PREFIX)/$(LIBRARY_PATH)/$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) + SSL_DYLIB_MAKE_CMD=$(CC) -dynamiclib -Wl,-install_name,$(PREFIX)/$(LIBRARY_PATH)/$(SSL_DYLIB_MINOR_NAME) -o $(SSL_DYLIBNAME) $(LDFLAGS) $(SSL_LDFLAGS) + DYLIB_PLUGIN=-Wl,-undefined -Wl,dynamic_lookup endif all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME) +ifeq ($(USE_SSL),1) +all: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME) +endif # Deps (use make dep to generate this) -async.o: async.c fmacros.h async.h hiredis.h read.h sds.h net.h dict.c dict.h -dict.o: dict.c fmacros.h dict.h -hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h -net.o: net.c fmacros.h net.h hiredis.h read.h sds.h -read.o: read.c fmacros.h read.h sds.h -sds.o: sds.c sds.h -test.o: test.c fmacros.h hiredis.h read.h sds.h +alloc.o: alloc.c fmacros.h alloc.h +async.o: async.c fmacros.h alloc.h async.h hiredis.h read.h sds.h net.h dict.c dict.h win32.h async_private.h +dict.o: dict.c fmacros.h alloc.h dict.h +hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h alloc.h net.h async.h win32.h +net.o: net.c fmacros.h net.h hiredis.h read.h sds.h alloc.h sockcompat.h win32.h +read.o: read.c fmacros.h alloc.h read.h sds.h win32.h +sds.o: sds.c sds.h sdsalloc.h alloc.h +sockcompat.o: sockcompat.c sockcompat.h +ssl.o: ssl.c hiredis.h read.h sds.h alloc.h async.h win32.h async_private.h +test.o: test.c fmacros.h hiredis.h read.h sds.h alloc.h net.h sockcompat.h win32.h $(DYLIBNAME): $(OBJ) - $(DYLIB_MAKE_CMD) $(OBJ) + $(DYLIB_MAKE_CMD) -o $(DYLIBNAME) $(OBJ) $(REAL_LDFLAGS) $(STLIBNAME): $(OBJ) - $(STLIB_MAKE_CMD) $(OBJ) + $(STLIB_MAKE_CMD) $(STLIBNAME) $(OBJ) + +$(SSL_DYLIBNAME): $(SSL_OBJ) + $(SSL_DYLIB_MAKE_CMD) $(DYLIB_PLUGIN) -o $(SSL_DYLIBNAME) $(SSL_OBJ) $(REAL_LDFLAGS) $(LDFLAGS) $(SSL_LDFLAGS) + +$(SSL_STLIBNAME): $(SSL_OBJ) + $(STLIB_MAKE_CMD) $(SSL_STLIBNAME) $(SSL_OBJ) dynamic: $(DYLIBNAME) static: $(STLIBNAME) +ifeq ($(USE_SSL),1) +dynamic: $(SSL_DYLIBNAME) +static: $(SSL_STLIBNAME) +endif # Binaries: hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -levent $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(REAL_LDFLAGS) + +hiredis-example-libevent-ssl: examples/example-libevent-ssl.c adapters/libevent.h $(STLIBNAME) $(SSL_STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) hiredis-example-libev: examples/example-libev.c adapters/libev.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -lev $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -lev $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-glib: examples/example-glib.c adapters/glib.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) $(shell pkg-config --cflags --libs glib-2.0) -I. $< $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(shell pkg-config --cflags --libs glib-2.0) $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-ivykis: examples/example-ivykis.c adapters/ivykis.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -livykis $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -livykis $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) $(REAL_LDFLAGS) + +hiredis-example-ssl: examples/example-ssl.c $(STLIBNAME) $(SSL_STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) + ifndef AE_DIR hiredis-example-ae: @@ -117,7 +182,7 @@ hiredis-example-libuv: @false else hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) $(REAL_LDFLAGS) endif ifeq ($(and $(QT_MOC),$(QT_INCLUDE_DIR),$(QT_LIBRARY_DIR)),) @@ -134,38 +199,41 @@ hiredis-example-qt: examples/example-qt.cpp adapters/qt.h $(STLIBNAME) endif hiredis-example: examples/example.c $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS) + +hiredis-example-push: examples/example-push.c $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS) examples: $(EXAMPLES) -hiredis-test: test.o $(STLIBNAME) +TEST_LIBS = $(STLIBNAME) +ifeq ($(USE_SSL),1) + TEST_LIBS += $(SSL_STLIBNAME) + TEST_LDFLAGS = $(SSL_LDFLAGS) -lssl -lcrypto -lpthread +endif + +hiredis-test: test.o $(TEST_LIBS) + $(CC) -o $@ $(REAL_CFLAGS) -I. $^ $(REAL_LDFLAGS) $(TEST_LDFLAGS) hiredis-%: %.o $(STLIBNAME) - $(CC) $(REAL_CFLAGS) -o $@ $(REAL_LDFLAGS) $< $(STLIBNAME) + $(CC) $(REAL_CFLAGS) -o $@ $< $(TEST_LIBS) $(REAL_LDFLAGS) test: hiredis-test ./hiredis-test check: hiredis-test - @echo "$$REDIS_TEST_CONFIG" | $(REDIS_SERVER) - - $(PRE) ./hiredis-test -h 127.0.0.1 -p $(REDIS_PORT) -s /tmp/hiredis-test-redis.sock || \ - ( kill `cat /tmp/hiredis-test-redis.pid` && false ) - kill `cat /tmp/hiredis-test-redis.pid` + TEST_SSL=$(USE_SSL) ./test.sh .c.o: $(CC) -std=c99 -pedantic -c $(REAL_CFLAGS) $< clean: - rm -rf $(DYLIBNAME) $(STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov + rm -rf $(DYLIBNAME) $(STLIBNAME) $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov dep: - $(CC) -MM *.c + $(CC) $(CPPFLAGS) $(CFLAGS) -MM *.c -ifeq ($(uname_S),SunOS) - INSTALL?= cp -r -endif - -INSTALL?= cp -a +INSTALL?= cp -pPR $(PKGCONFNAME): hiredis.h @echo "Generating $@ for pkgconfig..." @@ -180,15 +248,43 @@ $(PKGCONFNAME): hiredis.h @echo Libs: -L\$${libdir} -lhiredis >> $@ @echo Cflags: -I\$${includedir} -D_FILE_OFFSET_BITS=64 >> $@ +$(SSL_PKGCONFNAME): hiredis_ssl.h + @echo "Generating $@ for pkgconfig..." + @echo prefix=$(PREFIX) > $@ + @echo exec_prefix=\$${prefix} >> $@ + @echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@ + @echo includedir=$(PREFIX)/$(INCLUDE_PATH) >> $@ + @echo >> $@ + @echo Name: hiredis_ssl >> $@ + @echo Description: SSL Support for hiredis. >> $@ + @echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@ + @echo Requires: hiredis >> $@ + @echo Libs: -L\$${libdir} -lhiredis_ssl >> $@ + @echo Libs.private: -lssl -lcrypto >> $@ + install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) - mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_LIBRARY_PATH) - $(INSTALL) hiredis.h async.h read.h sds.h adapters $(INSTALL_INCLUDE_PATH) + mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL_LIBRARY_PATH) + $(INSTALL) hiredis.h async.h read.h sds.h alloc.h $(INSTALL_INCLUDE_PATH) + $(INSTALL) adapters/*.h $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL) $(DYLIBNAME) $(INSTALL_LIBRARY_PATH)/$(DYLIB_MINOR_NAME) cd $(INSTALL_LIBRARY_PATH) && ln -sf $(DYLIB_MINOR_NAME) $(DYLIBNAME) $(INSTALL) $(STLIBNAME) $(INSTALL_LIBRARY_PATH) mkdir -p $(INSTALL_PKGCONF_PATH) $(INSTALL) $(PKGCONFNAME) $(INSTALL_PKGCONF_PATH) +ifeq ($(USE_SSL),1) +install: install-ssl + +install-ssl: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME) + mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_LIBRARY_PATH) + $(INSTALL) hiredis_ssl.h $(INSTALL_INCLUDE_PATH) + $(INSTALL) $(SSL_DYLIBNAME) $(INSTALL_LIBRARY_PATH)/$(SSL_DYLIB_MINOR_NAME) + cd $(INSTALL_LIBRARY_PATH) && ln -sf $(SSL_DYLIB_MINOR_NAME) $(SSL_DYLIBNAME) + $(INSTALL) $(SSL_STLIBNAME) $(INSTALL_LIBRARY_PATH) + mkdir -p $(INSTALL_PKGCONF_PATH) + $(INSTALL) $(SSL_PKGCONFNAME) $(INSTALL_PKGCONF_PATH) +endif + 32bit: @echo "" @echo "WARNING: if this fails under Linux you probably need to install libc6-dev-i386" diff --git a/redis.submodule/deps/hiredis/README.md b/redis.submodule/deps/hiredis/README.md index 01223ea..3a22553 100644 --- a/redis.submodule/deps/hiredis/README.md +++ b/redis.submodule/deps/hiredis/README.md @@ -1,6 +1,6 @@ [![Build Status](https://travis-ci.org/redis/hiredis.png)](https://travis-ci.org/redis/hiredis) -**This Readme reflects the latest changed in the master branch. See [v0.13.3](https://github.com/redis/hiredis/tree/v0.13.3) for the Readme and documentation for the latest release.** +**This Readme reflects the latest changed in the master branch. See [v1.0.0](https://github.com/redis/hiredis/tree/v1.0.0) for the Readme and documentation for the latest release ([API/ABI history](https://abi-laboratory.pro/?view=timeline&l=hiredis)).** # HIREDIS @@ -24,12 +24,31 @@ The library comes with multiple APIs. There is the ## Upgrading to `1.0.0` -Version 1.0.0 marks a stable release of hiredis. +Version 1.0.0 marks the first stable release of Hiredis. It includes some minor breaking changes, mostly to make the exposed API more uniform and self-explanatory. It also bundles the updated `sds` library, to sync up with upstream and Redis. -For most applications a recompile against the new hiredis should be enough. For code changes see the [Changelog](CHANGELOG.md). +_Note: As described below, a few member names have been changed but most applications should be able to upgrade with minor code changes and recompiling._ + +## IMPORTANT: Breaking changes from `0.14.1` -> `1.0.0` + +* `redisContext` has two additional members (`free_privdata`, and `privctx`). +* `redisOptions.timeout` has been renamed to `redisOptions.connect_timeout`, and we've added `redisOptions.command_timeout`. +* `redisReplyObjectFunctions.createArray` now takes `size_t` instead of `int` for its length parameter. + +## IMPORTANT: Breaking changes when upgrading from 0.13.x -> 0.14.x + +Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now +protocol errors. This is consistent with the RESP specification. On 32-bit +platforms, the upper bound is lowered to `SIZE_MAX`. + +Change `redisReply.len` to `size_t`, as it denotes the the size of a string + +User code should compare this to `size_t` values as well. If it was used to +compare to other values, casting might be necessary or can be removed, if +casting was applied before. + ## Upgrading from `<0.9.0` Version 0.9.0 is a major overhaul of hiredis in every aspect. However, upgrading existing @@ -110,6 +129,8 @@ The standard replies that `redisCommand` are of the type `redisReply`. The `type` field in the `redisReply` should be used to test what kind of reply was received: +### RESP2 + * **`REDIS_REPLY_STATUS`**: * The command replied with a status reply. The status string can be accessed using `reply->str`. The length of this string can be accessed using `reply->len`. @@ -134,16 +155,51 @@ was received: and can be accessed via `reply->element[..index..]`. Redis may reply with nested arrays but this is fully supported. +### RESP3 + +Hiredis also supports every new `RESP3` data type which are as follows. For more information about the protocol see the `RESP3` [specification.](https://github.com/antirez/RESP3/blob/master/spec.md) + +* **`REDIS_REPLY_DOUBLE`**: + * The command replied with a double-precision floating point number. + The value is stored as a string in the `str` member, and can be converted with `strtod` or similar. + +* **`REDIS_REPLY_BOOL`**: + * A boolean true/false reply. + The value is stored in the `integer` member and will be either `0` or `1`. + +* **`REDIS_REPLY_MAP`**: + * An array with the added invariant that there will always be an even number of elements. + The MAP is functionally equivelant to `REDIS_REPLY_ARRAY` except for the previously mentioned invariant. + +* **`REDIS_REPLY_SET`**: + * An array response where each entry is unique. + Like the MAP type, the data is identical to an array response except there are no duplicate values. + +* **`REDIS_REPLY_PUSH`**: + * An array that can be generated spontaneously by Redis. + This array response will always contain at least two subelements. The first contains the type of `PUSH` message (e.g. `message`, or `invalidate`), and the second being a sub-array with the `PUSH` payload itself. + +* **`REDIS_REPLY_ATTR`**: + * An array structurally identical to a `MAP` but intended as meta-data about a reply. + _As of Redis 6.0.6 this reply type is not used in Redis_ + +* **`REDIS_REPLY_BIGNUM`**: + * A string representing an arbitrarily large signed or unsigned integer value. + The number will be encoded as a string in the `str` member of `redisReply`. + +* **`REDIS_REPLY_VERB`**: + * A verbatim string, intended to be presented to the user without modification. + The string payload is stored in the `str` memeber, and type data is stored in the `vtype` member (e.g. `txt` for raw text or `md` for markdown). + Replies should be freed using the `freeReplyObject()` function. Note that this function will take care of freeing sub-reply objects contained in arrays and nested arrays, so there is no need for the user to free the sub replies (it is actually harmful and will corrupt the memory). -**Important:** the current version of hiredis (0.10.0) frees replies when the +**Important:** the current version of hiredis (1.0.0) frees replies when the asynchronous API is used. This means you should not call `freeReplyObject` when you use this API. The reply is cleaned up by hiredis _after_ the callback -returns. This behavior will probably change in future releases, so make sure to -keep an eye on the changelog when upgrading (see issue #39). +returns. We may introduce a flag to make this configurable in future versions of the library. ### Cleaning up @@ -205,16 +261,16 @@ a single call to `read(2)`): redisReply *reply; redisAppendCommand(context,"SET foo bar"); redisAppendCommand(context,"GET foo"); -redisGetReply(context,&reply); // reply for SET +redisGetReply(context,(void *)&reply); // reply for SET freeReplyObject(reply); -redisGetReply(context,&reply); // reply for GET +redisGetReply(context,(void *)&reply); // reply for GET freeReplyObject(reply); ``` This API can also be used to implement a blocking subscriber: ```c reply = redisCommand(context,"SUBSCRIBE foo"); freeReplyObject(reply); -while(redisGetReply(context,&reply) == REDIS_OK) { +while(redisGetReply(context,(void *)&reply) == REDIS_OK) { // consume message freeReplyObject(reply); } @@ -286,6 +342,7 @@ return `REDIS_ERR`. The function to set the disconnect callback has the followin ```c int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); ``` +`ac->data` may be used to pass user data to this callback, the same can be done for redisConnectCallback. ### Sending commands and their callbacks In an asynchronous context, commands are automatically pipelined due to the nature of an event loop. @@ -403,9 +460,199 @@ This should be done only in order to maximize performances when working with large payloads. The context should be set back to `REDIS_READER_MAX_BUF` again as soon as possible in order to prevent allocation of useless memory. +### Reader max array elements + +By default the hiredis reply parser sets the maximum number of multi-bulk elements +to 2^32 - 1 or 4,294,967,295 entries. If you need to process multi-bulk replies +with more than this many elements you can set the value higher or to zero, meaning +unlimited with: +```c +context->reader->maxelements = 0; +``` + +## SSL/TLS Support + +### Building + +SSL/TLS support is not built by default and requires an explicit flag: + + make USE_SSL=1 + +This requires OpenSSL development package (e.g. including header files to be +available. + +When enabled, SSL/TLS support is built into extra `libhiredis_ssl.a` and +`libhiredis_ssl.so` static/dynamic libraries. This leaves the original libraries +unaffected so no additional dependencies are introduced. + +### Using it + +First, you'll need to make sure you include the SSL header file: + +```c +#include "hiredis.h" +#include "hiredis_ssl.h" +``` + +You will also need to link against `libhiredis_ssl`, **in addition** to +`libhiredis` and add `-lssl -lcrypto` to satisfy its dependencies. + +Hiredis implements SSL/TLS on top of its normal `redisContext` or +`redisAsyncContext`, so you will need to establish a connection first and then +initiate an SSL/TLS handshake. + +#### Hiredis OpenSSL Wrappers + +Before Hiredis can negotiate an SSL/TLS connection, it is necessary to +initialize OpenSSL and create a context. You can do that in two ways: + +1. Work directly with the OpenSSL API to initialize the library's global context + and create `SSL_CTX *` and `SSL *` contexts. With an `SSL *` object you can + call `redisInitiateSSL()`. +2. Work with a set of Hiredis-provided wrappers around OpenSSL, create a + `redisSSLContext` object to hold configuration and use + `redisInitiateSSLWithContext()` to initiate the SSL/TLS handshake. + +```c +/* An Hiredis SSL context. It holds SSL configuration and can be reused across + * many contexts. + */ +redisSSLContext *ssl; + +/* An error variable to indicate what went wrong, if the context fails to + * initialize. + */ +redisSSLContextError ssl_error; + +/* Initialize global OpenSSL state. + * + * You should call this only once when your app initializes, and only if + * you don't explicitly or implicitly initialize OpenSSL it elsewhere. + */ +redisInitOpenSSL(); + +/* Create SSL context */ +ssl = redisCreateSSLContext( + "cacertbundle.crt", /* File name of trusted CA/ca bundle file, optional */ + "/path/to/certs", /* Path of trusted certificates, optional */ + "client_cert.pem", /* File name of client certificate file, optional */ + "client_key.pem", /* File name of client private key, optional */ + "redis.mydomain.com", /* Server name to request (SNI), optional */ + &ssl_error + ) != REDIS_OK) { + printf("SSL error: %s\n", redisSSLContextGetError(ssl_error); + /* Abort... */ + } + +/* Create Redis context and establish connection */ +c = redisConnect("localhost", 6443); +if (c == NULL || c->err) { + /* Handle error and abort... */ +} + +/* Negotiate SSL/TLS */ +if (redisInitiateSSLWithContext(c, ssl) != REDIS_OK) { + /* Handle error, in c->err / c->errstr */ +} +``` + +## RESP3 PUSH replies +Redis 6.0 introduced PUSH replies with the reply-type `>`. These messages are generated spontaneously and can arrive at any time, so must be handled using callbacks. + +### Default behavior +Hiredis installs handlers on `redisContext` and `redisAsyncContext` by default, which will intercept and free any PUSH replies detected. This means existing code will work as-is after upgrading to Redis 6 and switching to `RESP3`. + +### Custom PUSH handler prototypes +The callback prototypes differ between `redisContext` and `redisAsyncContext`. + +#### redisContext +```c +void my_push_handler(void *privdata, void *reply) { + /* Handle the reply */ + + /* Note: We need to free the reply in our custom handler for + blocking contexts. This lets us keep the reply if + we want. */ + freeReplyObject(reply); +} +``` + +#### redisAsyncContext +```c +void my_async_push_handler(redisAsyncContext *ac, void *reply) { + /* Handle the reply */ + + /* Note: Because async hiredis always frees replies, you should + not call freeReplyObject in an async push callback. */ +} +``` + +### Installing a custom handler +There are two ways to set your own PUSH handlers. + +1. Set `push_cb` or `async_push_cb` in the `redisOptions` struct and connect with `redisConnectWithOptions` or `redisAsyncConnectWithOptions`. + ```c + redisOptions = {0}; + REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379); + options->push_cb = my_push_handler; + redisContext *context = redisConnectWithOptions(&options); + ``` +2. Call `redisSetPushCallback` or `redisAsyncSetPushCallback` on a connected context. + ```c + redisContext *context = redisConnect("127.0.0.1", 6379); + redisSetPushCallback(context, my_push_handler); + ``` + + _Note `redisSetPushCallback` and `redisAsyncSetPushCallback` both return any currently configured handler, making it easy to override and then return to the old value._ + +### Specifying no handler +If you have a unique use-case where you don't want hiredis to automatically intercept and free PUSH replies, you will want to configure no handler at all. This can be done in two ways. +1. Set the `REDIS_OPT_NO_PUSH_AUTOFREE` flag in `redisOptions` and leave the callback function pointer `NULL`. + ```c + redisOptions = {0}; + REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379); + options->options |= REDIS_OPT_NO_PUSH_AUTOFREE; + redisContext *context = redisConnectWithOptions(&options); + ``` +3. Call `redisSetPushCallback` with `NULL` once connected. + ```c + redisContext *context = redisConnect("127.0.0.1", 6379); + redisSetPushCallback(context, NULL); + ``` + + _Note: With no handler configured, calls to `redisCommand` may generate more than one reply, so this strategy is only applicable when there's some kind of blocking`redisGetReply()` loop (e.g. `MONITOR` or `SUBSCRIBE` workloads)._ + +## Allocator injection + +Hiredis uses a pass-thru structure of function pointers defined in [alloc.h](https://github.com/redis/hiredis/blob/f5d25850/alloc.h#L41) that contain the currently configured allocation and deallocation functions. By default they just point to libc (`malloc`, `calloc`, `realloc`, etc). + +### Overriding + +One can override the allocators like so: + +```c +hiredisAllocFuncs myfuncs = { + .mallocFn = my_malloc, + .callocFn = my_calloc, + .reallocFn = my_realloc, + .strdupFn = my_strdup, + .freeFn = my_free, +}; + +// Override allocators (function returns current allocators if needed) +hiredisAllocFuncs orig = hiredisSetAllocators(&myfuncs); +``` + +To reset the allocators to their default libc function simply call: + +```c +hiredisResetAllocators(); +``` + ## AUTHORS -Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and -Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. -Hiredis is currently maintained by Matt Stancliff (matt at genges dot com) and -Jan-Erik Rediger (janerik at fnordig dot com) +Salvatore Sanfilippo (antirez at gmail),\ +Pieter Noordhuis (pcnoordhuis at gmail)\ +Michael Grunder (michael dot grunder at gmail) + +_Hiredis is released under the BSD license._ diff --git a/redis.submodule/deps/hiredis/adapters/ae.h b/redis.submodule/deps/hiredis/adapters/ae.h index 5c551c2..660d82e 100644 --- a/redis.submodule/deps/hiredis/adapters/ae.h +++ b/redis.submodule/deps/hiredis/adapters/ae.h @@ -96,7 +96,7 @@ static void redisAeCleanup(void *privdata) { redisAeEvents *e = (redisAeEvents*)privdata; redisAeDelRead(privdata); redisAeDelWrite(privdata); - free(e); + hi_free(e); } static int redisAeAttach(aeEventLoop *loop, redisAsyncContext *ac) { @@ -108,7 +108,10 @@ static int redisAeAttach(aeEventLoop *loop, redisAsyncContext *ac) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisAeEvents*)malloc(sizeof(*e)); + e = (redisAeEvents*)hi_malloc(sizeof(*e)); + if (e == NULL) + return REDIS_ERR; + e->context = ac; e->loop = loop; e->fd = c->fd; diff --git a/redis.submodule/deps/hiredis/adapters/glib.h b/redis.submodule/deps/hiredis/adapters/glib.h index e0a6411..ad59dd1 100644 --- a/redis.submodule/deps/hiredis/adapters/glib.h +++ b/redis.submodule/deps/hiredis/adapters/glib.h @@ -134,6 +134,9 @@ redis_source_new (redisAsyncContext *ac) g_return_val_if_fail(ac != NULL, NULL); source = (RedisSource *)g_source_new(&source_funcs, sizeof *source); + if (source == NULL) + return NULL; + source->ac = ac; source->poll_fd.fd = c->fd; source->poll_fd.events = 0; diff --git a/redis.submodule/deps/hiredis/adapters/ivykis.h b/redis.submodule/deps/hiredis/adapters/ivykis.h index 6a12a86..179f6ab 100644 --- a/redis.submodule/deps/hiredis/adapters/ivykis.h +++ b/redis.submodule/deps/hiredis/adapters/ivykis.h @@ -43,7 +43,7 @@ static void redisIvykisCleanup(void *privdata) { redisIvykisEvents *e = (redisIvykisEvents*)privdata; iv_fd_unregister(&e->fd); - free(e); + hi_free(e); } static int redisIvykisAttach(redisAsyncContext *ac) { @@ -55,7 +55,10 @@ static int redisIvykisAttach(redisAsyncContext *ac) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisIvykisEvents*)malloc(sizeof(*e)); + e = (redisIvykisEvents*)hi_malloc(sizeof(*e)); + if (e == NULL) + return REDIS_ERR; + e->context = ac; /* Register functions to start/stop listening for events */ diff --git a/redis.submodule/deps/hiredis/adapters/libev.h b/redis.submodule/deps/hiredis/adapters/libev.h index 2bf8d52..e1e7bbd 100644 --- a/redis.submodule/deps/hiredis/adapters/libev.h +++ b/redis.submodule/deps/hiredis/adapters/libev.h @@ -41,6 +41,7 @@ typedef struct redisLibevEvents { struct ev_loop *loop; int reading, writing; ev_io rev, wev; + ev_timer timer; } redisLibevEvents; static void redisLibevReadEvent(EV_P_ ev_io *watcher, int revents) { @@ -103,11 +104,39 @@ static void redisLibevDelWrite(void *privdata) { } } +static void redisLibevStopTimer(void *privdata) { + redisLibevEvents *e = (redisLibevEvents*)privdata; + struct ev_loop *loop = e->loop; + ((void)loop); + ev_timer_stop(EV_A_ &e->timer); +} + static void redisLibevCleanup(void *privdata) { redisLibevEvents *e = (redisLibevEvents*)privdata; redisLibevDelRead(privdata); redisLibevDelWrite(privdata); - free(e); + redisLibevStopTimer(privdata); + hi_free(e); +} + +static void redisLibevTimeout(EV_P_ ev_timer *timer, int revents) { + ((void)revents); + redisLibevEvents *e = (redisLibevEvents*)timer->data; + redisAsyncHandleTimeout(e->context); +} + +static void redisLibevSetTimeout(void *privdata, struct timeval tv) { + redisLibevEvents *e = (redisLibevEvents*)privdata; + struct ev_loop *loop = e->loop; + ((void)loop); + + if (!ev_is_active(&e->timer)) { + ev_init(&e->timer, redisLibevTimeout); + e->timer.data = e; + } + + e->timer.repeat = tv.tv_sec + tv.tv_usec / 1000000.00; + ev_timer_again(EV_A_ &e->timer); } static int redisLibevAttach(EV_P_ redisAsyncContext *ac) { @@ -119,14 +148,16 @@ static int redisLibevAttach(EV_P_ redisAsyncContext *ac) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisLibevEvents*)malloc(sizeof(*e)); + e = (redisLibevEvents*)hi_calloc(1, sizeof(*e)); + if (e == NULL) + return REDIS_ERR; + e->context = ac; #if EV_MULTIPLICITY e->loop = loop; #else e->loop = NULL; #endif - e->reading = e->writing = 0; e->rev.data = e; e->wev.data = e; @@ -136,6 +167,7 @@ static int redisLibevAttach(EV_P_ redisAsyncContext *ac) { ac->ev.addWrite = redisLibevAddWrite; ac->ev.delWrite = redisLibevDelWrite; ac->ev.cleanup = redisLibevCleanup; + ac->ev.scheduleTimer = redisLibevSetTimeout; ac->ev.data = e; /* Initialize read/write events */ diff --git a/redis.submodule/deps/hiredis/adapters/libevent.h b/redis.submodule/deps/hiredis/adapters/libevent.h index 273d8b2..9150979 100644 --- a/redis.submodule/deps/hiredis/adapters/libevent.h +++ b/redis.submodule/deps/hiredis/adapters/libevent.h @@ -34,48 +34,113 @@ #include "../hiredis.h" #include "../async.h" +#define REDIS_LIBEVENT_DELETED 0x01 +#define REDIS_LIBEVENT_ENTERED 0x02 + typedef struct redisLibeventEvents { redisAsyncContext *context; - struct event *rev, *wev; + struct event *ev; + struct event_base *base; + struct timeval tv; + short flags; + short state; } redisLibeventEvents; -static void redisLibeventReadEvent(int fd, short event, void *arg) { - ((void)fd); ((void)event); - redisLibeventEvents *e = (redisLibeventEvents*)arg; - redisAsyncHandleRead(e->context); +static void redisLibeventDestroy(redisLibeventEvents *e) { + hi_free(e); } -static void redisLibeventWriteEvent(int fd, short event, void *arg) { - ((void)fd); ((void)event); +static void redisLibeventHandler(int fd, short event, void *arg) { + ((void)fd); redisLibeventEvents *e = (redisLibeventEvents*)arg; - redisAsyncHandleWrite(e->context); + e->state |= REDIS_LIBEVENT_ENTERED; + + #define CHECK_DELETED() if (e->state & REDIS_LIBEVENT_DELETED) {\ + redisLibeventDestroy(e);\ + return; \ + } + + if ((event & EV_TIMEOUT) && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleTimeout(e->context); + CHECK_DELETED(); + } + + if ((event & EV_READ) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleRead(e->context); + CHECK_DELETED(); + } + + if ((event & EV_WRITE) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleWrite(e->context); + CHECK_DELETED(); + } + + e->state &= ~REDIS_LIBEVENT_ENTERED; + #undef CHECK_DELETED +} + +static void redisLibeventUpdate(void *privdata, short flag, int isRemove) { + redisLibeventEvents *e = (redisLibeventEvents *)privdata; + const struct timeval *tv = e->tv.tv_sec || e->tv.tv_usec ? &e->tv : NULL; + + if (isRemove) { + if ((e->flags & flag) == 0) { + return; + } else { + e->flags &= ~flag; + } + } else { + if (e->flags & flag) { + return; + } else { + e->flags |= flag; + } + } + + event_del(e->ev); + event_assign(e->ev, e->base, e->context->c.fd, e->flags | EV_PERSIST, + redisLibeventHandler, privdata); + event_add(e->ev, tv); } static void redisLibeventAddRead(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(e->rev,NULL); + redisLibeventUpdate(privdata, EV_READ, 0); } static void redisLibeventDelRead(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(e->rev); + redisLibeventUpdate(privdata, EV_READ, 1); } static void redisLibeventAddWrite(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(e->wev,NULL); + redisLibeventUpdate(privdata, EV_WRITE, 0); } static void redisLibeventDelWrite(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(e->wev); + redisLibeventUpdate(privdata, EV_WRITE, 1); } static void redisLibeventCleanup(void *privdata) { redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(e->rev); - event_del(e->wev); - free(e); + if (!e) { + return; + } + event_del(e->ev); + event_free(e->ev); + e->ev = NULL; + + if (e->state & REDIS_LIBEVENT_ENTERED) { + e->state |= REDIS_LIBEVENT_DELETED; + } else { + redisLibeventDestroy(e); + } +} + +static void redisLibeventSetTimeout(void *privdata, struct timeval tv) { + redisLibeventEvents *e = (redisLibeventEvents *)privdata; + short flags = e->flags; + e->flags = 0; + e->tv = tv; + redisLibeventUpdate(e, flags, 0); } static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { @@ -87,7 +152,10 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisLibeventEvents*)malloc(sizeof(*e)); + e = (redisLibeventEvents*)hi_calloc(1, sizeof(*e)); + if (e == NULL) + return REDIS_ERR; + e->context = ac; /* Register functions to start/stop listening for events */ @@ -96,13 +164,12 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { ac->ev.addWrite = redisLibeventAddWrite; ac->ev.delWrite = redisLibeventDelWrite; ac->ev.cleanup = redisLibeventCleanup; + ac->ev.scheduleTimer = redisLibeventSetTimeout; ac->ev.data = e; /* Initialize and install read/write events */ - e->rev = event_new(base, c->fd, EV_READ, redisLibeventReadEvent, e); - e->wev = event_new(base, c->fd, EV_WRITE, redisLibeventWriteEvent, e); - event_add(e->rev, NULL); - event_add(e->wev, NULL); + e->ev = event_new(base, c->fd, EV_READ | EV_WRITE, redisLibeventHandler, e); + e->base = base; return REDIS_OK; } #endif diff --git a/redis.submodule/deps/hiredis/adapters/libuv.h b/redis.submodule/deps/hiredis/adapters/libuv.h index ff08c25..c120b1b 100644 --- a/redis.submodule/deps/hiredis/adapters/libuv.h +++ b/redis.submodule/deps/hiredis/adapters/libuv.h @@ -15,15 +15,12 @@ typedef struct redisLibuvEvents { static void redisLibuvPoll(uv_poll_t* handle, int status, int events) { redisLibuvEvents* p = (redisLibuvEvents*)handle->data; + int ev = (status ? p->events : events); - if (status != 0) { - return; - } - - if (p->context != NULL && (events & UV_READABLE)) { + if (p->context != NULL && (ev & UV_READABLE)) { redisAsyncHandleRead(p->context); } - if (p->context != NULL && (events & UV_WRITABLE)) { + if (p->context != NULL && (ev & UV_WRITABLE)) { redisAsyncHandleWrite(p->context); } } @@ -76,7 +73,7 @@ static void redisLibuvDelWrite(void *privdata) { static void on_close(uv_handle_t* handle) { redisLibuvEvents* p = (redisLibuvEvents*)handle->data; - free(p); + hi_free(p); } @@ -101,15 +98,13 @@ static int redisLibuvAttach(redisAsyncContext* ac, uv_loop_t* loop) { ac->ev.delWrite = redisLibuvDelWrite; ac->ev.cleanup = redisLibuvCleanup; - redisLibuvEvents* p = (redisLibuvEvents*)malloc(sizeof(*p)); - - if (!p) { - return REDIS_ERR; - } + redisLibuvEvents* p = (redisLibuvEvents*)hi_malloc(sizeof(*p)); + if (p == NULL) + return REDIS_ERR; memset(p, 0, sizeof(*p)); - if (uv_poll_init(loop, &p->handle, c->fd) != 0) { + if (uv_poll_init_socket(loop, &p->handle, c->fd) != 0) { return REDIS_ERR; } diff --git a/redis.submodule/deps/hiredis/adapters/macosx.h b/redis.submodule/deps/hiredis/adapters/macosx.h index 72121f6..3c87f1b 100644 --- a/redis.submodule/deps/hiredis/adapters/macosx.h +++ b/redis.submodule/deps/hiredis/adapters/macosx.h @@ -27,7 +27,7 @@ static int freeRedisRunLoop(RedisRunLoop* redisRunLoop) { CFSocketInvalidate(redisRunLoop->socketRef); CFRelease(redisRunLoop->socketRef); } - free(redisRunLoop); + hi_free(redisRunLoop); } return REDIS_ERR; } @@ -80,8 +80,9 @@ static int redisMacOSAttach(redisAsyncContext *redisAsyncCtx, CFRunLoopRef runLo /* Nothing should be attached when something is already attached */ if( redisAsyncCtx->ev.data != NULL ) return REDIS_ERR; - RedisRunLoop* redisRunLoop = (RedisRunLoop*) calloc(1, sizeof(RedisRunLoop)); - if( !redisRunLoop ) return REDIS_ERR; + RedisRunLoop* redisRunLoop = (RedisRunLoop*) hi_calloc(1, sizeof(RedisRunLoop)); + if (redisRunLoop == NULL) + return REDIS_ERR; /* Setup redis stuff */ redisRunLoop->context = redisAsyncCtx; diff --git a/redis.submodule/deps/hiredis/alloc.c b/redis.submodule/deps/hiredis/alloc.c new file mode 100644 index 0000000..7fb6b35 --- /dev/null +++ b/redis.submodule/deps/hiredis/alloc.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2020, Michael Grunder + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "fmacros.h" +#include "alloc.h" +#include +#include + +hiredisAllocFuncs hiredisAllocFns = { + .mallocFn = malloc, + .callocFn = calloc, + .reallocFn = realloc, + .strdupFn = strdup, + .freeFn = free, +}; + +/* Override hiredis' allocators with ones supplied by the user */ +hiredisAllocFuncs hiredisSetAllocators(hiredisAllocFuncs *override) { + hiredisAllocFuncs orig = hiredisAllocFns; + + hiredisAllocFns = *override; + + return orig; +} + +/* Reset allocators to use libc defaults */ +void hiredisResetAllocators(void) { + hiredisAllocFns = (hiredisAllocFuncs) { + .mallocFn = malloc, + .callocFn = calloc, + .reallocFn = realloc, + .strdupFn = strdup, + .freeFn = free, + }; +} + +#ifdef _WIN32 + +void *hi_malloc(size_t size) { + return hiredisAllocFns.mallocFn(size); +} + +void *hi_calloc(size_t nmemb, size_t size) { + return hiredisAllocFns.callocFn(nmemb, size); +} + +void *hi_realloc(void *ptr, size_t size) { + return hiredisAllocFns.reallocFn(ptr, size); +} + +char *hi_strdup(const char *str) { + return hiredisAllocFns.strdupFn(str); +} + +void hi_free(void *ptr) { + hiredisAllocFns.freeFn(ptr); +} + +#endif diff --git a/redis.submodule/deps/hiredis/alloc.h b/redis.submodule/deps/hiredis/alloc.h new file mode 100644 index 0000000..34a05f4 --- /dev/null +++ b/redis.submodule/deps/hiredis/alloc.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2020, Michael Grunder + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef HIREDIS_ALLOC_H +#define HIREDIS_ALLOC_H + +#include /* for size_t */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Structure pointing to our actually configured allocators */ +typedef struct hiredisAllocFuncs { + void *(*mallocFn)(size_t); + void *(*callocFn)(size_t,size_t); + void *(*reallocFn)(void*,size_t); + char *(*strdupFn)(const char*); + void (*freeFn)(void*); +} hiredisAllocFuncs; + +hiredisAllocFuncs hiredisSetAllocators(hiredisAllocFuncs *ha); +void hiredisResetAllocators(void); + +#ifndef _WIN32 + +/* Hiredis' configured allocator function pointer struct */ +extern hiredisAllocFuncs hiredisAllocFns; + +static inline void *hi_malloc(size_t size) { + return hiredisAllocFns.mallocFn(size); +} + +static inline void *hi_calloc(size_t nmemb, size_t size) { + return hiredisAllocFns.callocFn(nmemb, size); +} + +static inline void *hi_realloc(void *ptr, size_t size) { + return hiredisAllocFns.reallocFn(ptr, size); +} + +static inline char *hi_strdup(const char *str) { + return hiredisAllocFns.strdupFn(str); +} + +static inline void hi_free(void *ptr) { + hiredisAllocFns.freeFn(ptr); +} + +#else + +void *hi_malloc(size_t size); +void *hi_calloc(size_t nmemb, size_t size); +void *hi_realloc(void *ptr, size_t size); +char *hi_strdup(const char *str); +void hi_free(void *ptr); + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* HIREDIS_ALLOC_H */ diff --git a/redis.submodule/deps/hiredis/appveyor.yml b/redis.submodule/deps/hiredis/appveyor.yml index 06bbef1..5b43fdb 100644 --- a/redis.submodule/deps/hiredis/appveyor.yml +++ b/redis.submodule/deps/hiredis/appveyor.yml @@ -1,24 +1,14 @@ # Appveyor configuration file for CI build of hiredis on Windows (under Cygwin) environment: matrix: - - CYG_ROOT: C:\cygwin64 - CYG_SETUP: setup-x86_64.exe - CYG_MIRROR: http://cygwin.mirror.constant.com - CYG_CACHE: C:\cygwin64\var\cache\setup - CYG_BASH: C:\cygwin64\bin\bash + - CYG_BASH: C:\cygwin64\bin\bash CC: gcc - - CYG_ROOT: C:\cygwin - CYG_SETUP: setup-x86.exe - CYG_MIRROR: http://cygwin.mirror.constant.com - CYG_CACHE: C:\cygwin\var\cache\setup - CYG_BASH: C:\cygwin\bin\bash + - CYG_BASH: C:\cygwin\bin\bash CC: gcc - TARGET: 32bit - TARGET_VARS: 32bit-vars + CFLAGS: -m32 + CXXFLAGS: -m32 + LDFLAGS: -m32 -# Cache Cygwin files to speed up build -cache: - - '%CYG_CACHE%' clone_depth: 1 # Attempt to ensure we don't try to convert line endings to Win32 CRLF as this will cause build to fail @@ -27,10 +17,8 @@ init: # Install needed build dependencies install: - - ps: 'Start-FileDownload "http://cygwin.com/$env:CYG_SETUP" -FileName "$env:CYG_SETUP"' - - '%CYG_SETUP% --quiet-mode --no-shortcuts --only-site --root "%CYG_ROOT%" --site "%CYG_MIRROR%" --local-package-dir "%CYG_CACHE%" --packages automake,bison,gcc-core,libtool,make,gettext-devel,gettext,intltool,pkg-config,clang,llvm > NUL 2>&1' - '%CYG_BASH% -lc "cygcheck -dc cygwin"' build_script: - 'echo building...' - - '%CYG_BASH% -lc "cd $APPVEYOR_BUILD_FOLDER; exec 0 #include +#ifndef _MSC_VER #include +#endif #include #include #include @@ -40,35 +43,28 @@ #include "net.h" #include "dict.c" #include "sds.h" +#include "win32.h" -#define _EL_ADD_READ(ctx) do { \ - if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \ - } while(0) -#define _EL_DEL_READ(ctx) do { \ - if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \ - } while(0) -#define _EL_ADD_WRITE(ctx) do { \ - if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \ - } while(0) -#define _EL_DEL_WRITE(ctx) do { \ - if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \ - } while(0) -#define _EL_CLEANUP(ctx) do { \ - if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ - } while(0); - -/* Forward declaration of function in hiredis.c */ +#include "async_private.h" + +/* Forward declarations of hiredis.c functions */ int __redisAppendCommand(redisContext *c, const char *cmd, size_t len); +void __redisSetError(redisContext *c, int type, const char *str); /* Functions managing dictionary of callbacks for pub/sub. */ static unsigned int callbackHash(const void *key) { return dictGenHashFunction((const unsigned char *)key, - sdslen((const sds)key)); + hi_sdslen((const hisds)key)); } static void *callbackValDup(void *privdata, const void *src) { ((void) privdata); - redisCallback *dup = malloc(sizeof(*dup)); + redisCallback *dup; + + dup = hi_malloc(sizeof(*dup)); + if (dup == NULL) + return NULL; + memcpy(dup,src,sizeof(*dup)); return dup; } @@ -77,20 +73,20 @@ static int callbackKeyCompare(void *privdata, const void *key1, const void *key2 int l1, l2; ((void) privdata); - l1 = sdslen((const sds)key1); - l2 = sdslen((const sds)key2); + l1 = hi_sdslen((const hisds)key1); + l2 = hi_sdslen((const hisds)key2); if (l1 != l2) return 0; return memcmp(key1,key2,l1) == 0; } static void callbackKeyDestructor(void *privdata, void *key) { ((void) privdata); - sdsfree((sds)key); + hi_sdsfree((hisds)key); } static void callbackValDestructor(void *privdata, void *val) { ((void) privdata); - free(val); + hi_free(val); } static dictType callbackDict = { @@ -104,10 +100,19 @@ static dictType callbackDict = { static redisAsyncContext *redisAsyncInitialize(redisContext *c) { redisAsyncContext *ac; + dict *channels = NULL, *patterns = NULL; + + channels = dictCreate(&callbackDict,NULL); + if (channels == NULL) + goto oom; - ac = realloc(c,sizeof(redisAsyncContext)); + patterns = dictCreate(&callbackDict,NULL); + if (patterns == NULL) + goto oom; + + ac = hi_realloc(c,sizeof(redisAsyncContext)); if (ac == NULL) - return NULL; + goto oom; c = &(ac->c); @@ -119,6 +124,7 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) { ac->err = 0; ac->errstr = NULL; ac->data = NULL; + ac->dataCleanup = NULL; ac->ev.data = NULL; ac->ev.addRead = NULL; @@ -126,6 +132,7 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) { ac->ev.addWrite = NULL; ac->ev.delWrite = NULL; ac->ev.cleanup = NULL; + ac->ev.scheduleTimer = NULL; ac->onConnect = NULL; ac->onDisconnect = NULL; @@ -134,9 +141,14 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) { ac->replies.tail = NULL; ac->sub.invalid.head = NULL; ac->sub.invalid.tail = NULL; - ac->sub.channels = dictCreate(&callbackDict,NULL); - ac->sub.patterns = dictCreate(&callbackDict,NULL); + ac->sub.channels = channels; + ac->sub.patterns = patterns; + return ac; +oom: + if (channels) dictRelease(channels); + if (patterns) dictRelease(patterns); + return NULL; } /* We want the error field to be accessible directly instead of requiring @@ -150,13 +162,21 @@ static void __redisAsyncCopyError(redisAsyncContext *ac) { ac->errstr = c->errstr; } -redisAsyncContext *redisAsyncConnect(const char *ip, int port) { +redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options) { + redisOptions myOptions = *options; redisContext *c; redisAsyncContext *ac; - c = redisConnectNonBlock(ip,port); - if (c == NULL) + /* Clear any erroneously set sync callback and flag that we don't want to + * use freeReplyObject by default. */ + myOptions.push_cb = NULL; + myOptions.options |= REDIS_OPT_NO_PUSH_AUTOFREE; + + myOptions.options |= REDIS_OPT_NONBLOCK; + c = redisConnectWithOptions(&myOptions); + if (c == NULL) { return NULL; + } ac = redisAsyncInitialize(c); if (ac == NULL) { @@ -164,42 +184,40 @@ redisAsyncContext *redisAsyncConnect(const char *ip, int port) { return NULL; } + /* Set any configured async push handler */ + redisAsyncSetPushCallback(ac, myOptions.async_push_cb); + __redisAsyncCopyError(ac); return ac; } +redisAsyncContext *redisAsyncConnect(const char *ip, int port) { + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + return redisAsyncConnectWithOptions(&options); +} + redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr) { - redisContext *c = redisConnectBindNonBlock(ip,port,source_addr); - redisAsyncContext *ac = redisAsyncInitialize(c); - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + return redisAsyncConnectWithOptions(&options); } redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, const char *source_addr) { - redisContext *c = redisConnectBindNonBlockWithReuse(ip,port,source_addr); - redisAsyncContext *ac = redisAsyncInitialize(c); - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.options |= REDIS_OPT_REUSEADDR; + options.endpoint.tcp.source_addr = source_addr; + return redisAsyncConnectWithOptions(&options); } redisAsyncContext *redisAsyncConnectUnix(const char *path) { - redisContext *c; - redisAsyncContext *ac; - - c = redisConnectUnixNonBlock(path); - if (c == NULL) - return NULL; - - ac = redisAsyncInitialize(c); - if (ac == NULL) { - redisFree(c); - return NULL; - } - - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + return redisAsyncConnectWithOptions(&options); } int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn) { @@ -228,7 +246,7 @@ static int __redisPushCallback(redisCallbackList *list, redisCallback *source) { redisCallback *cb; /* Copy callback from stack to heap */ - cb = malloc(sizeof(*cb)); + cb = hi_malloc(sizeof(*cb)); if (cb == NULL) return REDIS_ERR_OOM; @@ -256,7 +274,7 @@ static int __redisShiftCallback(redisCallbackList *list, redisCallback *target) /* Copy callback from heap to stack */ if (target != NULL) memcpy(target,cb,sizeof(*cb)); - free(cb); + hi_free(cb); return REDIS_OK; } return REDIS_ERR; @@ -271,6 +289,14 @@ static void __redisRunCallback(redisAsyncContext *ac, redisCallback *cb, redisRe } } +static void __redisRunPushCallback(redisAsyncContext *ac, redisReply *reply) { + if (ac->push_cb != NULL) { + ac->c.flags |= REDIS_IN_CALLBACK; + ac->push_cb(ac, reply); + ac->c.flags &= ~REDIS_IN_CALLBACK; + } +} + /* Helper function to free the context. */ static void __redisAsyncFree(redisAsyncContext *ac) { redisContext *c = &(ac->c); @@ -286,18 +312,28 @@ static void __redisAsyncFree(redisAsyncContext *ac) { while (__redisShiftCallback(&ac->sub.invalid,&cb) == REDIS_OK) __redisRunCallback(ac,&cb,NULL); - /* Run subscription callbacks callbacks with NULL reply */ - it = dictGetIterator(ac->sub.channels); - while ((de = dictNext(it)) != NULL) - __redisRunCallback(ac,dictGetEntryVal(de),NULL); - dictReleaseIterator(it); - dictRelease(ac->sub.channels); + /* Run subscription callbacks with NULL reply */ + if (ac->sub.channels) { + it = dictGetIterator(ac->sub.channels); + if (it != NULL) { + while ((de = dictNext(it)) != NULL) + __redisRunCallback(ac,dictGetEntryVal(de),NULL); + dictReleaseIterator(it); + } + + dictRelease(ac->sub.channels); + } - it = dictGetIterator(ac->sub.patterns); - while ((de = dictNext(it)) != NULL) - __redisRunCallback(ac,dictGetEntryVal(de),NULL); - dictReleaseIterator(it); - dictRelease(ac->sub.patterns); + if (ac->sub.patterns) { + it = dictGetIterator(ac->sub.patterns); + if (it != NULL) { + while ((de = dictNext(it)) != NULL) + __redisRunCallback(ac,dictGetEntryVal(de),NULL); + dictReleaseIterator(it); + } + + dictRelease(ac->sub.patterns); + } /* Signal event lib to clean up */ _EL_CLEANUP(ac); @@ -312,6 +348,10 @@ static void __redisAsyncFree(redisAsyncContext *ac) { } } + if (ac->dataCleanup) { + ac->dataCleanup(ac->data); + } + /* Cleanup self */ redisFree(c); } @@ -328,7 +368,7 @@ void redisAsyncFree(redisAsyncContext *ac) { } /* Helper function to make the disconnect happen and clean up. */ -static void __redisAsyncDisconnect(redisAsyncContext *ac) { +void __redisAsyncDisconnect(redisAsyncContext *ac) { redisContext *c = &(ac->c); /* Make sure error is accessible if there is any */ @@ -336,16 +376,23 @@ static void __redisAsyncDisconnect(redisAsyncContext *ac) { if (ac->err == 0) { /* For clean disconnects, there should be no pending callbacks. */ - assert(__redisShiftCallback(&ac->replies,NULL) == REDIS_ERR); + int ret = __redisShiftCallback(&ac->replies,NULL); + assert(ret == REDIS_ERR); } else { /* Disconnection is caused by an error, make sure that pending * callbacks cannot call new commands. */ c->flags |= REDIS_DISCONNECTING; } + /* cleanup event library on disconnect. + * this is safe to call multiple times */ + _EL_CLEANUP(ac); + /* For non-clean disconnects, __redisAsyncFree() will execute pending * callbacks with a NULL-reply. */ - __redisAsyncFree(ac); + if (!(c->flags & REDIS_NO_AUTO_FREE)) { + __redisAsyncFree(ac); + } } /* Tries to do a clean disconnect from Redis, meaning it stops new commands @@ -357,6 +404,9 @@ static void __redisAsyncDisconnect(redisAsyncContext *ac) { void redisAsyncDisconnect(redisAsyncContext *ac) { redisContext *c = &(ac->c); c->flags |= REDIS_DISCONNECTING; + + /** unset the auto-free flag here, because disconnect undoes this */ + c->flags &= ~REDIS_NO_AUTO_FREE; if (!(c->flags & REDIS_IN_CALLBACK) && ac->replies.head == NULL) __redisAsyncDisconnect(ac); } @@ -364,14 +414,15 @@ void redisAsyncDisconnect(redisAsyncContext *ac) { static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, redisCallback *dstcb) { redisContext *c = &(ac->c); dict *callbacks; + redisCallback *cb; dictEntry *de; int pvariant; char *stype; - sds sname; + hisds sname; /* Custom reply functions are not supported for pub/sub. This will fail * very hard when they are used... */ - if (reply->type == REDIS_REPLY_ARRAY) { + if (reply->type == REDIS_REPLY_ARRAY || reply->type == REDIS_REPLY_PUSH) { assert(reply->elements >= 2); assert(reply->element[0]->type == REDIS_REPLY_STRING); stype = reply->element[0]->str; @@ -384,33 +435,75 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, /* Locate the right callback */ assert(reply->element[1]->type == REDIS_REPLY_STRING); - sname = sdsnewlen(reply->element[1]->str,reply->element[1]->len); + sname = hi_sdsnewlen(reply->element[1]->str,reply->element[1]->len); + if (sname == NULL) + goto oom; + de = dictFind(callbacks,sname); if (de != NULL) { - memcpy(dstcb,dictGetEntryVal(de),sizeof(*dstcb)); + cb = dictGetEntryVal(de); + + /* If this is an subscribe reply decrease pending counter. */ + if (strcasecmp(stype+pvariant,"subscribe") == 0) { + cb->pending_subs -= 1; + } + + memcpy(dstcb,cb,sizeof(*dstcb)); /* If this is an unsubscribe message, remove it. */ if (strcasecmp(stype+pvariant,"unsubscribe") == 0) { - dictDelete(callbacks,sname); + if (cb->pending_subs == 0) + dictDelete(callbacks,sname); /* If this was the last unsubscribe message, revert to * non-subscribe mode. */ assert(reply->element[2]->type == REDIS_REPLY_INTEGER); - if (reply->element[2]->integer == 0) + + /* Unset subscribed flag only when no pipelined pending subscribe. */ + if (reply->element[2]->integer == 0 + && dictSize(ac->sub.channels) == 0 + && dictSize(ac->sub.patterns) == 0) c->flags &= ~REDIS_SUBSCRIBED; } } - sdsfree(sname); + hi_sdsfree(sname); } else { /* Shift callback for invalid commands. */ __redisShiftCallback(&ac->sub.invalid,dstcb); } return REDIS_OK; +oom: + __redisSetError(&(ac->c), REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; +} + +#define redisIsSpontaneousPushReply(r) \ + (redisIsPushReply(r) && !redisIsSubscribeReply(r)) + +static int redisIsSubscribeReply(redisReply *reply) { + char *str; + size_t len, off; + + /* We will always have at least one string with the subscribe/message type */ + if (reply->elements < 1 || reply->element[0]->type != REDIS_REPLY_STRING || + reply->element[0]->len < sizeof("message") - 1) + { + return 0; + } + + /* Get the string/len moving past 'p' if needed */ + off = tolower(reply->element[0]->str[0]) == 'p'; + str = reply->element[0]->str + off; + len = reply->element[0]->len - off; + + return !strncasecmp(str, "subscribe", len) || + !strncasecmp(str, "message", len); + } void redisProcessCallbacks(redisAsyncContext *ac) { redisContext *c = &(ac->c); - redisCallback cb = {NULL, NULL, NULL}; + redisCallback cb = {NULL, NULL, 0, NULL}; void *reply = NULL; int status; @@ -418,7 +511,7 @@ void redisProcessCallbacks(redisAsyncContext *ac) { if (reply == NULL) { /* When the connection is being disconnected and there are * no more replies, this is the cue to really disconnect. */ - if (c->flags & REDIS_DISCONNECTING && sdslen(c->obuf) == 0 + if (c->flags & REDIS_DISCONNECTING && hi_sdslen(c->obuf) == 0 && ac->replies.head == NULL) { __redisAsyncDisconnect(ac); return; @@ -434,8 +527,18 @@ void redisProcessCallbacks(redisAsyncContext *ac) { break; } - /* Even if the context is subscribed, pending regular callbacks will - * get a reply before pub/sub messages arrive. */ + /* Send any non-subscribe related PUSH messages to our PUSH handler + * while allowing subscribe related PUSH messages to pass through. + * This allows existing code to be backward compatible and work in + * either RESP2 or RESP3 mode. */ + if (redisIsSpontaneousPushReply(reply)) { + __redisRunPushCallback(ac, reply); + c->reader->fn->freeObject(reply); + continue; + } + + /* Even if the context is subscribed, pending regular + * callbacks will get a reply before pub/sub messages arrive. */ if (__redisShiftCallback(&ac->replies,&cb) != REDIS_OK) { /* * A spontaneous reply in a not-subscribed context can be the error @@ -488,26 +591,49 @@ void redisProcessCallbacks(redisAsyncContext *ac) { __redisAsyncDisconnect(ac); } +static void __redisAsyncHandleConnectFailure(redisAsyncContext *ac) { + if (ac->onConnect) ac->onConnect(ac, REDIS_ERR); + __redisAsyncDisconnect(ac); +} + /* Internal helper function to detect socket status the first time a read or * write event fires. When connecting was not successful, the connect callback * is called with a REDIS_ERR status and the context is free'd. */ static int __redisAsyncHandleConnect(redisAsyncContext *ac) { + int completed = 0; redisContext *c = &(ac->c); - if (redisCheckSocketError(c) == REDIS_ERR) { - /* Try again later when connect(2) is still in progress. */ - if (errno == EINPROGRESS) - return REDIS_OK; - - if (ac->onConnect) ac->onConnect(ac,REDIS_ERR); - __redisAsyncDisconnect(ac); + if (redisCheckConnectDone(c, &completed) == REDIS_ERR) { + /* Error! */ + redisCheckSocketError(c); + __redisAsyncHandleConnectFailure(ac); return REDIS_ERR; + } else if (completed == 1) { + /* connected! */ + if (c->connection_type == REDIS_CONN_TCP && + redisSetTcpNoDelay(c) == REDIS_ERR) { + __redisAsyncHandleConnectFailure(ac); + return REDIS_ERR; + } + + if (ac->onConnect) ac->onConnect(ac, REDIS_OK); + c->flags |= REDIS_CONNECTED; + return REDIS_OK; + } else { + return REDIS_OK; } +} - /* Mark context as connected. */ - c->flags |= REDIS_CONNECTED; - if (ac->onConnect) ac->onConnect(ac,REDIS_OK); - return REDIS_OK; +void redisAsyncRead(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + + if (redisBufferRead(c) == REDIS_ERR) { + __redisAsyncDisconnect(ac); + } else { + /* Always re-schedule reads */ + _EL_ADD_READ(ac); + redisProcessCallbacks(ac); + } } /* This function should be called when the socket is readable. @@ -525,18 +651,29 @@ void redisAsyncHandleRead(redisAsyncContext *ac) { return; } - if (redisBufferRead(c) == REDIS_ERR) { + c->funcs->async_read(ac); +} + +void redisAsyncWrite(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + int done = 0; + + if (redisBufferWrite(c,&done) == REDIS_ERR) { __redisAsyncDisconnect(ac); } else { - /* Always re-schedule reads */ + /* Continue writing when not done, stop writing otherwise */ + if (!done) + _EL_ADD_WRITE(ac); + else + _EL_DEL_WRITE(ac); + + /* Always schedule reads after writes */ _EL_ADD_READ(ac); - redisProcessCallbacks(ac); } } void redisAsyncHandleWrite(redisAsyncContext *ac) { redisContext *c = &(ac->c); - int done = 0; if (!(c->flags & REDIS_CONNECTED)) { /* Abort connect was not successful. */ @@ -547,18 +684,35 @@ void redisAsyncHandleWrite(redisAsyncContext *ac) { return; } - if (redisBufferWrite(c,&done) == REDIS_ERR) { - __redisAsyncDisconnect(ac); - } else { - /* Continue writing when not done, stop writing otherwise */ - if (!done) - _EL_ADD_WRITE(ac); - else - _EL_DEL_WRITE(ac); + c->funcs->async_write(ac); +} - /* Always schedule reads after writes */ - _EL_ADD_READ(ac); +void redisAsyncHandleTimeout(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + redisCallback cb; + + if ((c->flags & REDIS_CONNECTED) && ac->replies.head == NULL) { + /* Nothing to do - just an idle timeout */ + return; } + + if (!c->err) { + __redisSetError(c, REDIS_ERR_TIMEOUT, "Timeout"); + } + + if (!(c->flags & REDIS_CONNECTED) && ac->onConnect) { + ac->onConnect(ac, REDIS_ERR); + } + + while (__redisShiftCallback(&ac->replies, &cb) == REDIS_OK) { + __redisRunCallback(ac, &cb, NULL); + } + + /** + * TODO: Don't automatically sever the connection, + * rather, allow to ignore responses before the queue is clear + */ + __redisAsyncDisconnect(ac); } /* Sets a pointer to the first argument and its length starting at p. Returns @@ -583,11 +737,14 @@ static const char *nextArgument(const char *start, const char **str, size_t *len static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) { redisContext *c = &(ac->c); redisCallback cb; + struct dict *cbdict; + dictEntry *de; + redisCallback *existcb; int pvariant, hasnext; const char *cstr, *astr; size_t clen, alen; const char *p; - sds sname; + hisds sname; int ret; /* Don't accept new commands when the connection is about to be closed. */ @@ -596,6 +753,7 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void /* Setup callback */ cb.fn = fn; cb.privdata = privdata; + cb.pending_subs = 1; /* Find out which command will be appended. */ p = nextArgument(cmd,&cstr,&clen); @@ -610,13 +768,25 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void /* Add every channel/pattern to the list of subscription callbacks. */ while ((p = nextArgument(p,&astr,&alen)) != NULL) { - sname = sdsnewlen(astr,alen); + sname = hi_sdsnewlen(astr,alen); + if (sname == NULL) + goto oom; + if (pvariant) - ret = dictReplace(ac->sub.patterns,sname,&cb); + cbdict = ac->sub.patterns; else - ret = dictReplace(ac->sub.channels,sname,&cb); + cbdict = ac->sub.channels; - if (ret == 0) sdsfree(sname); + de = dictFind(cbdict,sname); + + if (de != NULL) { + existcb = dictGetEntryVal(de); + cb.pending_subs = existcb->pending_subs + 1; + } + + ret = dictReplace(cbdict,sname,&cb); + + if (ret == 0) hi_sdsfree(sname); } } else if (strncasecmp(cstr,"unsubscribe\r\n",13) == 0) { /* It is only useful to call (P)UNSUBSCRIBE when the context is @@ -645,6 +815,9 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void _EL_ADD_WRITE(ac); return REDIS_OK; +oom: + __redisSetError(&(ac->c), REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; } int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap) { @@ -658,7 +831,7 @@ int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdat return REDIS_ERR; status = __redisAsyncCommand(ac,fn,privdata,cmd,len); - free(cmd); + hi_free(cmd); return status; } @@ -672,12 +845,14 @@ int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata } int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen) { - sds cmd; + hisds cmd; int len; int status; len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen); + if (len < 0) + return REDIS_ERR; status = __redisAsyncCommand(ac,fn,privdata,cmd,len); - sdsfree(cmd); + hi_sdsfree(cmd); return status; } @@ -685,3 +860,28 @@ int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void int status = __redisAsyncCommand(ac,fn,privdata,cmd,len); return status; } + +redisAsyncPushFn *redisAsyncSetPushCallback(redisAsyncContext *ac, redisAsyncPushFn *fn) { + redisAsyncPushFn *old = ac->push_cb; + ac->push_cb = fn; + return old; +} + +int redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv) { + if (!ac->c.command_timeout) { + ac->c.command_timeout = hi_calloc(1, sizeof(tv)); + if (ac->c.command_timeout == NULL) { + __redisSetError(&ac->c, REDIS_ERR_OOM, "Out of memory"); + __redisAsyncCopyError(ac); + return REDIS_ERR; + } + } + + if (tv.tv_sec != ac->c.command_timeout->tv_sec || + tv.tv_usec != ac->c.command_timeout->tv_usec) + { + *ac->c.command_timeout = tv; + } + + return REDIS_OK; +} diff --git a/redis.submodule/deps/hiredis/async.h b/redis.submodule/deps/hiredis/async.h index 59cbf46..b1d2cb2 100644 --- a/redis.submodule/deps/hiredis/async.h +++ b/redis.submodule/deps/hiredis/async.h @@ -45,6 +45,7 @@ typedef void (redisCallbackFn)(struct redisAsyncContext*, void*, void*); typedef struct redisCallback { struct redisCallback *next; /* simple singly linked list */ redisCallbackFn *fn; + int pending_subs; void *privdata; } redisCallback; @@ -56,6 +57,7 @@ typedef struct redisCallbackList { /* Connection callback prototypes */ typedef void (redisDisconnectCallback)(const struct redisAsyncContext*, int status); typedef void (redisConnectCallback)(const struct redisAsyncContext*, int status); +typedef void(redisTimerCallback)(void *timer, void *privdata); /* Context for an async connection to Redis */ typedef struct redisAsyncContext { @@ -68,6 +70,7 @@ typedef struct redisAsyncContext { /* Not used by hiredis */ void *data; + void (*dataCleanup)(void *privdata); /* Event library data and hooks */ struct { @@ -80,6 +83,7 @@ typedef struct redisAsyncContext { void (*addWrite)(void *privdata); void (*delWrite)(void *privdata); void (*cleanup)(void *privdata); + void (*scheduleTimer)(void *privdata, struct timeval tv); } ev; /* Called when either the connection is terminated due to an error or per @@ -92,15 +96,23 @@ typedef struct redisAsyncContext { /* Regular command callbacks */ redisCallbackList replies; + /* Address used for connect() */ + struct sockaddr *saddr; + size_t addrlen; + /* Subscription callbacks */ struct { redisCallbackList invalid; struct dict *channels; struct dict *patterns; } sub; + + /* Any configured RESP3 PUSH handler */ + redisAsyncPushFn *push_cb; } redisAsyncContext; /* Functions that proxy to hiredis */ +redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options); redisAsyncContext *redisAsyncConnect(const char *ip, int port); redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr); redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, @@ -108,12 +120,18 @@ redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, redisAsyncContext *redisAsyncConnectUnix(const char *path); int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn); int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); + +redisAsyncPushFn *redisAsyncSetPushCallback(redisAsyncContext *ac, redisAsyncPushFn *fn); +int redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv); void redisAsyncDisconnect(redisAsyncContext *ac); void redisAsyncFree(redisAsyncContext *ac); /* Handle read/write events */ void redisAsyncHandleRead(redisAsyncContext *ac); void redisAsyncHandleWrite(redisAsyncContext *ac); +void redisAsyncHandleTimeout(redisAsyncContext *ac); +void redisAsyncRead(redisAsyncContext *ac); +void redisAsyncWrite(redisAsyncContext *ac); /* Command functions for an async context. Write the command to the * output buffer and register the provided callback. */ diff --git a/redis.submodule/deps/hiredis/async_private.h b/redis.submodule/deps/hiredis/async_private.h new file mode 100644 index 0000000..ea0558d --- /dev/null +++ b/redis.submodule/deps/hiredis/async_private.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_ASYNC_PRIVATE_H +#define __HIREDIS_ASYNC_PRIVATE_H + +#define _EL_ADD_READ(ctx) \ + do { \ + refreshTimeout(ctx); \ + if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \ + } while (0) +#define _EL_DEL_READ(ctx) do { \ + if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \ + } while(0) +#define _EL_ADD_WRITE(ctx) \ + do { \ + refreshTimeout(ctx); \ + if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \ + } while (0) +#define _EL_DEL_WRITE(ctx) do { \ + if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \ + } while(0) +#define _EL_CLEANUP(ctx) do { \ + if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ + ctx->ev.cleanup = NULL; \ + } while(0) + +static inline void refreshTimeout(redisAsyncContext *ctx) { + #define REDIS_TIMER_ISSET(tvp) \ + (tvp && ((tvp)->tv_sec || (tvp)->tv_usec)) + + #define REDIS_EL_TIMER(ac, tvp) \ + if ((ac)->ev.scheduleTimer && REDIS_TIMER_ISSET(tvp)) { \ + (ac)->ev.scheduleTimer((ac)->ev.data, *(tvp)); \ + } + + if (ctx->c.flags & REDIS_CONNECTED) { + REDIS_EL_TIMER(ctx, ctx->c.command_timeout); + } else { + REDIS_EL_TIMER(ctx, ctx->c.connect_timeout); + } +} + +void __redisAsyncDisconnect(redisAsyncContext *ac); +void redisProcessCallbacks(redisAsyncContext *ac); + +#endif /* __HIREDIS_ASYNC_PRIVATE_H */ diff --git a/redis.submodule/deps/hiredis/dict.c b/redis.submodule/deps/hiredis/dict.c index e17a625..34a33ea 100644 --- a/redis.submodule/deps/hiredis/dict.c +++ b/redis.submodule/deps/hiredis/dict.c @@ -34,6 +34,7 @@ */ #include "fmacros.h" +#include "alloc.h" #include #include #include @@ -71,7 +72,10 @@ static void _dictReset(dict *ht) { /* Create a new hash table */ static dict *dictCreate(dictType *type, void *privDataPtr) { - dict *ht = malloc(sizeof(*ht)); + dict *ht = hi_malloc(sizeof(*ht)); + if (ht == NULL) + return NULL; + _dictInit(ht,type,privDataPtr); return ht; } @@ -97,7 +101,9 @@ static int dictExpand(dict *ht, unsigned long size) { _dictInit(&n, ht->type, ht->privdata); n.size = realsize; n.sizemask = realsize-1; - n.table = calloc(realsize,sizeof(dictEntry*)); + n.table = hi_calloc(realsize,sizeof(dictEntry*)); + if (n.table == NULL) + return DICT_ERR; /* Copy all the elements from the old to the new table: * note that if the old hash table is empty ht->size is zero, @@ -124,7 +130,7 @@ static int dictExpand(dict *ht, unsigned long size) { } } assert(ht->used == 0); - free(ht->table); + hi_free(ht->table); /* Remap the new hashtable in the old */ *ht = n; @@ -142,7 +148,10 @@ static int dictAdd(dict *ht, void *key, void *val) { return DICT_ERR; /* Allocates the memory and stores key */ - entry = malloc(sizeof(*entry)); + entry = hi_malloc(sizeof(*entry)); + if (entry == NULL) + return DICT_ERR; + entry->next = ht->table[index]; ht->table[index] = entry; @@ -166,6 +175,9 @@ static int dictReplace(dict *ht, void *key, void *val) { return 1; /* It already exists, get the entry */ entry = dictFind(ht, key); + if (entry == NULL) + return 0; + /* Free the old value and set the new one */ /* Set the new value and free the old one. Note that it is important * to do that in this order, as the value may just be exactly the same @@ -199,7 +211,7 @@ static int dictDelete(dict *ht, const void *key) { dictFreeEntryKey(ht,de); dictFreeEntryVal(ht,de); - free(de); + hi_free(de); ht->used--; return DICT_OK; } @@ -222,13 +234,13 @@ static int _dictClear(dict *ht) { nextHe = he->next; dictFreeEntryKey(ht, he); dictFreeEntryVal(ht, he); - free(he); + hi_free(he); ht->used--; he = nextHe; } } /* Free the table and the allocated cache structure */ - free(ht->table); + hi_free(ht->table); /* Re-initialize the table */ _dictReset(ht); return DICT_OK; /* never fails */ @@ -237,7 +249,7 @@ static int _dictClear(dict *ht) { /* Clear & Release the hash table */ static void dictRelease(dict *ht) { _dictClear(ht); - free(ht); + hi_free(ht); } static dictEntry *dictFind(dict *ht, const void *key) { @@ -256,7 +268,9 @@ static dictEntry *dictFind(dict *ht, const void *key) { } static dictIterator *dictGetIterator(dict *ht) { - dictIterator *iter = malloc(sizeof(*iter)); + dictIterator *iter = hi_malloc(sizeof(*iter)); + if (iter == NULL) + return NULL; iter->ht = ht; iter->index = -1; @@ -286,7 +300,7 @@ static dictEntry *dictNext(dictIterator *iter) { } static void dictReleaseIterator(dictIterator *iter) { - free(iter); + hi_free(iter); } /* ------------------------- private functions ------------------------------ */ @@ -294,7 +308,7 @@ static void dictReleaseIterator(dictIterator *iter) { /* Expand the hash table if needed */ static int _dictExpandIfNeeded(dict *ht) { /* If the hash table is empty expand it to the initial size, - * if the table is "full" dobule its size. */ + * if the table is "full" double its size. */ if (ht->size == 0) return dictExpand(ht, DICT_HT_INITIAL_SIZE); if (ht->used == ht->size) diff --git a/redis.submodule/deps/hiredis/examples/CMakeLists.txt b/redis.submodule/deps/hiredis/examples/CMakeLists.txt new file mode 100644 index 0000000..1d5bc56 --- /dev/null +++ b/redis.submodule/deps/hiredis/examples/CMakeLists.txt @@ -0,0 +1,49 @@ +INCLUDE(FindPkgConfig) +# Check for GLib + +PKG_CHECK_MODULES(GLIB2 glib-2.0) +if (GLIB2_FOUND) + INCLUDE_DIRECTORIES(${GLIB2_INCLUDE_DIRS}) + LINK_DIRECTORIES(${GLIB2_LIBRARY_DIRS}) + ADD_EXECUTABLE(example-glib example-glib.c) + TARGET_LINK_LIBRARIES(example-glib hiredis ${GLIB2_LIBRARIES}) +ENDIF(GLIB2_FOUND) + +FIND_PATH(LIBEV ev.h + HINTS /usr/local /usr/opt/local + ENV LIBEV_INCLUDE_DIR) + +if (LIBEV) + # Just compile and link with libev + ADD_EXECUTABLE(example-libev example-libev.c) + TARGET_LINK_LIBRARIES(example-libev hiredis ev) +ENDIF() + +FIND_PATH(LIBEVENT event.h) +if (LIBEVENT) + ADD_EXECUTABLE(example-libevent example-libevent) + TARGET_LINK_LIBRARIES(example-libevent hiredis event) +ENDIF() + +FIND_PATH(LIBUV uv.h) +IF (LIBUV) + ADD_EXECUTABLE(example-libuv example-libuv.c) + TARGET_LINK_LIBRARIES(example-libuv hiredis uv) +ENDIF() + +IF (APPLE) + FIND_LIBRARY(CF CoreFoundation) + ADD_EXECUTABLE(example-macosx example-macosx.c) + TARGET_LINK_LIBRARIES(example-macosx hiredis ${CF}) +ENDIF() + +IF (ENABLE_SSL) + ADD_EXECUTABLE(example-ssl example-ssl.c) + TARGET_LINK_LIBRARIES(example-ssl hiredis hiredis_ssl) +ENDIF() + +ADD_EXECUTABLE(example example.c) +TARGET_LINK_LIBRARIES(example hiredis) + +ADD_EXECUTABLE(example-push example-push.c) +TARGET_LINK_LIBRARIES(example-push hiredis) diff --git a/redis.submodule/deps/hiredis/examples/example-ivykis.c b/redis.submodule/deps/hiredis/examples/example-ivykis.c index 67affce..f57dc38 100644 --- a/redis.submodule/deps/hiredis/examples/example-ivykis.c +++ b/redis.submodule/deps/hiredis/examples/example-ivykis.c @@ -33,7 +33,9 @@ void disconnectCallback(const redisAsyncContext *c, int status) { } int main (int argc, char **argv) { +#ifndef _WIN32 signal(SIGPIPE, SIG_IGN); +#endif iv_init(); diff --git a/redis.submodule/deps/hiredis/examples/example-libev.c b/redis.submodule/deps/hiredis/examples/example-libev.c index cc8b166..ec47430 100644 --- a/redis.submodule/deps/hiredis/examples/example-libev.c +++ b/redis.submodule/deps/hiredis/examples/example-libev.c @@ -33,7 +33,9 @@ void disconnectCallback(const redisAsyncContext *c, int status) { } int main (int argc, char **argv) { +#ifndef _WIN32 signal(SIGPIPE, SIG_IGN); +#endif redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); if (c->err) { diff --git a/redis.submodule/deps/hiredis/examples/example-libevent-ssl.c b/redis.submodule/deps/hiredis/examples/example-libevent-ssl.c new file mode 100644 index 0000000..7d99af1 --- /dev/null +++ b/redis.submodule/deps/hiredis/examples/example-libevent-ssl.c @@ -0,0 +1,90 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Disconnected...\n"); +} + +int main (int argc, char **argv) { +#ifndef _WIN32 + signal(SIGPIPE, SIG_IGN); +#endif + + struct event_base *base = event_base_new(); + if (argc < 5) { + fprintf(stderr, + "Usage: %s [ca]\n", argv[0]); + exit(1); + } + + const char *value = argv[1]; + size_t nvalue = strlen(value); + + const char *hostname = argv[2]; + int port = atoi(argv[3]); + + const char *cert = argv[4]; + const char *certKey = argv[5]; + const char *caCert = argc > 5 ? argv[6] : NULL; + + redisSSLContext *ssl; + redisSSLContextError ssl_error; + + redisInitOpenSSL(); + + ssl = redisCreateSSLContext(caCert, NULL, + cert, certKey, NULL, &ssl_error); + if (!ssl) { + printf("Error: %s\n", redisSSLContextGetError(ssl_error)); + return 1; + } + + redisAsyncContext *c = redisAsyncConnect(hostname, port); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + if (redisInitiateSSLWithContext(&c->c, ssl) != REDIS_OK) { + printf("SSL Error!\n"); + exit(1); + } + + redisLibeventAttach(c,base); + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + redisAsyncCommand(c, NULL, NULL, "SET key %b", value, nvalue); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + event_base_dispatch(base); + + redisFreeSSLContext(ssl); + return 0; +} diff --git a/redis.submodule/deps/hiredis/examples/example-libevent.c b/redis.submodule/deps/hiredis/examples/example-libevent.c index d333c22..49bddd0 100644 --- a/redis.submodule/deps/hiredis/examples/example-libevent.c +++ b/redis.submodule/deps/hiredis/examples/example-libevent.c @@ -9,7 +9,12 @@ void getCallback(redisAsyncContext *c, void *r, void *privdata) { redisReply *reply = r; - if (reply == NULL) return; + if (reply == NULL) { + if (c->errstr) { + printf("errstr: %s\n", c->errstr); + } + return; + } printf("argv[%s]: %s\n", (char*)privdata, reply->str); /* Disconnect after receiving the reply to GET */ @@ -33,10 +38,19 @@ void disconnectCallback(const redisAsyncContext *c, int status) { } int main (int argc, char **argv) { +#ifndef _WIN32 signal(SIGPIPE, SIG_IGN); +#endif + struct event_base *base = event_base_new(); + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379); + struct timeval tv = {0}; + tv.tv_sec = 1; + options.connect_timeout = &tv; + - redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + redisAsyncContext *c = redisAsyncConnectWithOptions(&options); if (c->err) { /* Let *c leak for now... */ printf("Error: %s\n", c->errstr); diff --git a/redis.submodule/deps/hiredis/examples/example-libuv.c b/redis.submodule/deps/hiredis/examples/example-libuv.c index a5462d4..cbde452 100644 --- a/redis.submodule/deps/hiredis/examples/example-libuv.c +++ b/redis.submodule/deps/hiredis/examples/example-libuv.c @@ -33,7 +33,10 @@ void disconnectCallback(const redisAsyncContext *c, int status) { } int main (int argc, char **argv) { +#ifndef _WIN32 signal(SIGPIPE, SIG_IGN); +#endif + uv_loop_t* loop = uv_default_loop(); redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); diff --git a/redis.submodule/deps/hiredis/examples/example-push.c b/redis.submodule/deps/hiredis/examples/example-push.c new file mode 100644 index 0000000..2d4ab4d --- /dev/null +++ b/redis.submodule/deps/hiredis/examples/example-push.c @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2020, Michael Grunder + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include + +#define KEY_COUNT 5 + +#define panicAbort(fmt, ...) \ + do { \ + fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, __LINE__, __func__, __VA_ARGS__); \ + exit(-1); \ + } while (0) + +static void assertReplyAndFree(redisContext *context, redisReply *reply, int type) { + if (reply == NULL) + panicAbort("NULL reply from server (error: %s)", context->errstr); + + if (reply->type != type) { + if (reply->type == REDIS_REPLY_ERROR) + fprintf(stderr, "Redis Error: %s\n", reply->str); + + panicAbort("Expected reply type %d but got type %d", type, reply->type); + } + + freeReplyObject(reply); +} + +/* Switch to the RESP3 protocol and enable client tracking */ +static void enableClientTracking(redisContext *c) { + redisReply *reply = redisCommand(c, "HELLO 3"); + if (reply == NULL || c->err) { + panicAbort("NULL reply or server error (error: %s)", c->errstr); + } + + if (reply->type != REDIS_REPLY_MAP) { + fprintf(stderr, "Error: Can't send HELLO 3 command. Are you sure you're "); + fprintf(stderr, "connected to redis-server >= 6.0.0?\nRedis error: %s\n", + reply->type == REDIS_REPLY_ERROR ? reply->str : "(unknown)"); + exit(-1); + } + + freeReplyObject(reply); + + /* Enable client tracking */ + reply = redisCommand(c, "CLIENT TRACKING ON"); + assertReplyAndFree(c, reply, REDIS_REPLY_STATUS); +} + +void pushReplyHandler(void *privdata, void *r) { + redisReply *reply = r; + int *invalidations = privdata; + + /* Sanity check on the invalidation reply */ + if (reply->type != REDIS_REPLY_PUSH || reply->elements != 2 || + reply->element[1]->type != REDIS_REPLY_ARRAY || + reply->element[1]->element[0]->type != REDIS_REPLY_STRING) + { + panicAbort("%s", "Can't parse PUSH message!"); + } + + /* Increment our invalidation count */ + *invalidations += 1; + + printf("pushReplyHandler(): INVALIDATE '%s' (invalidation count: %d)\n", + reply->element[1]->element[0]->str, *invalidations); + + freeReplyObject(reply); +} + +/* We aren't actually freeing anything here, but it is included to show that we can + * have hiredis call our data destructor when freeing the context */ +void privdata_dtor(void *privdata) { + unsigned int *icount = privdata; + printf("privdata_dtor(): In context privdata dtor (invalidations: %u)\n", *icount); +} + +int main(int argc, char **argv) { + unsigned int j, invalidations = 0; + redisContext *c; + redisReply *reply; + + const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + int port = (argc > 2) ? atoi(argv[2]) : 6379; + + redisOptions o = {0}; + REDIS_OPTIONS_SET_TCP(&o, hostname, port); + + /* Set our context privdata to the address of our invalidation counter. Each + * time our PUSH handler is called, hiredis will pass the privdata for context. + * + * This could also be done after we create the context like so: + * + * c->privdata = &invalidations; + * c->free_privdata = privdata_dtor; + */ + REDIS_OPTIONS_SET_PRIVDATA(&o, &invalidations, privdata_dtor); + + /* Set our custom PUSH message handler */ + o.push_cb = pushReplyHandler; + + c = redisConnectWithOptions(&o); + if (c == NULL || c->err) + panicAbort("Connection error: %s", c ? c->errstr : "OOM"); + + /* Enable RESP3 and turn on client tracking */ + enableClientTracking(c); + + /* Set some keys and then read them back. Once we do that, Redis will deliver + * invalidation push messages whenever the key is modified */ + for (j = 0; j < KEY_COUNT; j++) { + reply = redisCommand(c, "SET key:%d initial:%d", j, j); + assertReplyAndFree(c, reply, REDIS_REPLY_STATUS); + + reply = redisCommand(c, "GET key:%d", j); + assertReplyAndFree(c, reply, REDIS_REPLY_STRING); + } + + /* Trigger invalidation messages by updating keys we just read */ + for (j = 0; j < KEY_COUNT; j++) { + printf(" main(): SET key:%d update:%d\n", j, j); + reply = redisCommand(c, "SET key:%d update:%d", j, j); + assertReplyAndFree(c, reply, REDIS_REPLY_STATUS); + printf(" main(): SET REPLY OK\n"); + } + + printf("\nTotal detected invalidations: %d, expected: %d\n", invalidations, KEY_COUNT); + + /* PING server */ + redisFree(c); +} diff --git a/redis.submodule/deps/hiredis/examples/example-ssl.c b/redis.submodule/deps/hiredis/examples/example-ssl.c new file mode 100644 index 0000000..c754177 --- /dev/null +++ b/redis.submodule/deps/hiredis/examples/example-ssl.c @@ -0,0 +1,110 @@ +#include +#include +#include + +#include +#include +#include + +int main(int argc, char **argv) { + unsigned int j; + redisSSLContext *ssl; + redisSSLContextError ssl_error; + redisContext *c; + redisReply *reply; + if (argc < 4) { + printf("Usage: %s [ca]\n", argv[0]); + exit(1); + } + const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + int port = atoi(argv[2]); + const char *cert = argv[3]; + const char *key = argv[4]; + const char *ca = argc > 4 ? argv[5] : NULL; + + redisInitOpenSSL(); + ssl = redisCreateSSLContext(ca, NULL, cert, key, NULL, &ssl_error); + if (!ssl) { + printf("SSL Context error: %s\n", + redisSSLContextGetError(ssl_error)); + exit(1); + } + + struct timeval tv = { 1, 500000 }; // 1.5 seconds + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, hostname, port); + options.connect_timeout = &tv; + c = redisConnectWithOptions(&options); + + if (c == NULL || c->err) { + if (c) { + printf("Connection error: %s\n", c->errstr); + redisFree(c); + } else { + printf("Connection error: can't allocate redis context\n"); + } + exit(1); + } + + if (redisInitiateSSLWithContext(c, ssl) != REDIS_OK) { + printf("Couldn't initialize SSL!\n"); + printf("Error: %s\n", c->errstr); + redisFree(c); + exit(1); + } + + /* PING server */ + reply = redisCommand(c,"PING"); + printf("PING: %s\n", reply->str); + freeReplyObject(reply); + + /* Set a key */ + reply = redisCommand(c,"SET %s %s", "foo", "hello world"); + printf("SET: %s\n", reply->str); + freeReplyObject(reply); + + /* Set a key using binary safe API */ + reply = redisCommand(c,"SET %b %b", "bar", (size_t) 3, "hello", (size_t) 5); + printf("SET (binary API): %s\n", reply->str); + freeReplyObject(reply); + + /* Try a GET and two INCR */ + reply = redisCommand(c,"GET foo"); + printf("GET foo: %s\n", reply->str); + freeReplyObject(reply); + + reply = redisCommand(c,"INCR counter"); + printf("INCR counter: %lld\n", reply->integer); + freeReplyObject(reply); + /* again ... */ + reply = redisCommand(c,"INCR counter"); + printf("INCR counter: %lld\n", reply->integer); + freeReplyObject(reply); + + /* Create a list of numbers, from 0 to 9 */ + reply = redisCommand(c,"DEL mylist"); + freeReplyObject(reply); + for (j = 0; j < 10; j++) { + char buf[64]; + + snprintf(buf,64,"%u",j); + reply = redisCommand(c,"LPUSH mylist element-%s", buf); + freeReplyObject(reply); + } + + /* Let's check what we have inside the list */ + reply = redisCommand(c,"LRANGE mylist 0 -1"); + if (reply->type == REDIS_REPLY_ARRAY) { + for (j = 0; j < reply->elements; j++) { + printf("%u) %s\n", j, reply->element[j]->str); + } + } + freeReplyObject(reply); + + /* Disconnects and frees the context */ + redisFree(c); + + redisFreeSSLContext(ssl); + + return 0; +} diff --git a/redis.submodule/deps/hiredis/examples/example.c b/redis.submodule/deps/hiredis/examples/example.c index 4d494c5..15dacbd 100644 --- a/redis.submodule/deps/hiredis/examples/example.c +++ b/redis.submodule/deps/hiredis/examples/example.c @@ -1,18 +1,31 @@ #include #include #include - #include +#include int main(int argc, char **argv) { - unsigned int j; + unsigned int j, isunix = 0; redisContext *c; redisReply *reply; const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + + if (argc > 2) { + if (*argv[2] == 'u' || *argv[2] == 'U') { + isunix = 1; + /* in this case, host is the path to the unix socket */ + printf("Will connect to unix socket @%s\n", hostname); + } + } + int port = (argc > 2) ? atoi(argv[2]) : 6379; struct timeval timeout = { 1, 500000 }; // 1.5 seconds - c = redisConnectWithTimeout(hostname, port, timeout); + if (isunix) { + c = redisConnectUnixWithTimeout(hostname, timeout); + } else { + c = redisConnectWithTimeout(hostname, port, timeout); + } if (c == NULL || c->err) { if (c) { printf("Connection error: %s\n", c->errstr); diff --git a/redis.submodule/deps/hiredis/fmacros.h b/redis.submodule/deps/hiredis/fmacros.h index 9a56643..3227faa 100644 --- a/redis.submodule/deps/hiredis/fmacros.h +++ b/redis.submodule/deps/hiredis/fmacros.h @@ -1,25 +1,12 @@ #ifndef __HIREDIS_FMACRO_H #define __HIREDIS_FMACRO_H -#if defined(__linux__) -#define _BSD_SOURCE -#define _DEFAULT_SOURCE -#endif - -#if defined(__CYGWIN__) -#include -#endif - -#if defined(__sun__) -#define _POSIX_C_SOURCE 200112L -#else -#if !(defined(__APPLE__) && defined(__MACH__)) && !(defined(__FreeBSD__)) #define _XOPEN_SOURCE 600 -#endif -#endif +#define _POSIX_C_SOURCE 200112L #if defined(__APPLE__) && defined(__MACH__) -#define _OSX +/* Enable TCP_KEEPALIVE */ +#define _DARWIN_C_SOURCE #endif #endif diff --git a/redis.submodule/deps/hiredis/hiredis-config.cmake.in b/redis.submodule/deps/hiredis/hiredis-config.cmake.in new file mode 100644 index 0000000..98851dc --- /dev/null +++ b/redis.submodule/deps/hiredis/hiredis-config.cmake.in @@ -0,0 +1,13 @@ +@PACKAGE_INIT@ + +set_and_check(hiredis_INCLUDEDIR "@PACKAGE_INCLUDE_INSTALL_DIR@") + +IF (NOT TARGET hiredis::hiredis) + INCLUDE(${CMAKE_CURRENT_LIST_DIR}/hiredis-targets.cmake) +ENDIF() + +SET(hiredis_LIBRARIES hiredis::hiredis) +SET(hiredis_INCLUDE_DIRS ${hiredis_INCLUDEDIR}) + +check_required_components(hiredis) + diff --git a/redis.submodule/deps/hiredis/hiredis.c b/redis.submodule/deps/hiredis/hiredis.c index 18bdfc9..990f619 100644 --- a/redis.submodule/deps/hiredis/hiredis.c +++ b/redis.submodule/deps/hiredis/hiredis.c @@ -34,7 +34,6 @@ #include "fmacros.h" #include #include -#include #include #include #include @@ -42,12 +41,27 @@ #include "hiredis.h" #include "net.h" #include "sds.h" +#include "async.h" +#include "win32.h" + +extern int redisContextUpdateConnectTimeout(redisContext *c, const struct timeval *timeout); +extern int redisContextUpdateCommandTimeout(redisContext *c, const struct timeval *timeout); + +static redisContextFuncs redisContextDefaultFuncs = { + .free_privctx = NULL, + .async_read = redisAsyncRead, + .async_write = redisAsyncWrite, + .read = redisNetRead, + .write = redisNetWrite +}; static redisReply *createReplyObject(int type); static void *createStringObject(const redisReadTask *task, char *str, size_t len); -static void *createArrayObject(const redisReadTask *task, int elements); +static void *createArrayObject(const redisReadTask *task, size_t elements); static void *createIntegerObject(const redisReadTask *task, long long value); +static void *createDoubleObject(const redisReadTask *task, double value, char *str, size_t len); static void *createNilObject(const redisReadTask *task); +static void *createBoolObject(const redisReadTask *task, int bval); /* Default set of functions to build the reply. Keep in mind that such a * function returning NULL is interpreted as OOM. */ @@ -55,13 +69,15 @@ static redisReplyObjectFunctions defaultFunctions = { createStringObject, createArrayObject, createIntegerObject, + createDoubleObject, createNilObject, + createBoolObject, freeReplyObject }; /* Create a reply object */ static redisReply *createReplyObject(int type) { - redisReply *r = calloc(1,sizeof(*r)); + redisReply *r = hi_calloc(1,sizeof(*r)); if (r == NULL) return NULL; @@ -82,21 +98,24 @@ void freeReplyObject(void *reply) { case REDIS_REPLY_INTEGER: break; /* Nothing to free */ case REDIS_REPLY_ARRAY: + case REDIS_REPLY_MAP: + case REDIS_REPLY_SET: + case REDIS_REPLY_PUSH: if (r->element != NULL) { for (j = 0; j < r->elements; j++) - if (r->element[j] != NULL) - freeReplyObject(r->element[j]); - free(r->element); + freeReplyObject(r->element[j]); + hi_free(r->element); } break; case REDIS_REPLY_ERROR: case REDIS_REPLY_STATUS: case REDIS_REPLY_STRING: - if (r->str != NULL) - free(r->str); + case REDIS_REPLY_DOUBLE: + case REDIS_REPLY_VERB: + hi_free(r->str); break; } - free(r); + hi_free(r); } static void *createStringObject(const redisReadTask *task, char *str, size_t len) { @@ -107,39 +126,56 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len if (r == NULL) return NULL; - buf = malloc(len+1); - if (buf == NULL) { - freeReplyObject(r); - return NULL; - } - assert(task->type == REDIS_REPLY_ERROR || task->type == REDIS_REPLY_STATUS || - task->type == REDIS_REPLY_STRING); + task->type == REDIS_REPLY_STRING || + task->type == REDIS_REPLY_VERB); /* Copy string value */ - memcpy(buf,str,len); - buf[len] = '\0'; + if (task->type == REDIS_REPLY_VERB) { + buf = hi_malloc(len-4+1); /* Skip 4 bytes of verbatim type header. */ + if (buf == NULL) goto oom; + + memcpy(r->vtype,str,3); + r->vtype[3] = '\0'; + memcpy(buf,str+4,len-4); + buf[len-4] = '\0'; + r->len = len - 4; + } else { + buf = hi_malloc(len+1); + if (buf == NULL) goto oom; + + memcpy(buf,str,len); + buf[len] = '\0'; + r->len = len; + } r->str = buf; - r->len = len; if (task->parent) { parent = task->parent->obj; - assert(parent->type == REDIS_REPLY_ARRAY); + assert(parent->type == REDIS_REPLY_ARRAY || + parent->type == REDIS_REPLY_MAP || + parent->type == REDIS_REPLY_SET || + parent->type == REDIS_REPLY_PUSH); parent->element[task->idx] = r; } return r; + +oom: + freeReplyObject(r); + return NULL; } -static void *createArrayObject(const redisReadTask *task, int elements) { +static void *createArrayObject(const redisReadTask *task, size_t elements) { redisReply *r, *parent; - r = createReplyObject(REDIS_REPLY_ARRAY); + r = createReplyObject(task->type); if (r == NULL) return NULL; if (elements > 0) { - r->element = calloc(elements,sizeof(redisReply*)); + if (SIZE_MAX / sizeof(redisReply*) < elements) return NULL; /* Don't overflow */ + r->element = hi_calloc(elements,sizeof(redisReply*)); if (r->element == NULL) { freeReplyObject(r); return NULL; @@ -150,7 +186,10 @@ static void *createArrayObject(const redisReadTask *task, int elements) { if (task->parent) { parent = task->parent->obj; - assert(parent->type == REDIS_REPLY_ARRAY); + assert(parent->type == REDIS_REPLY_ARRAY || + parent->type == REDIS_REPLY_MAP || + parent->type == REDIS_REPLY_SET || + parent->type == REDIS_REPLY_PUSH); parent->element[task->idx] = r; } return r; @@ -167,7 +206,42 @@ static void *createIntegerObject(const redisReadTask *task, long long value) { if (task->parent) { parent = task->parent->obj; - assert(parent->type == REDIS_REPLY_ARRAY); + assert(parent->type == REDIS_REPLY_ARRAY || + parent->type == REDIS_REPLY_MAP || + parent->type == REDIS_REPLY_SET || + parent->type == REDIS_REPLY_PUSH); + parent->element[task->idx] = r; + } + return r; +} + +static void *createDoubleObject(const redisReadTask *task, double value, char *str, size_t len) { + redisReply *r, *parent; + + r = createReplyObject(REDIS_REPLY_DOUBLE); + if (r == NULL) + return NULL; + + r->dval = value; + r->str = hi_malloc(len+1); + if (r->str == NULL) { + freeReplyObject(r); + return NULL; + } + + /* The double reply also has the original protocol string representing a + * double as a null terminated string. This way the caller does not need + * to format back for string conversion, especially since Redis does efforts + * to make the string more human readable avoiding the calssical double + * decimal string conversion artifacts. */ + memcpy(r->str, str, len); + r->str[len] = '\0'; + + if (task->parent) { + parent = task->parent->obj; + assert(parent->type == REDIS_REPLY_ARRAY || + parent->type == REDIS_REPLY_MAP || + parent->type == REDIS_REPLY_SET); parent->element[task->idx] = r; } return r; @@ -182,7 +256,29 @@ static void *createNilObject(const redisReadTask *task) { if (task->parent) { parent = task->parent->obj; - assert(parent->type == REDIS_REPLY_ARRAY); + assert(parent->type == REDIS_REPLY_ARRAY || + parent->type == REDIS_REPLY_MAP || + parent->type == REDIS_REPLY_SET || + parent->type == REDIS_REPLY_PUSH); + parent->element[task->idx] = r; + } + return r; +} + +static void *createBoolObject(const redisReadTask *task, int bval) { + redisReply *r, *parent; + + r = createReplyObject(REDIS_REPLY_BOOL); + if (r == NULL) + return NULL; + + r->integer = bval != 0; + + if (task->parent) { + parent = task->parent->obj; + assert(parent->type == REDIS_REPLY_ARRAY || + parent->type == REDIS_REPLY_MAP || + parent->type == REDIS_REPLY_SET); parent->element[task->idx] = r; } return r; @@ -211,7 +307,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { const char *c = format; char *cmd = NULL; /* final command */ int pos; /* position in final command */ - sds curarg, newarg; /* current argument */ + hisds curarg, newarg; /* current argument */ int touched = 0; /* was the current argument touched? */ char **curargv = NULL, **newargv = NULL; int argc = 0; @@ -224,7 +320,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { return -1; /* Build the command string accordingly to protocol */ - curarg = sdsempty(); + curarg = hi_sdsempty(); if (curarg == NULL) return -1; @@ -232,19 +328,19 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { if (*c != '%' || c[1] == '\0') { if (*c == ' ') { if (touched) { - newargv = realloc(curargv,sizeof(char*)*(argc+1)); + newargv = hi_realloc(curargv,sizeof(char*)*(argc+1)); if (newargv == NULL) goto memory_err; curargv = newargv; curargv[argc++] = curarg; - totlen += bulklen(sdslen(curarg)); + totlen += bulklen(hi_sdslen(curarg)); /* curarg is put in argv so it can be overwritten. */ - curarg = sdsempty(); + curarg = hi_sdsempty(); if (curarg == NULL) goto memory_err; touched = 0; } } else { - newarg = sdscatlen(curarg,c,1); + newarg = hi_sdscatlen(curarg,c,1); if (newarg == NULL) goto memory_err; curarg = newarg; touched = 1; @@ -261,16 +357,16 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { arg = va_arg(ap,char*); size = strlen(arg); if (size > 0) - newarg = sdscatlen(curarg,arg,size); + newarg = hi_sdscatlen(curarg,arg,size); break; case 'b': arg = va_arg(ap,char*); size = va_arg(ap,size_t); if (size > 0) - newarg = sdscatlen(curarg,arg,size); + newarg = hi_sdscatlen(curarg,arg,size); break; case '%': - newarg = sdscat(curarg,"%"); + newarg = hi_sdscat(curarg,"%"); break; default: /* Try to detect printf format */ @@ -358,7 +454,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { if (_l < sizeof(_format)-2) { memcpy(_format,c,_l); _format[_l] = '\0'; - newarg = sdscatvprintf(curarg,_format,_cpy); + newarg = hi_sdscatvprintf(curarg,_format,_cpy); /* Update current position (note: outer blocks * increment c twice so compensate here) */ @@ -381,13 +477,13 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { /* Add the last argument if needed */ if (touched) { - newargv = realloc(curargv,sizeof(char*)*(argc+1)); + newargv = hi_realloc(curargv,sizeof(char*)*(argc+1)); if (newargv == NULL) goto memory_err; curargv = newargv; curargv[argc++] = curarg; - totlen += bulklen(sdslen(curarg)); + totlen += bulklen(hi_sdslen(curarg)); } else { - sdsfree(curarg); + hi_sdsfree(curarg); } /* Clear curarg because it was put in curargv or was free'd. */ @@ -397,22 +493,22 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { totlen += 1+countDigits(argc)+2; /* Build the command at protocol level */ - cmd = malloc(totlen+1); + cmd = hi_malloc(totlen+1); if (cmd == NULL) goto memory_err; pos = sprintf(cmd,"*%d\r\n",argc); for (j = 0; j < argc; j++) { - pos += sprintf(cmd+pos,"$%zu\r\n",sdslen(curargv[j])); - memcpy(cmd+pos,curargv[j],sdslen(curargv[j])); - pos += sdslen(curargv[j]); - sdsfree(curargv[j]); + pos += sprintf(cmd+pos,"$%zu\r\n",hi_sdslen(curargv[j])); + memcpy(cmd+pos,curargv[j],hi_sdslen(curargv[j])); + pos += hi_sdslen(curargv[j]); + hi_sdsfree(curargv[j]); cmd[pos++] = '\r'; cmd[pos++] = '\n'; } assert(pos == totlen); cmd[pos] = '\0'; - free(curargv); + hi_free(curargv); *target = cmd; return totlen; @@ -427,16 +523,12 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) { cleanup: if (curargv) { while(argc--) - sdsfree(curargv[argc]); - free(curargv); + hi_sdsfree(curargv[argc]); + hi_free(curargv); } - sdsfree(curarg); - - /* No need to check cmd since it is the last statement that can fail, - * but do it anyway to be as defensive as possible. */ - if (cmd != NULL) - free(cmd); + hi_sdsfree(curarg); + hi_free(cmd); return error_type; } @@ -468,16 +560,16 @@ int redisFormatCommand(char **target, const char *format, ...) { return len; } -/* Format a command according to the Redis protocol using an sds string and - * sdscatfmt for the processing of arguments. This function takes the +/* Format a command according to the Redis protocol using an hisds string and + * hi_sdscatfmt for the processing of arguments. This function takes the * number of arguments, an array with arguments and an array with their * lengths. If the latter is set to NULL, strlen will be used to compute the * argument lengths. */ -int redisFormatSdsCommandArgv(sds *target, int argc, const char **argv, +int redisFormatSdsCommandArgv(hisds *target, int argc, const char **argv, const size_t *argvlen) { - sds cmd; + hisds cmd, aux; unsigned long long totlen; int j; size_t len; @@ -494,32 +586,36 @@ int redisFormatSdsCommandArgv(sds *target, int argc, const char **argv, } /* Use an SDS string for command construction */ - cmd = sdsempty(); + cmd = hi_sdsempty(); if (cmd == NULL) return -1; /* We already know how much storage we need */ - cmd = sdsMakeRoomFor(cmd, totlen); - if (cmd == NULL) + aux = hi_sdsMakeRoomFor(cmd, totlen); + if (aux == NULL) { + hi_sdsfree(cmd); return -1; + } + + cmd = aux; /* Construct command */ - cmd = sdscatfmt(cmd, "*%i\r\n", argc); + cmd = hi_sdscatfmt(cmd, "*%i\r\n", argc); for (j=0; j < argc; j++) { len = argvlen ? argvlen[j] : strlen(argv[j]); - cmd = sdscatfmt(cmd, "$%u\r\n", len); - cmd = sdscatlen(cmd, argv[j], len); - cmd = sdscatlen(cmd, "\r\n", sizeof("\r\n")-1); + cmd = hi_sdscatfmt(cmd, "$%u\r\n", len); + cmd = hi_sdscatlen(cmd, argv[j], len); + cmd = hi_sdscatlen(cmd, "\r\n", sizeof("\r\n")-1); } - assert(sdslen(cmd)==totlen); + assert(hi_sdslen(cmd)==totlen); *target = cmd; return totlen; } -void redisFreeSdsCommand(sds cmd) { - sdsfree(cmd); +void redisFreeSdsCommand(hisds cmd) { + hi_sdsfree(cmd); } /* Format a command according to the Redis protocol. This function takes the @@ -545,7 +641,7 @@ int redisFormatCommandArgv(char **target, int argc, const char **argv, const siz } /* Build the command at protocol level */ - cmd = malloc(totlen+1); + cmd = hi_malloc(totlen+1); if (cmd == NULL) return -1; @@ -566,7 +662,7 @@ int redisFormatCommandArgv(char **target, int argc, const char **argv, const siz } void redisFreeCommand(char *cmd) { - free(cmd); + hi_free(cmd); } void __redisSetError(redisContext *c, int type, const char *str) { @@ -581,7 +677,7 @@ void __redisSetError(redisContext *c, int type, const char *str) { } else { /* Only REDIS_ERR_IO may lack a description! */ assert(type == REDIS_ERR_IO); - __redis_strerror_r(errno, c->errstr, sizeof(c->errstr)); + strerror_r(errno, c->errstr, sizeof(c->errstr)); } } @@ -589,21 +685,23 @@ redisReader *redisReaderCreate(void) { return redisReaderCreateWithFunctions(&defaultFunctions); } +static void redisPushAutoFree(void *privdata, void *reply) { + (void)privdata; + freeReplyObject(reply); +} + static redisContext *redisContextInit(void) { redisContext *c; - c = calloc(1,sizeof(redisContext)); + c = hi_calloc(1, sizeof(*c)); if (c == NULL) return NULL; - c->err = 0; - c->errstr[0] = '\0'; - c->obuf = sdsempty(); + c->funcs = &redisContextDefaultFuncs; + + c->obuf = hi_sdsempty(); c->reader = redisReaderCreate(); - c->tcp.host = NULL; - c->tcp.source_addr = NULL; - c->unix_sock.path = NULL; - c->timeout = NULL; + c->fd = REDIS_INVALID_FD; if (c->obuf == NULL || c->reader == NULL) { redisFree(c); @@ -616,26 +714,30 @@ static redisContext *redisContextInit(void) { void redisFree(redisContext *c) { if (c == NULL) return; - if (c->fd > 0) - close(c->fd); - if (c->obuf != NULL) - sdsfree(c->obuf); - if (c->reader != NULL) - redisReaderFree(c->reader); - if (c->tcp.host) - free(c->tcp.host); - if (c->tcp.source_addr) - free(c->tcp.source_addr); - if (c->unix_sock.path) - free(c->unix_sock.path); - if (c->timeout) - free(c->timeout); - free(c); -} - -int redisFreeKeepFd(redisContext *c) { - int fd = c->fd; - c->fd = -1; + redisNetClose(c); + + hi_sdsfree(c->obuf); + redisReaderFree(c->reader); + hi_free(c->tcp.host); + hi_free(c->tcp.source_addr); + hi_free(c->unix_sock.path); + hi_free(c->connect_timeout); + hi_free(c->command_timeout); + hi_free(c->saddr); + + if (c->privdata && c->free_privdata) + c->free_privdata(c->privdata); + + if (c->funcs->free_privctx) + c->funcs->free_privctx(c->privctx); + + memset(c, 0xff, sizeof(*c)); + hi_free(c); +} + +redisFD redisFreeKeepFd(redisContext *c) { + redisFD fd = c->fd; + c->fd = REDIS_INVALID_FD; redisFree(c); return fd; } @@ -644,132 +746,163 @@ int redisReconnect(redisContext *c) { c->err = 0; memset(c->errstr, '\0', strlen(c->errstr)); - if (c->fd > 0) { - close(c->fd); + if (c->privctx && c->funcs->free_privctx) { + c->funcs->free_privctx(c->privctx); + c->privctx = NULL; } - sdsfree(c->obuf); + redisNetClose(c); + + hi_sdsfree(c->obuf); redisReaderFree(c->reader); - c->obuf = sdsempty(); + c->obuf = hi_sdsempty(); c->reader = redisReaderCreate(); + if (c->obuf == NULL || c->reader == NULL) { + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; + } + + int ret = REDIS_ERR; if (c->connection_type == REDIS_CONN_TCP) { - return redisContextConnectBindTcp(c, c->tcp.host, c->tcp.port, - c->timeout, c->tcp.source_addr); + ret = redisContextConnectBindTcp(c, c->tcp.host, c->tcp.port, + c->connect_timeout, c->tcp.source_addr); } else if (c->connection_type == REDIS_CONN_UNIX) { - return redisContextConnectUnix(c, c->unix_sock.path, c->timeout); + ret = redisContextConnectUnix(c, c->unix_sock.path, c->connect_timeout); } else { /* Something bad happened here and shouldn't have. There isn't enough information in the context to reconnect. */ __redisSetError(c,REDIS_ERR_OTHER,"Not enough information to reconnect"); + ret = REDIS_ERR; } - return REDIS_ERR; -} + if (c->command_timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) { + redisContextSetTimeout(c, *c->command_timeout); + } -/* Connect to a Redis instance. On error the field error in the returned - * context will be set to the return value of the error function. - * When no set of reply functions is given, the default set will be used. */ -redisContext *redisConnect(const char *ip, int port) { - redisContext *c; + return ret; +} - c = redisContextInit(); - if (c == NULL) +redisContext *redisConnectWithOptions(const redisOptions *options) { + redisContext *c = redisContextInit(); + if (c == NULL) { return NULL; + } + if (!(options->options & REDIS_OPT_NONBLOCK)) { + c->flags |= REDIS_BLOCK; + } + if (options->options & REDIS_OPT_REUSEADDR) { + c->flags |= REDIS_REUSEADDR; + } + if (options->options & REDIS_OPT_NOAUTOFREE) { + c->flags |= REDIS_NO_AUTO_FREE; + } - c->flags |= REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,NULL); - return c; -} + /* Set any user supplied RESP3 PUSH handler or use freeReplyObject + * as a default unless specifically flagged that we don't want one. */ + if (options->push_cb != NULL) + redisSetPushCallback(c, options->push_cb); + else if (!(options->options & REDIS_OPT_NO_PUSH_AUTOFREE)) + redisSetPushCallback(c, redisPushAutoFree); -redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) { - redisContext *c; + c->privdata = options->privdata; + c->free_privdata = options->free_privdata; - c = redisContextInit(); - if (c == NULL) + if (redisContextUpdateConnectTimeout(c, options->connect_timeout) != REDIS_OK || + redisContextUpdateCommandTimeout(c, options->command_timeout) != REDIS_OK) { + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); + return c; + } + + if (options->type == REDIS_CONN_TCP) { + redisContextConnectBindTcp(c, options->endpoint.tcp.ip, + options->endpoint.tcp.port, options->connect_timeout, + options->endpoint.tcp.source_addr); + } else if (options->type == REDIS_CONN_UNIX) { + redisContextConnectUnix(c, options->endpoint.unix_socket, + options->connect_timeout); + } else if (options->type == REDIS_CONN_USERFD) { + c->fd = options->endpoint.fd; + c->flags |= REDIS_CONNECTED; + } else { + // Unknown type - FIXME - FREE return NULL; + } + + if (options->command_timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) { + redisContextSetTimeout(c, *options->command_timeout); + } - c->flags |= REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,&tv); return c; } -redisContext *redisConnectNonBlock(const char *ip, int port) { - redisContext *c; +/* Connect to a Redis instance. On error the field error in the returned + * context will be set to the return value of the error function. + * When no set of reply functions is given, the default set will be used. */ +redisContext *redisConnect(const char *ip, int port) { + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + return redisConnectWithOptions(&options); +} - c = redisContextInit(); - if (c == NULL) - return NULL; +redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) { + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.connect_timeout = &tv; + return redisConnectWithOptions(&options); +} - c->flags &= ~REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,NULL); - return c; +redisContext *redisConnectNonBlock(const char *ip, int port) { + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } redisContext *redisConnectBindNonBlock(const char *ip, int port, const char *source_addr) { - redisContext *c = redisContextInit(); - c->flags &= ~REDIS_BLOCK; - redisContextConnectBindTcp(c,ip,port,NULL,source_addr); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, const char *source_addr) { - redisContext *c = redisContextInit(); - c->flags &= ~REDIS_BLOCK; - c->flags |= REDIS_REUSEADDR; - redisContextConnectBindTcp(c,ip,port,NULL,source_addr); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + options.options |= REDIS_OPT_NONBLOCK|REDIS_OPT_REUSEADDR; + return redisConnectWithOptions(&options); } redisContext *redisConnectUnix(const char *path) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectUnix(c,path,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + return redisConnectWithOptions(&options); } redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectUnix(c,path,&tv); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + options.connect_timeout = &tv; + return redisConnectWithOptions(&options); } redisContext *redisConnectUnixNonBlock(const char *path) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags &= ~REDIS_BLOCK; - redisContextConnectUnix(c,path,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } -redisContext *redisConnectFd(int fd) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->fd = fd; - c->flags |= REDIS_BLOCK | REDIS_CONNECTED; - return c; +redisContext *redisConnectFd(redisFD fd) { + redisOptions options = {0}; + options.type = REDIS_CONN_USERFD; + options.endpoint.fd = fd; + return redisConnectWithOptions(&options); } /* Set read/write timeout on a blocking socket. */ @@ -786,10 +919,17 @@ int redisEnableKeepAlive(redisContext *c) { return REDIS_OK; } +/* Set a user provided RESP3 PUSH handler and return any old one set. */ +redisPushFn *redisSetPushCallback(redisContext *c, redisPushFn *fn) { + redisPushFn *old = c->push_cb; + c->push_cb = fn; + return old; +} + /* Use this function to handle a read event on the descriptor. It will try * and read some bytes from the socket and feed them to the reply parser. * - * After this function is called, you may use redisContextReadReply to + * After this function is called, you may use redisGetReplyFromReader to * see if there is a reply available. */ int redisBufferRead(redisContext *c) { char buf[1024*16]; @@ -799,22 +939,15 @@ int redisBufferRead(redisContext *c) { if (c->err) return REDIS_ERR; - nread = read(c->fd,buf,sizeof(buf)); - if (nread == -1) { - if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { - /* Try again later */ - } else { - __redisSetError(c,REDIS_ERR_IO,NULL); + nread = c->funcs->read(c, buf, sizeof(buf)); + if (nread > 0) { + if (redisReaderFeed(c->reader, buf, nread) != REDIS_OK) { + __redisSetError(c, c->reader->err, c->reader->errstr); return REDIS_ERR; + } else { } - } else if (nread == 0) { - __redisSetError(c,REDIS_ERR_EOF,"Server closed the connection"); + } else if (nread < 0) { return REDIS_ERR; - } else { - if (redisReaderFeed(c->reader,buf,nread) != REDIS_OK) { - __redisSetError(c,c->reader->err,c->reader->errstr); - return REDIS_ERR; - } } return REDIS_OK; } @@ -829,32 +962,32 @@ int redisBufferRead(redisContext *c) { * c->errstr to hold the appropriate error string. */ int redisBufferWrite(redisContext *c, int *done) { - int nwritten; /* Return early when the context has seen an error. */ if (c->err) return REDIS_ERR; - if (sdslen(c->obuf) > 0) { - nwritten = write(c->fd,c->obuf,sdslen(c->obuf)); - if (nwritten == -1) { - if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { - /* Try again later */ - } else { - __redisSetError(c,REDIS_ERR_IO,NULL); - return REDIS_ERR; - } + if (hi_sdslen(c->obuf) > 0) { + ssize_t nwritten = c->funcs->write(c); + if (nwritten < 0) { + return REDIS_ERR; } else if (nwritten > 0) { - if (nwritten == (signed)sdslen(c->obuf)) { - sdsfree(c->obuf); - c->obuf = sdsempty(); + if (nwritten == (ssize_t)hi_sdslen(c->obuf)) { + hi_sdsfree(c->obuf); + c->obuf = hi_sdsempty(); + if (c->obuf == NULL) + goto oom; } else { - sdsrange(c->obuf,nwritten,-1); + if (hi_sdsrange(c->obuf,nwritten,-1) < 0) goto oom; } } } - if (done != NULL) *done = (sdslen(c->obuf) == 0); + if (done != NULL) *done = (hi_sdslen(c->obuf) == 0); return REDIS_OK; + +oom: + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; } /* Internal helper function to try and get a reply from the reader, @@ -864,9 +997,21 @@ int redisGetReplyFromReader(redisContext *c, void **reply) { __redisSetError(c,c->reader->err,c->reader->errstr); return REDIS_ERR; } + return REDIS_OK; } +/* Internal helper that returns 1 if the reply was a RESP3 PUSH + * message and we handled it with a user-provided callback. */ +static int redisHandledPushReply(redisContext *c, void *reply) { + if (reply && c->push_cb && redisIsPushReply(reply)) { + c->push_cb(c->privdata, reply); + return 1; + } + + return 0; +} + int redisGetReply(redisContext *c, void **reply) { int wdone = 0; void *aux = NULL; @@ -887,13 +1032,23 @@ int redisGetReply(redisContext *c, void **reply) { do { if (redisBufferRead(c) == REDIS_ERR) return REDIS_ERR; - if (redisGetReplyFromReader(c,&aux) == REDIS_ERR) - return REDIS_ERR; + + /* We loop here in case the user has specified a RESP3 + * PUSH handler (e.g. for client tracking). */ + do { + if (redisGetReplyFromReader(c,&aux) == REDIS_ERR) + return REDIS_ERR; + } while (redisHandledPushReply(c, aux)); } while (aux == NULL); } - /* Set reply object */ - if (reply != NULL) *reply = aux; + /* Set reply or free it if we were passed NULL */ + if (reply != NULL) { + *reply = aux; + } else { + freeReplyObject(aux); + } + return REDIS_OK; } @@ -905,9 +1060,9 @@ int redisGetReply(redisContext *c, void **reply) { * the reply (or replies in pub/sub). */ int __redisAppendCommand(redisContext *c, const char *cmd, size_t len) { - sds newbuf; + hisds newbuf; - newbuf = sdscatlen(c->obuf,cmd,len); + newbuf = hi_sdscatlen(c->obuf,cmd,len); if (newbuf == NULL) { __redisSetError(c,REDIS_ERR_OOM,"Out of memory"); return REDIS_ERR; @@ -940,11 +1095,11 @@ int redisvAppendCommand(redisContext *c, const char *format, va_list ap) { } if (__redisAppendCommand(c,cmd,len) != REDIS_OK) { - free(cmd); + hi_free(cmd); return REDIS_ERR; } - free(cmd); + hi_free(cmd); return REDIS_OK; } @@ -959,7 +1114,7 @@ int redisAppendCommand(redisContext *c, const char *format, ...) { } int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) { - sds cmd; + hisds cmd; int len; len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen); @@ -969,11 +1124,11 @@ int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const s } if (__redisAppendCommand(c,cmd,len) != REDIS_OK) { - sdsfree(cmd); + hi_sdsfree(cmd); return REDIS_ERR; } - sdsfree(cmd); + hi_sdsfree(cmd); return REDIS_OK; } @@ -1007,9 +1162,8 @@ void *redisvCommand(redisContext *c, const char *format, va_list ap) { void *redisCommand(redisContext *c, const char *format, ...) { va_list ap; - void *reply = NULL; va_start(ap,format); - reply = redisvCommand(c,format,ap); + void *reply = redisvCommand(c,format,ap); va_end(ap); return reply; } diff --git a/redis.submodule/deps/hiredis/hiredis.h b/redis.submodule/deps/hiredis/hiredis.h index 423d5e5..b597394 100644 --- a/redis.submodule/deps/hiredis/hiredis.h +++ b/redis.submodule/deps/hiredis/hiredis.h @@ -35,14 +35,20 @@ #define __HIREDIS_H #include "read.h" #include /* for va_list */ +#ifndef _MSC_VER #include /* for struct timeval */ +#else +struct timeval; /* forward declaration */ +typedef long long ssize_t; +#endif #include /* uintXX_t, etc */ -#include "sds.h" /* for sds */ +#include "sds.h" /* for hisds */ +#include "alloc.h" /* for allocation wrappers */ -#define HIREDIS_MAJOR 0 -#define HIREDIS_MINOR 13 -#define HIREDIS_PATCH 3 -#define HIREDIS_SONAME 0.13 +#define HIREDIS_MAJOR 1 +#define HIREDIS_MINOR 0 +#define HIREDIS_PATCH 0 +#define HIREDIS_SONAME 1.0.0 /* Connection type can be blocking or non-blocking and is set in the * least significant bit of the flags field in redisContext. */ @@ -74,35 +80,26 @@ /* Flag that is set when we should set SO_REUSEADDR before calling bind() */ #define REDIS_REUSEADDR 0x80 +/** + * Flag that indicates the user does not want the context to + * be automatically freed upon error + */ +#define REDIS_NO_AUTO_FREE 0x200 + #define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */ /* number of times we retry to connect in the case of EADDRNOTAVAIL and * SO_REUSEADDR is being used. */ #define REDIS_CONNECT_RETRIES 10 -/* strerror_r has two completely different prototypes and behaviors - * depending on system issues, so we need to operate on the error buffer - * differently depending on which strerror_r we're using. */ -#ifndef _GNU_SOURCE -/* "regular" POSIX strerror_r that does the right thing. */ -#define __redis_strerror_r(errno, buf, len) \ - do { \ - strerror_r((errno), (buf), (len)); \ - } while (0) -#else -/* "bad" GNU strerror_r we need to clean up after. */ -#define __redis_strerror_r(errno, buf, len) \ - do { \ - char *err_str = strerror_r((errno), (buf), (len)); \ - /* If return value _isn't_ the start of the buffer we passed in, \ - * then GNU strerror_r returned an internal static buffer and we \ - * need to copy the result into our private buffer. */ \ - if (err_str != (buf)) { \ - strncpy((buf), err_str, ((len) - 1)); \ - buf[(len)-1] = '\0'; \ - } \ - } while (0) -#endif +/* Forward declarations for structs defined elsewhere */ +struct redisAsyncContext; +struct redisContext; + +/* RESP3 push helpers and callback prototypes */ +#define redisIsPushReply(r) (((redisReply*)(r))->type == REDIS_REPLY_PUSH) +typedef void (redisPushFn)(void *, void *); +typedef void (redisAsyncPushFn)(struct redisAsyncContext *, void *); #ifdef __cplusplus extern "C" { @@ -112,8 +109,12 @@ extern "C" { typedef struct redisReply { int type; /* REDIS_REPLY_* */ long long integer; /* The integer when type is REDIS_REPLY_INTEGER */ + double dval; /* The double when type is REDIS_REPLY_DOUBLE */ size_t len; /* Length of string */ - char *str; /* Used for both REDIS_REPLY_ERROR and REDIS_REPLY_STRING */ + char *str; /* Used for REDIS_REPLY_ERROR, REDIS_REPLY_STRING + REDIS_REPLY_VERB, and REDIS_REPLY_DOUBLE (in additional to dval). */ + char vtype[4]; /* Used for REDIS_REPLY_VERB, contains the null + terminated 3 character content type, such as "txt". */ size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */ struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */ } redisReply; @@ -127,26 +128,121 @@ void freeReplyObject(void *reply); int redisvFormatCommand(char **target, const char *format, va_list ap); int redisFormatCommand(char **target, const char *format, ...); int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen); -int redisFormatSdsCommandArgv(sds *target, int argc, const char ** argv, const size_t *argvlen); +int redisFormatSdsCommandArgv(hisds *target, int argc, const char ** argv, const size_t *argvlen); void redisFreeCommand(char *cmd); -void redisFreeSdsCommand(sds cmd); +void redisFreeSdsCommand(hisds cmd); enum redisConnectionType { REDIS_CONN_TCP, - REDIS_CONN_UNIX + REDIS_CONN_UNIX, + REDIS_CONN_USERFD }; +struct redisSsl; + +#define REDIS_OPT_NONBLOCK 0x01 +#define REDIS_OPT_REUSEADDR 0x02 + +/** + * Don't automatically free the async object on a connection failure, + * or other implicit conditions. Only free on an explicit call to disconnect() or free() + */ +#define REDIS_OPT_NOAUTOFREE 0x04 + +/* Don't automatically intercept and free RESP3 PUSH replies. */ +#define REDIS_OPT_NO_PUSH_AUTOFREE 0x08 + +/* In Unix systems a file descriptor is a regular signed int, with -1 + * representing an invalid descriptor. In Windows it is a SOCKET + * (32- or 64-bit unsigned integer depending on the architecture), where + * all bits set (~0) is INVALID_SOCKET. */ +#ifndef _WIN32 +typedef int redisFD; +#define REDIS_INVALID_FD -1 +#else +#ifdef _WIN64 +typedef unsigned long long redisFD; /* SOCKET = 64-bit UINT_PTR */ +#else +typedef unsigned long redisFD; /* SOCKET = 32-bit UINT_PTR */ +#endif +#define REDIS_INVALID_FD ((redisFD)(~0)) /* INVALID_SOCKET */ +#endif + +typedef struct { + /* + * the type of connection to use. This also indicates which + * `endpoint` member field to use + */ + int type; + /* bit field of REDIS_OPT_xxx */ + int options; + /* timeout value for connect operation. If NULL, no timeout is used */ + const struct timeval *connect_timeout; + /* timeout value for commands. If NULL, no timeout is used. This can be + * updated at runtime with redisSetTimeout/redisAsyncSetTimeout. */ + const struct timeval *command_timeout; + union { + /** use this field for tcp/ip connections */ + struct { + const char *source_addr; + const char *ip; + int port; + } tcp; + /** use this field for unix domain sockets */ + const char *unix_socket; + /** + * use this field to have hiredis operate an already-open + * file descriptor */ + redisFD fd; + } endpoint; + + /* Optional user defined data/destructor */ + void *privdata; + void (*free_privdata)(void *); + + /* A user defined PUSH message callback */ + redisPushFn *push_cb; + redisAsyncPushFn *async_push_cb; +} redisOptions; + +/** + * Helper macros to initialize options to their specified fields. + */ +#define REDIS_OPTIONS_SET_TCP(opts, ip_, port_) \ + (opts)->type = REDIS_CONN_TCP; \ + (opts)->endpoint.tcp.ip = ip_; \ + (opts)->endpoint.tcp.port = port_; + +#define REDIS_OPTIONS_SET_UNIX(opts, path) \ + (opts)->type = REDIS_CONN_UNIX; \ + (opts)->endpoint.unix_socket = path; + +#define REDIS_OPTIONS_SET_PRIVDATA(opts, data, dtor) \ + (opts)->privdata = data; \ + (opts)->free_privdata = dtor; \ + +typedef struct redisContextFuncs { + void (*free_privctx)(void *); + void (*async_read)(struct redisAsyncContext *); + void (*async_write)(struct redisAsyncContext *); + ssize_t (*read)(struct redisContext *, char *, size_t); + ssize_t (*write)(struct redisContext *); +} redisContextFuncs; + /* Context for a connection to Redis */ typedef struct redisContext { + const redisContextFuncs *funcs; /* Function table */ + int err; /* Error flags, 0 when there is no error */ char errstr[128]; /* String representation of error when applicable */ - int fd; + redisFD fd; int flags; char *obuf; /* Write buffer */ redisReader *reader; /* Protocol reader */ enum redisConnectionType connection_type; - struct timeval *timeout; + struct timeval *connect_timeout; + struct timeval *command_timeout; struct { char *host; @@ -158,8 +254,24 @@ typedef struct redisContext { char *path; } unix_sock; + /* For non-blocking connect */ + struct sockadr *saddr; + size_t addrlen; + + /* Optional data and corresponding destructor users can use to provide + * context to a given redisContext. Not used by hiredis. */ + void *privdata; + void (*free_privdata)(void *); + + /* Internal context pointer presently used by hiredis to manage + * SSL connections. */ + void *privctx; + + /* An optional RESP3 PUSH handler */ + redisPushFn *push_cb; } redisContext; +redisContext *redisConnectWithOptions(const redisOptions *options); redisContext *redisConnect(const char *ip, int port); redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv); redisContext *redisConnectNonBlock(const char *ip, int port); @@ -170,7 +282,7 @@ redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, redisContext *redisConnectUnix(const char *path); redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv); redisContext *redisConnectUnixNonBlock(const char *path); -redisContext *redisConnectFd(int fd); +redisContext *redisConnectFd(redisFD fd); /** * Reconnect the given context using the saved information. @@ -183,10 +295,11 @@ redisContext *redisConnectFd(int fd); */ int redisReconnect(redisContext *c); +redisPushFn *redisSetPushCallback(redisContext *c, redisPushFn *fn); int redisSetTimeout(redisContext *c, const struct timeval tv); int redisEnableKeepAlive(redisContext *c); void redisFree(redisContext *c); -int redisFreeKeepFd(redisContext *c); +redisFD redisFreeKeepFd(redisContext *c); int redisBufferRead(redisContext *c); int redisBufferWrite(redisContext *c, int *done); diff --git a/redis.submodule/deps/hiredis/hiredis.pc.in b/redis.submodule/deps/hiredis/hiredis.pc.in new file mode 100644 index 0000000..91b7731 --- /dev/null +++ b/redis.submodule/deps/hiredis/hiredis.pc.in @@ -0,0 +1,12 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +install_libdir=@CMAKE_INSTALL_LIBDIR@ +exec_prefix=${prefix} +libdir=${exec_prefix}/${install_libdir} +includedir=${prefix}/include +pkgincludedir=${includedir}/hiredis + +Name: hiredis +Description: Minimalistic C client library for Redis. +Version: @PROJECT_VERSION@ +Libs: -L${libdir} -lhiredis +Cflags: -I${pkgincludedir} -D_FILE_OFFSET_BITS=64 diff --git a/redis.submodule/deps/hiredis/hiredis_ssl-config.cmake.in b/redis.submodule/deps/hiredis/hiredis_ssl-config.cmake.in new file mode 100644 index 0000000..9a283df --- /dev/null +++ b/redis.submodule/deps/hiredis/hiredis_ssl-config.cmake.in @@ -0,0 +1,13 @@ +@PACKAGE_INIT@ + +set_and_check(hiredis_ssl_INCLUDEDIR "@PACKAGE_INCLUDE_INSTALL_DIR@") + +IF (NOT TARGET hiredis::hiredis_ssl) + INCLUDE(${CMAKE_CURRENT_LIST_DIR}/hiredis_ssl-targets.cmake) +ENDIF() + +SET(hiredis_ssl_LIBRARIES hiredis::hiredis_ssl) +SET(hiredis_ssl_INCLUDE_DIRS ${hiredis_ssl_INCLUDEDIR}) + +check_required_components(hiredis_ssl) + diff --git a/redis.submodule/deps/hiredis/hiredis_ssl.h b/redis.submodule/deps/hiredis/hiredis_ssl.h new file mode 100644 index 0000000..604efe0 --- /dev/null +++ b/redis.submodule/deps/hiredis/hiredis_ssl.h @@ -0,0 +1,127 @@ + +/* + * Copyright (c) 2019, Redis Labs + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_SSL_H +#define __HIREDIS_SSL_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* This is the underlying struct for SSL in ssl.h, which is not included to + * keep build dependencies short here. + */ +struct ssl_st; + +/* A wrapper around OpenSSL SSL_CTX to allow easy SSL use without directly + * calling OpenSSL. + */ +typedef struct redisSSLContext redisSSLContext; + +/** + * Initialization errors that redisCreateSSLContext() may return. + */ + +typedef enum { + REDIS_SSL_CTX_NONE = 0, /* No Error */ + REDIS_SSL_CTX_CREATE_FAILED, /* Failed to create OpenSSL SSL_CTX */ + REDIS_SSL_CTX_CERT_KEY_REQUIRED, /* Client cert and key must both be specified or skipped */ + REDIS_SSL_CTX_CA_CERT_LOAD_FAILED, /* Failed to load CA Certificate or CA Path */ + REDIS_SSL_CTX_CLIENT_CERT_LOAD_FAILED, /* Failed to load client certificate */ + REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED /* Failed to load private key */ +} redisSSLContextError; + +/** + * Return the error message corresponding with the specified error code. + */ + +const char *redisSSLContextGetError(redisSSLContextError error); + +/** + * Helper function to initialize the OpenSSL library. + * + * OpenSSL requires one-time initialization before it can be used. Callers should + * call this function only once, and only if OpenSSL is not directly initialized + * elsewhere. + */ +int redisInitOpenSSL(void); + +/** + * Helper function to initialize an OpenSSL context that can be used + * to initiate SSL connections. + * + * cacert_filename is an optional name of a CA certificate/bundle file to load + * and use for validation. + * + * capath is an optional directory path where trusted CA certificate files are + * stored in an OpenSSL-compatible structure. + * + * cert_filename and private_key_filename are optional names of a client side + * certificate and private key files to use for authentication. They need to + * be both specified or omitted. + * + * server_name is an optional and will be used as a server name indication + * (SNI) TLS extension. + * + * If error is non-null, it will be populated in case the context creation fails + * (returning a NULL). + */ + +redisSSLContext *redisCreateSSLContext(const char *cacert_filename, const char *capath, + const char *cert_filename, const char *private_key_filename, + const char *server_name, redisSSLContextError *error); + +/** + * Free a previously created OpenSSL context. + */ +void redisFreeSSLContext(redisSSLContext *redis_ssl_ctx); + +/** + * Initiate SSL on an existing redisContext. + * + * This is similar to redisInitiateSSL() but does not require the caller + * to directly interact with OpenSSL, and instead uses a redisSSLContext + * previously created using redisCreateSSLContext(). + */ + +int redisInitiateSSLWithContext(redisContext *c, redisSSLContext *redis_ssl_ctx); + +/** + * Initiate SSL/TLS negotiation on a provided OpenSSL SSL object. + */ + +int redisInitiateSSL(redisContext *c, struct ssl_st *ssl); + +#ifdef __cplusplus +} +#endif + +#endif /* __HIREDIS_SSL_H */ diff --git a/redis.submodule/deps/hiredis/hiredis_ssl.pc.in b/redis.submodule/deps/hiredis/hiredis_ssl.pc.in new file mode 100644 index 0000000..588a978 --- /dev/null +++ b/redis.submodule/deps/hiredis/hiredis_ssl.pc.in @@ -0,0 +1,12 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include +pkgincludedir=${includedir}/hiredis + +Name: hiredis_ssl +Description: SSL Support for hiredis. +Version: @PROJECT_VERSION@ +Requires: hiredis +Libs: -L${libdir} -lhiredis_ssl +Libs.private: -lssl -lcrypto diff --git a/redis.submodule/deps/hiredis/net.c b/redis.submodule/deps/hiredis/net.c index 7d41209..88f9aff 100644 --- a/redis.submodule/deps/hiredis/net.c +++ b/redis.submodule/deps/hiredis/net.c @@ -34,43 +34,72 @@ #include "fmacros.h" #include -#include -#include -#include -#include -#include -#include -#include #include #include -#include #include #include #include -#include #include #include #include "net.h" #include "sds.h" +#include "sockcompat.h" +#include "win32.h" /* Defined in hiredis.c */ void __redisSetError(redisContext *c, int type, const char *str); -static void redisContextCloseFd(redisContext *c) { - if (c && c->fd >= 0) { +void redisNetClose(redisContext *c) { + if (c && c->fd != REDIS_INVALID_FD) { close(c->fd); - c->fd = -1; + c->fd = REDIS_INVALID_FD; } } +ssize_t redisNetRead(redisContext *c, char *buf, size_t bufcap) { + ssize_t nread = recv(c->fd, buf, bufcap, 0); + if (nread == -1) { + if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { + /* Try again later */ + return 0; + } else if(errno == ETIMEDOUT && (c->flags & REDIS_BLOCK)) { + /* especially in windows */ + __redisSetError(c, REDIS_ERR_TIMEOUT, "recv timeout"); + return -1; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } else if (nread == 0) { + __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection"); + return -1; + } else { + return nread; + } +} + +ssize_t redisNetWrite(redisContext *c) { + ssize_t nwritten = send(c->fd, c->obuf, hi_sdslen(c->obuf), 0); + if (nwritten < 0) { + if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { + /* Try again later */ + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } + return nwritten; +} + static void __redisSetErrorFromErrno(redisContext *c, int type, const char *prefix) { + int errorno = errno; /* snprintf() may change errno */ char buf[128] = { 0 }; size_t len = 0; if (prefix != NULL) len = snprintf(buf,sizeof(buf),"%s: ",prefix); - __redis_strerror_r(errno, (char *)(buf + len), sizeof(buf) - len); + strerror_r(errorno, (char *)(buf + len), sizeof(buf) - len); __redisSetError(c,type,buf); } @@ -78,15 +107,15 @@ static int redisSetReuseAddr(redisContext *c) { int on = 1; if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } return REDIS_OK; } static int redisCreateSocket(redisContext *c, int type) { - int s; - if ((s = socket(type, SOCK_STREAM, 0)) == -1) { + redisFD s; + if ((s = socket(type, SOCK_STREAM, 0)) == REDIS_INVALID_FD) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); return REDIS_ERR; } @@ -100,6 +129,7 @@ static int redisCreateSocket(redisContext *c, int type) { } static int redisSetBlocking(redisContext *c, int blocking) { +#ifndef _WIN32 int flags; /* Set the socket nonblocking. @@ -107,7 +137,7 @@ static int redisSetBlocking(redisContext *c, int blocking) { * interrupted by a signal. */ if ((flags = fcntl(c->fd, F_GETFL)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_GETFL)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -118,15 +148,23 @@ static int redisSetBlocking(redisContext *c, int blocking) { if (fcntl(c->fd, F_SETFL, flags) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_SETFL)"); - redisContextCloseFd(c); + redisNetClose(c); + return REDIS_ERR; + } +#else + u_long mode = blocking ? 0 : 1; + if (ioctl(c->fd, FIONBIO, &mode) == -1) { + __redisSetErrorFromErrno(c, REDIS_ERR_IO, "ioctl(FIONBIO)"); + redisNetClose(c); return REDIS_ERR; } +#endif /* _WIN32 */ return REDIS_OK; } int redisKeepAlive(redisContext *c, int interval) { int val = 1; - int fd = c->fd; + redisFD fd = c->fd; if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1){ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); @@ -135,14 +173,13 @@ int redisKeepAlive(redisContext *c, int interval) { val = interval; -#ifdef _OSX +#if defined(__APPLE__) && defined(__MACH__) if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE, &val, sizeof(val)) < 0) { __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); return REDIS_ERR; } #else #if defined(__GLIBC__) && !defined(__FreeBSD_kernel__) - val = interval; if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) { __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); return REDIS_ERR; @@ -166,11 +203,11 @@ int redisKeepAlive(redisContext *c, int interval) { return REDIS_OK; } -static int redisSetTcpNoDelay(redisContext *c) { +int redisSetTcpNoDelay(redisContext *c) { int yes = 1; if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } return REDIS_OK; @@ -180,7 +217,7 @@ static int redisSetTcpNoDelay(redisContext *c) { static int redisContextTimeoutMsec(redisContext *c, long *result) { - const struct timeval *timeout = c->timeout; + const struct timeval *timeout = c->connect_timeout; long msec = -1; /* Only use timeout when not NULL. */ @@ -212,28 +249,50 @@ static int redisContextWaitReady(redisContext *c, long msec) { if ((res = poll(wfd, 1, msec)) == -1) { __redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } else if (res == 0) { errno = ETIMEDOUT; __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } - if (redisCheckSocketError(c) != REDIS_OK) + if (redisCheckConnectDone(c, &res) != REDIS_OK || res == 0) { + redisCheckSocketError(c); return REDIS_ERR; + } return REDIS_OK; } __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } +int redisCheckConnectDone(redisContext *c, int *completed) { + int rc = connect(c->fd, (const struct sockaddr *)c->saddr, c->addrlen); + if (rc == 0) { + *completed = 1; + return REDIS_OK; + } + switch (errno) { + case EISCONN: + *completed = 1; + return REDIS_OK; + case EALREADY: + case EINPROGRESS: + case EWOULDBLOCK: + *completed = 0; + return REDIS_OK; + default: + return REDIS_ERR; + } +} + int redisCheckSocketError(redisContext *c) { - int err = 0; + int err = 0, errno_saved = errno; socklen_t errlen = sizeof(err); if (getsockopt(c->fd, SOL_SOCKET, SO_ERROR, &err, &errlen) == -1) { @@ -241,6 +300,10 @@ int redisCheckSocketError(redisContext *c) { return REDIS_ERR; } + if (err == 0) { + err = errno_saved; + } + if (err) { errno = err; __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); @@ -251,21 +314,57 @@ int redisCheckSocketError(redisContext *c) { } int redisContextSetTimeout(redisContext *c, const struct timeval tv) { - if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,&tv,sizeof(tv)) == -1) { + const void *to_ptr = &tv; + size_t to_sz = sizeof(tv); + + if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,to_ptr,to_sz) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)"); return REDIS_ERR; } - if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,&tv,sizeof(tv)) == -1) { + if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,to_ptr,to_sz) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_SNDTIMEO)"); return REDIS_ERR; } return REDIS_OK; } +int redisContextUpdateConnectTimeout(redisContext *c, const struct timeval *timeout) { + /* Same timeval struct, short circuit */ + if (c->connect_timeout == timeout) + return REDIS_OK; + + /* Allocate context timeval if we need to */ + if (c->connect_timeout == NULL) { + c->connect_timeout = hi_malloc(sizeof(*c->connect_timeout)); + if (c->connect_timeout == NULL) + return REDIS_ERR; + } + + memcpy(c->connect_timeout, timeout, sizeof(*c->connect_timeout)); + return REDIS_OK; +} + +int redisContextUpdateCommandTimeout(redisContext *c, const struct timeval *timeout) { + /* Same timeval struct, short circuit */ + if (c->command_timeout == timeout) + return REDIS_OK; + + /* Allocate context timeval if we need to */ + if (c->command_timeout == NULL) { + c->command_timeout = hi_malloc(sizeof(*c->command_timeout)); + if (c->command_timeout == NULL) + return REDIS_ERR; + } + + memcpy(c->command_timeout, timeout, sizeof(*c->command_timeout)); + return REDIS_OK; +} + static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout, const char *source_addr) { - int s, rv, n; + redisFD s; + int rv, n; char _port[6]; /* strlen("65535"); */ struct addrinfo hints, *servinfo, *bservinfo, *p, *b; int blocking = (c->flags & REDIS_BLOCK); @@ -285,23 +384,19 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, * This is a bit ugly, but atleast it works and doesn't leak memory. **/ if (c->tcp.host != addr) { - if (c->tcp.host) - free(c->tcp.host); + hi_free(c->tcp.host); - c->tcp.host = strdup(addr); + c->tcp.host = hi_strdup(addr); + if (c->tcp.host == NULL) + goto oom; } if (timeout) { - if (c->timeout != timeout) { - if (c->timeout == NULL) - c->timeout = malloc(sizeof(struct timeval)); - - memcpy(c->timeout, timeout, sizeof(struct timeval)); - } + if (redisContextUpdateConnectTimeout(c, timeout) == REDIS_ERR) + goto oom; } else { - if (c->timeout) - free(c->timeout); - c->timeout = NULL; + hi_free(c->connect_timeout); + c->connect_timeout = NULL; } if (redisContextTimeoutMsec(c, &timeout_msec) != REDIS_OK) { @@ -310,11 +405,11 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, } if (source_addr == NULL) { - free(c->tcp.source_addr); + hi_free(c->tcp.source_addr); c->tcp.source_addr = NULL; } else if (c->tcp.source_addr != source_addr) { - free(c->tcp.source_addr); - c->tcp.source_addr = strdup(source_addr); + hi_free(c->tcp.source_addr); + c->tcp.source_addr = hi_strdup(source_addr); } snprintf(_port, 6, "%d", port); @@ -336,7 +431,7 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, } for (p = servinfo; p != NULL; p = p->ai_next) { addrretry: - if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) + if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == REDIS_INVALID_FD) continue; c->fd = s; @@ -356,6 +451,7 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, n = 1; if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char*) &n, sizeof(n)) < 0) { + freeaddrinfo(bservinfo); goto error; } } @@ -374,28 +470,45 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, goto error; } } + + /* For repeat connection */ + hi_free(c->saddr); + c->saddr = hi_malloc(p->ai_addrlen); + if (c->saddr == NULL) + goto oom; + + memcpy(c->saddr, p->ai_addr, p->ai_addrlen); + c->addrlen = p->ai_addrlen; + if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { if (errno == EHOSTUNREACH) { - redisContextCloseFd(c); + redisNetClose(c); continue; - } else if (errno == EINPROGRESS && !blocking) { - /* This is ok. */ + } else if (errno == EINPROGRESS) { + if (blocking) { + goto wait_for_ready; + } + /* This is ok. + * Note that even when it's in blocking mode, we unset blocking + * for `connect()` + */ } else if (errno == EADDRNOTAVAIL && reuseaddr) { if (++reuses >= REDIS_CONNECT_RETRIES) { goto error; } else { - redisContextCloseFd(c); + redisNetClose(c); goto addrretry; } } else { + wait_for_ready: if (redisContextWaitReady(c,timeout_msec) != REDIS_OK) goto error; + if (redisSetTcpNoDelay(c) != REDIS_OK) + goto error; } } if (blocking && redisSetBlocking(c,1) != REDIS_OK) goto error; - if (redisSetTcpNoDelay(c) != REDIS_OK) - goto error; c->flags |= REDIS_CONNECTED; rv = REDIS_OK; @@ -408,10 +521,15 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, goto error; } +oom: + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); error: rv = REDIS_ERR; end: - freeaddrinfo(servinfo); + if(servinfo) { + freeaddrinfo(servinfo); + } + return rv; // Need to return REDIS_OK if alright } @@ -427,38 +545,47 @@ int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, } int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) { +#ifndef _WIN32 int blocking = (c->flags & REDIS_BLOCK); - struct sockaddr_un sa; + struct sockaddr_un *sa; long timeout_msec = -1; - if (redisCreateSocket(c,AF_LOCAL) < 0) + if (redisCreateSocket(c,AF_UNIX) < 0) return REDIS_ERR; if (redisSetBlocking(c,0) != REDIS_OK) return REDIS_ERR; c->connection_type = REDIS_CONN_UNIX; - if (c->unix_sock.path != path) - c->unix_sock.path = strdup(path); + if (c->unix_sock.path != path) { + hi_free(c->unix_sock.path); - if (timeout) { - if (c->timeout != timeout) { - if (c->timeout == NULL) - c->timeout = malloc(sizeof(struct timeval)); + c->unix_sock.path = hi_strdup(path); + if (c->unix_sock.path == NULL) + goto oom; + } - memcpy(c->timeout, timeout, sizeof(struct timeval)); - } + if (timeout) { + if (redisContextUpdateConnectTimeout(c, timeout) == REDIS_ERR) + goto oom; } else { - if (c->timeout) - free(c->timeout); - c->timeout = NULL; + hi_free(c->connect_timeout); + c->connect_timeout = NULL; } if (redisContextTimeoutMsec(c,&timeout_msec) != REDIS_OK) return REDIS_ERR; - sa.sun_family = AF_LOCAL; - strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1); - if (connect(c->fd, (struct sockaddr*)&sa, sizeof(sa)) == -1) { + /* Don't leak sockaddr if we're reconnecting */ + if (c->saddr) hi_free(c->saddr); + + sa = (struct sockaddr_un*)(c->saddr = hi_malloc(sizeof(struct sockaddr_un))); + if (sa == NULL) + goto oom; + + c->addrlen = sizeof(struct sockaddr_un); + sa->sun_family = AF_UNIX; + strncpy(sa->sun_path, path, sizeof(sa->sun_path) - 1); + if (connect(c->fd, (struct sockaddr*)sa, sizeof(*sa)) == -1) { if (errno == EINPROGRESS && !blocking) { /* This is ok. */ } else { @@ -473,4 +600,13 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time c->flags |= REDIS_CONNECTED; return REDIS_OK; +#else + /* We currently do not support Unix sockets for Windows. */ + /* TODO(m): https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/ */ + errno = EPROTONOSUPPORT; + return REDIS_ERR; +#endif /* _WIN32 */ +oom: + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; } diff --git a/redis.submodule/deps/hiredis/net.h b/redis.submodule/deps/hiredis/net.h index 2f1a0bf..9f43283 100644 --- a/redis.submodule/deps/hiredis/net.h +++ b/redis.submodule/deps/hiredis/net.h @@ -37,9 +37,9 @@ #include "hiredis.h" -#if defined(__sun) -#define AF_LOCAL AF_UNIX -#endif +void redisNetClose(redisContext *c); +ssize_t redisNetRead(redisContext *c, char *buf, size_t bufcap); +ssize_t redisNetWrite(redisContext *c); int redisCheckSocketError(redisContext *c); int redisContextSetTimeout(redisContext *c, const struct timeval tv); @@ -49,5 +49,8 @@ int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, const char *source_addr); int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout); int redisKeepAlive(redisContext *c, int interval); +int redisCheckConnectDone(redisContext *c, int *completed); + +int redisSetTcpNoDelay(redisContext *c); #endif diff --git a/redis.submodule/deps/hiredis/read.c b/redis.submodule/deps/hiredis/read.c index 50333b5..682b9a6 100644 --- a/redis.submodule/deps/hiredis/read.c +++ b/redis.submodule/deps/hiredis/read.c @@ -29,19 +29,26 @@ * POSSIBILITY OF SUCH DAMAGE. */ - #include "fmacros.h" #include #include #ifndef _MSC_VER #include +#include #endif #include #include #include +#include +#include +#include "alloc.h" #include "read.h" #include "sds.h" +#include "win32.h" + +/* Initial size of our nested reply stack and how much we grow it when needd */ +#define REDIS_READER_STACK_SIZE 9 static void __redisReaderSetError(redisReader *r, int type, const char *str) { size_t len; @@ -52,11 +59,9 @@ static void __redisReaderSetError(redisReader *r, int type, const char *str) { } /* Clear input buffer on errors. */ - if (r->buf != NULL) { - sdsfree(r->buf); - r->buf = NULL; - r->pos = r->len = 0; - } + hi_sdsfree(r->buf); + r->buf = NULL; + r->pos = r->len = 0; /* Reset task stack. */ r->ridx = -1; @@ -143,33 +148,79 @@ static char *seekNewline(char *s, size_t len) { return NULL; } -/* Read a long long value starting at *s, under the assumption that it will be - * terminated by \r\n. Ambiguously returns -1 for unexpected input. */ -static long long readLongLong(char *s) { - long long v = 0; - int dec, mult = 1; - char c; - - if (*s == '-') { - mult = -1; - s++; - } else if (*s == '+') { - mult = 1; - s++; +/* Convert a string into a long long. Returns REDIS_OK if the string could be + * parsed into a (non-overflowing) long long, REDIS_ERR otherwise. The value + * will be set to the parsed value when appropriate. + * + * Note that this function demands that the string strictly represents + * a long long: no spaces or other characters before or after the string + * representing the number are accepted, nor zeroes at the start if not + * for the string "0" representing the zero number. + * + * Because of its strictness, it is safe to use this function to check if + * you can convert a string into a long long, and obtain back the string + * from the number without any loss in the string representation. */ +static int string2ll(const char *s, size_t slen, long long *value) { + const char *p = s; + size_t plen = 0; + int negative = 0; + unsigned long long v; + + if (plen == slen) + return REDIS_ERR; + + /* Special case: first and only digit is 0. */ + if (slen == 1 && p[0] == '0') { + if (value != NULL) *value = 0; + return REDIS_OK; } - while ((c = *(s++)) != '\r') { - dec = c - '0'; - if (dec >= 0 && dec < 10) { - v *= 10; - v += dec; - } else { - /* Should not happen... */ - return -1; - } + if (p[0] == '-') { + negative = 1; + p++; plen++; + + /* Abort on only a negative sign. */ + if (plen == slen) + return REDIS_ERR; } - return mult*v; + /* First digit should be 1-9, otherwise the string should just be 0. */ + if (p[0] >= '1' && p[0] <= '9') { + v = p[0]-'0'; + p++; plen++; + } else if (p[0] == '0' && slen == 1) { + *value = 0; + return REDIS_OK; + } else { + return REDIS_ERR; + } + + while (plen < slen && p[0] >= '0' && p[0] <= '9') { + if (v > (ULLONG_MAX / 10)) /* Overflow. */ + return REDIS_ERR; + v *= 10; + + if (v > (ULLONG_MAX - (p[0]-'0'))) /* Overflow. */ + return REDIS_ERR; + v += p[0]-'0'; + + p++; plen++; + } + + /* Return if not all bytes were used. */ + if (plen < slen) + return REDIS_ERR; + + if (negative) { + if (v > ((unsigned long long)(-(LLONG_MIN+1))+1)) /* Overflow. */ + return REDIS_ERR; + if (value != NULL) *value = -v; + } else { + if (v > LLONG_MAX) /* Overflow. */ + return REDIS_ERR; + if (value != NULL) *value = v; + } + return REDIS_OK; } static char *readLine(redisReader *r, int *_len) { @@ -196,9 +247,12 @@ static void moveToNextTask(redisReader *r) { return; } - cur = &(r->rstack[r->ridx]); - prv = &(r->rstack[r->ridx-1]); - assert(prv->type == REDIS_REPLY_ARRAY); + cur = r->task[r->ridx]; + prv = r->task[r->ridx-1]; + assert(prv->type == REDIS_REPLY_ARRAY || + prv->type == REDIS_REPLY_MAP || + prv->type == REDIS_REPLY_SET || + prv->type == REDIS_REPLY_PUSH); if (cur->idx == prv->elements-1) { r->ridx--; } else { @@ -213,17 +267,65 @@ static void moveToNextTask(redisReader *r) { } static int processLineItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); + redisReadTask *cur = r->task[r->ridx]; void *obj; char *p; int len; if ((p = readLine(r,&len)) != NULL) { if (cur->type == REDIS_REPLY_INTEGER) { - if (r->fn && r->fn->createInteger) - obj = r->fn->createInteger(cur,readLongLong(p)); - else + if (r->fn && r->fn->createInteger) { + long long v; + if (string2ll(p, len, &v) == REDIS_ERR) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Bad integer value"); + return REDIS_ERR; + } + obj = r->fn->createInteger(cur,v); + } else { obj = (void*)REDIS_REPLY_INTEGER; + } + } else if (cur->type == REDIS_REPLY_DOUBLE) { + if (r->fn && r->fn->createDouble) { + char buf[326], *eptr; + double d; + + if ((size_t)len >= sizeof(buf)) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Double value is too large"); + return REDIS_ERR; + } + + memcpy(buf,p,len); + buf[len] = '\0'; + + if (strcasecmp(buf,",inf") == 0) { + d = INFINITY; /* Positive infinite. */ + } else if (strcasecmp(buf,",-inf") == 0) { + d = -INFINITY; /* Negative infinite. */ + } else { + d = strtod((char*)buf,&eptr); + if (buf[0] == '\0' || eptr[0] != '\0' || isnan(d)) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Bad double value"); + return REDIS_ERR; + } + } + obj = r->fn->createDouble(cur,d,buf,len); + } else { + obj = (void*)REDIS_REPLY_DOUBLE; + } + } else if (cur->type == REDIS_REPLY_NIL) { + if (r->fn && r->fn->createNil) + obj = r->fn->createNil(cur); + else + obj = (void*)REDIS_REPLY_NIL; + } else if (cur->type == REDIS_REPLY_BOOL) { + int bval = p[0] == 't' || p[0] == 'T'; + if (r->fn && r->fn->createBool) + obj = r->fn->createBool(cur,bval); + else + obj = (void*)REDIS_REPLY_BOOL; } else { /* Type will be error or status. */ if (r->fn && r->fn->createString) @@ -247,10 +349,10 @@ static int processLineItem(redisReader *r) { } static int processBulkItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); + redisReadTask *cur = r->task[r->ridx]; void *obj = NULL; char *p, *s; - long len; + long long len; unsigned long bytelen; int success = 0; @@ -259,9 +361,20 @@ static int processBulkItem(redisReader *r) { if (s != NULL) { p = r->buf+r->pos; bytelen = s-(r->buf+r->pos)+2; /* include \r\n */ - len = readLongLong(p); - if (len < 0) { + if (string2ll(p, bytelen - 2, &len) == REDIS_ERR) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Bad bulk string length"); + return REDIS_ERR; + } + + if (len < -1 || (LLONG_MAX > SIZE_MAX && len > (long long)SIZE_MAX)) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Bulk string length out of range"); + return REDIS_ERR; + } + + if (len == -1) { /* The nil object can always be created. */ if (r->fn && r->fn->createNil) obj = r->fn->createNil(cur); @@ -272,10 +385,18 @@ static int processBulkItem(redisReader *r) { /* Only continue when the buffer contains the entire bulk item. */ bytelen += len+2; /* include \r\n */ if (r->pos+bytelen <= r->len) { + if ((cur->type == REDIS_REPLY_VERB && len < 4) || + (cur->type == REDIS_REPLY_VERB && s[5] != ':')) + { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Verbatim string 4 bytes of content type are " + "missing or incorrectly encoded."); + return REDIS_ERR; + } if (r->fn && r->fn->createString) obj = r->fn->createString(cur,s+2,len); else - obj = (void*)REDIS_REPLY_STRING; + obj = (void*)(long)cur->type; success = 1; } } @@ -299,24 +420,62 @@ static int processBulkItem(redisReader *r) { return REDIS_ERR; } -static int processMultiBulkItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); +static int redisReaderGrow(redisReader *r) { + redisReadTask **aux; + int newlen; + + /* Grow our stack size */ + newlen = r->tasks + REDIS_READER_STACK_SIZE; + aux = hi_realloc(r->task, sizeof(*r->task) * newlen); + if (aux == NULL) + goto oom; + + r->task = aux; + + /* Allocate new tasks */ + for (; r->tasks < newlen; r->tasks++) { + r->task[r->tasks] = hi_calloc(1, sizeof(**r->task)); + if (r->task[r->tasks] == NULL) + goto oom; + } + + return REDIS_OK; +oom: + __redisReaderSetErrorOOM(r); + return REDIS_ERR; +} + +/* Process the array, map and set types. */ +static int processAggregateItem(redisReader *r) { + redisReadTask *cur = r->task[r->ridx]; void *obj; char *p; - long elements; - int root = 0; + long long elements; + int root = 0, len; /* Set error for nested multi bulks with depth > 7 */ - if (r->ridx == 8) { - __redisReaderSetError(r,REDIS_ERR_PROTOCOL, - "No support for nested multi bulk replies with depth > 7"); - return REDIS_ERR; + if (r->ridx == r->tasks - 1) { + if (redisReaderGrow(r) == REDIS_ERR) + return REDIS_ERR; } - if ((p = readLine(r,NULL)) != NULL) { - elements = readLongLong(p); + if ((p = readLine(r,&len)) != NULL) { + if (string2ll(p, len, &elements) == REDIS_ERR) { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Bad multi-bulk length"); + return REDIS_ERR; + } + root = (r->ridx == 0); + if (elements < -1 || (LLONG_MAX > SIZE_MAX && elements > SIZE_MAX) || + (r->maxelements > 0 && elements > r->maxelements)) + { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Multi-bulk length out of range"); + return REDIS_ERR; + } + if (elements == -1) { if (r->fn && r->fn->createNil) obj = r->fn->createNil(cur); @@ -330,10 +489,12 @@ static int processMultiBulkItem(redisReader *r) { moveToNextTask(r); } else { + if (cur->type == REDIS_REPLY_MAP) elements *= 2; + if (r->fn && r->fn->createArray) obj = r->fn->createArray(cur,elements); else - obj = (void*)REDIS_REPLY_ARRAY; + obj = (void*)(long)cur->type; if (obj == NULL) { __redisReaderSetErrorOOM(r); @@ -345,12 +506,12 @@ static int processMultiBulkItem(redisReader *r) { cur->elements = elements; cur->obj = obj; r->ridx++; - r->rstack[r->ridx].type = -1; - r->rstack[r->ridx].elements = -1; - r->rstack[r->ridx].idx = 0; - r->rstack[r->ridx].obj = NULL; - r->rstack[r->ridx].parent = cur; - r->rstack[r->ridx].privdata = r->privdata; + r->task[r->ridx]->type = -1; + r->task[r->ridx]->elements = -1; + r->task[r->ridx]->idx = 0; + r->task[r->ridx]->obj = NULL; + r->task[r->ridx]->parent = cur; + r->task[r->ridx]->privdata = r->privdata; } else { moveToNextTask(r); } @@ -365,7 +526,7 @@ static int processMultiBulkItem(redisReader *r) { } static int processItem(redisReader *r) { - redisReadTask *cur = &(r->rstack[r->ridx]); + redisReadTask *cur = r->task[r->ridx]; char *p; /* check if we need to read type */ @@ -381,12 +542,33 @@ static int processItem(redisReader *r) { case ':': cur->type = REDIS_REPLY_INTEGER; break; + case ',': + cur->type = REDIS_REPLY_DOUBLE; + break; + case '_': + cur->type = REDIS_REPLY_NIL; + break; case '$': cur->type = REDIS_REPLY_STRING; break; case '*': cur->type = REDIS_REPLY_ARRAY; break; + case '%': + cur->type = REDIS_REPLY_MAP; + break; + case '~': + cur->type = REDIS_REPLY_SET; + break; + case '#': + cur->type = REDIS_REPLY_BOOL; + break; + case '=': + cur->type = REDIS_REPLY_VERB; + break; + case '>': + cur->type = REDIS_REPLY_PUSH; + break; default: __redisReaderSetErrorProtocolByte(r,*p); return REDIS_ERR; @@ -402,11 +584,18 @@ static int processItem(redisReader *r) { case REDIS_REPLY_ERROR: case REDIS_REPLY_STATUS: case REDIS_REPLY_INTEGER: + case REDIS_REPLY_DOUBLE: + case REDIS_REPLY_NIL: + case REDIS_REPLY_BOOL: return processLineItem(r); case REDIS_REPLY_STRING: + case REDIS_REPLY_VERB: return processBulkItem(r); case REDIS_REPLY_ARRAY: - return processMultiBulkItem(r); + case REDIS_REPLY_MAP: + case REDIS_REPLY_SET: + case REDIS_REPLY_PUSH: + return processAggregateItem(r); default: assert(NULL); return REDIS_ERR; /* Avoid warning. */ @@ -416,34 +605,57 @@ static int processItem(redisReader *r) { redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn) { redisReader *r; - r = calloc(sizeof(redisReader),1); + r = hi_calloc(1,sizeof(redisReader)); if (r == NULL) return NULL; - r->err = 0; - r->errstr[0] = '\0'; - r->fn = fn; - r->buf = sdsempty(); - r->maxbuf = REDIS_READER_MAX_BUF; - if (r->buf == NULL) { - free(r); - return NULL; + r->buf = hi_sdsempty(); + if (r->buf == NULL) + goto oom; + + r->task = hi_calloc(REDIS_READER_STACK_SIZE, sizeof(*r->task)); + if (r->task == NULL) + goto oom; + + for (; r->tasks < REDIS_READER_STACK_SIZE; r->tasks++) { + r->task[r->tasks] = hi_calloc(1, sizeof(**r->task)); + if (r->task[r->tasks] == NULL) + goto oom; } + r->fn = fn; + r->maxbuf = REDIS_READER_MAX_BUF; + r->maxelements = REDIS_READER_MAX_ARRAY_ELEMENTS; r->ridx = -1; + return r; +oom: + redisReaderFree(r); + return NULL; } void redisReaderFree(redisReader *r) { + if (r == NULL) + return; + if (r->reply != NULL && r->fn && r->fn->freeObject) r->fn->freeObject(r->reply); - if (r->buf != NULL) - sdsfree(r->buf); - free(r); + + if (r->task) { + /* We know r->task[i] is allocated if i < r->tasks */ + for (int i = 0; i < r->tasks; i++) { + hi_free(r->task[i]); + } + + hi_free(r->task); + } + + hi_sdsfree(r->buf); + hi_free(r); } int redisReaderFeed(redisReader *r, const char *buf, size_t len) { - sds newbuf; + hisds newbuf; /* Return early when this reader is in an erroneous state. */ if (r->err) @@ -452,26 +664,25 @@ int redisReaderFeed(redisReader *r, const char *buf, size_t len) { /* Copy the provided buffer. */ if (buf != NULL && len >= 1) { /* Destroy internal buffer when it is empty and is quite large. */ - if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) { - sdsfree(r->buf); - r->buf = sdsempty(); - r->pos = 0; + if (r->len == 0 && r->maxbuf != 0 && hi_sdsavail(r->buf) > r->maxbuf) { + hi_sdsfree(r->buf); + r->buf = hi_sdsempty(); + if (r->buf == 0) goto oom; - /* r->buf should not be NULL since we just free'd a larger one. */ - assert(r->buf != NULL); + r->pos = 0; } - newbuf = sdscatlen(r->buf,buf,len); - if (newbuf == NULL) { - __redisReaderSetErrorOOM(r); - return REDIS_ERR; - } + newbuf = hi_sdscatlen(r->buf,buf,len); + if (newbuf == NULL) goto oom; r->buf = newbuf; - r->len = sdslen(r->buf); + r->len = hi_sdslen(r->buf); } return REDIS_OK; +oom: + __redisReaderSetErrorOOM(r); + return REDIS_ERR; } int redisReaderGetReply(redisReader *r, void **reply) { @@ -489,12 +700,12 @@ int redisReaderGetReply(redisReader *r, void **reply) { /* Set first item to process when the stack is empty. */ if (r->ridx == -1) { - r->rstack[0].type = -1; - r->rstack[0].elements = -1; - r->rstack[0].idx = -1; - r->rstack[0].obj = NULL; - r->rstack[0].parent = NULL; - r->rstack[0].privdata = r->privdata; + r->task[0]->type = -1; + r->task[0]->elements = -1; + r->task[0]->idx = -1; + r->task[0]->obj = NULL; + r->task[0]->parent = NULL; + r->task[0]->privdata = r->privdata; r->ridx = 0; } @@ -510,15 +721,18 @@ int redisReaderGetReply(redisReader *r, void **reply) { /* Discard part of the buffer when we've consumed at least 1k, to avoid * doing unnecessary calls to memmove() in sds.c. */ if (r->pos >= 1024) { - sdsrange(r->buf,r->pos,-1); + if (hi_sdsrange(r->buf,r->pos,-1) < 0) return REDIS_ERR; r->pos = 0; - r->len = sdslen(r->buf); + r->len = hi_sdslen(r->buf); } /* Emit a reply when there is one. */ if (r->ridx == -1) { - if (reply != NULL) + if (reply != NULL) { *reply = r->reply; + } else if (r->reply != NULL && r->fn && r->fn->freeObject) { + r->fn->freeObject(r->reply); + } r->reply = NULL; } return REDIS_OK; diff --git a/redis.submodule/deps/hiredis/read.h b/redis.submodule/deps/hiredis/read.h index 2988aa4..2d74d77 100644 --- a/redis.submodule/deps/hiredis/read.h +++ b/redis.submodule/deps/hiredis/read.h @@ -45,6 +45,7 @@ #define REDIS_ERR_EOF 3 /* End of file */ #define REDIS_ERR_PROTOCOL 4 /* Protocol error */ #define REDIS_ERR_OOM 5 /* Out of memory */ +#define REDIS_ERR_TIMEOUT 6 /* Timed out */ #define REDIS_ERR_OTHER 2 /* Everything else... */ #define REDIS_REPLY_STRING 1 @@ -53,8 +54,20 @@ #define REDIS_REPLY_NIL 4 #define REDIS_REPLY_STATUS 5 #define REDIS_REPLY_ERROR 6 +#define REDIS_REPLY_DOUBLE 7 +#define REDIS_REPLY_BOOL 8 +#define REDIS_REPLY_MAP 9 +#define REDIS_REPLY_SET 10 +#define REDIS_REPLY_ATTR 11 +#define REDIS_REPLY_PUSH 12 +#define REDIS_REPLY_BIGNUM 13 +#define REDIS_REPLY_VERB 14 -#define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */ +/* Default max unused reader buffer. */ +#define REDIS_READER_MAX_BUF (1024*16) + +/* Default multi-bulk element limit */ +#define REDIS_READER_MAX_ARRAY_ELEMENTS ((1LL<<32) - 1) #ifdef __cplusplus extern "C" { @@ -62,7 +75,7 @@ extern "C" { typedef struct redisReadTask { int type; - int elements; /* number of elements in multibulk container */ + long long elements; /* number of elements in multibulk container */ int idx; /* index in parent (array) object */ void *obj; /* holds user-generated value for a read task */ struct redisReadTask *parent; /* parent task */ @@ -71,9 +84,11 @@ typedef struct redisReadTask { typedef struct redisReplyObjectFunctions { void *(*createString)(const redisReadTask*, char*, size_t); - void *(*createArray)(const redisReadTask*, int); + void *(*createArray)(const redisReadTask*, size_t); void *(*createInteger)(const redisReadTask*, long long); + void *(*createDouble)(const redisReadTask*, double, char*, size_t); void *(*createNil)(const redisReadTask*); + void *(*createBool)(const redisReadTask*, int); void (*freeObject)(void*); } redisReplyObjectFunctions; @@ -85,8 +100,11 @@ typedef struct redisReader { size_t pos; /* Buffer cursor */ size_t len; /* Buffer length */ size_t maxbuf; /* Max length of unused buffer */ + long long maxelements; /* Max multi-bulk elements */ + + redisReadTask **task; + int tasks; - redisReadTask rstack[9]; int ridx; /* Index of current read task */ void *reply; /* Temporary reply pointer */ diff --git a/redis.submodule/deps/hiredis/sds.c b/redis.submodule/deps/hiredis/sds.c index 923ffd8..675e764 100644 --- a/redis.submodule/deps/hiredis/sds.c +++ b/redis.submodule/deps/hiredis/sds.c @@ -30,98 +30,100 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#include "fmacros.h" #include #include #include #include #include +#include #include "sds.h" #include "sdsalloc.h" -static inline int sdsHdrSize(char type) { - switch(type&SDS_TYPE_MASK) { - case SDS_TYPE_5: - return sizeof(struct sdshdr5); - case SDS_TYPE_8: - return sizeof(struct sdshdr8); - case SDS_TYPE_16: - return sizeof(struct sdshdr16); - case SDS_TYPE_32: - return sizeof(struct sdshdr32); - case SDS_TYPE_64: - return sizeof(struct sdshdr64); +static inline int hi_sdsHdrSize(char type) { + switch(type&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: + return sizeof(struct hisdshdr5); + case HI_SDS_TYPE_8: + return sizeof(struct hisdshdr8); + case HI_SDS_TYPE_16: + return sizeof(struct hisdshdr16); + case HI_SDS_TYPE_32: + return sizeof(struct hisdshdr32); + case HI_SDS_TYPE_64: + return sizeof(struct hisdshdr64); } return 0; } -static inline char sdsReqType(size_t string_size) { +static inline char hi_sdsReqType(size_t string_size) { if (string_size < 32) - return SDS_TYPE_5; + return HI_SDS_TYPE_5; if (string_size < 0xff) - return SDS_TYPE_8; + return HI_SDS_TYPE_8; if (string_size < 0xffff) - return SDS_TYPE_16; + return HI_SDS_TYPE_16; if (string_size < 0xffffffff) - return SDS_TYPE_32; - return SDS_TYPE_64; + return HI_SDS_TYPE_32; + return HI_SDS_TYPE_64; } -/* Create a new sds string with the content specified by the 'init' pointer +/* Create a new hisds string with the content specified by the 'init' pointer * and 'initlen'. * If NULL is used for 'init' the string is initialized with zero bytes. * - * The string is always null-termined (all the sds strings are, always) so - * even if you create an sds string with: + * The string is always null-termined (all the hisds strings are, always) so + * even if you create an hisds string with: * - * mystring = sdsnewlen("abc",3); + * mystring = hi_sdsnewlen("abc",3); * * You can print the string with printf() as there is an implicit \0 at the * end of the string. However the string is binary safe and can contain - * \0 characters in the middle, as the length is stored in the sds header. */ -sds sdsnewlen(const void *init, size_t initlen) { + * \0 characters in the middle, as the length is stored in the hisds header. */ +hisds hi_sdsnewlen(const void *init, size_t initlen) { void *sh; - sds s; - char type = sdsReqType(initlen); + hisds s; + char type = hi_sdsReqType(initlen); /* Empty strings are usually created in order to append. Use type 8 * since type 5 is not good at this. */ - if (type == SDS_TYPE_5 && initlen == 0) type = SDS_TYPE_8; - int hdrlen = sdsHdrSize(type); + if (type == HI_SDS_TYPE_5 && initlen == 0) type = HI_SDS_TYPE_8; + int hdrlen = hi_sdsHdrSize(type); unsigned char *fp; /* flags pointer. */ - sh = s_malloc(hdrlen+initlen+1); + sh = hi_s_malloc(hdrlen+initlen+1); if (sh == NULL) return NULL; if (!init) memset(sh, 0, hdrlen+initlen+1); s = (char*)sh+hdrlen; fp = ((unsigned char*)s)-1; switch(type) { - case SDS_TYPE_5: { - *fp = type | (initlen << SDS_TYPE_BITS); + case HI_SDS_TYPE_5: { + *fp = type | (initlen << HI_SDS_TYPE_BITS); break; } - case SDS_TYPE_8: { - SDS_HDR_VAR(8,s); + case HI_SDS_TYPE_8: { + HI_SDS_HDR_VAR(8,s); sh->len = initlen; sh->alloc = initlen; *fp = type; break; } - case SDS_TYPE_16: { - SDS_HDR_VAR(16,s); + case HI_SDS_TYPE_16: { + HI_SDS_HDR_VAR(16,s); sh->len = initlen; sh->alloc = initlen; *fp = type; break; } - case SDS_TYPE_32: { - SDS_HDR_VAR(32,s); + case HI_SDS_TYPE_32: { + HI_SDS_HDR_VAR(32,s); sh->len = initlen; sh->alloc = initlen; *fp = type; break; } - case SDS_TYPE_64: { - SDS_HDR_VAR(64,s); + case HI_SDS_TYPE_64: { + HI_SDS_HDR_VAR(64,s); sh->len = initlen; sh->alloc = initlen; *fp = type; @@ -134,164 +136,164 @@ sds sdsnewlen(const void *init, size_t initlen) { return s; } -/* Create an empty (zero length) sds string. Even in this case the string +/* Create an empty (zero length) hisds string. Even in this case the string * always has an implicit null term. */ -sds sdsempty(void) { - return sdsnewlen("",0); +hisds hi_sdsempty(void) { + return hi_sdsnewlen("",0); } -/* Create a new sds string starting from a null terminated C string. */ -sds sdsnew(const char *init) { +/* Create a new hisds string starting from a null terminated C string. */ +hisds hi_sdsnew(const char *init) { size_t initlen = (init == NULL) ? 0 : strlen(init); - return sdsnewlen(init, initlen); + return hi_sdsnewlen(init, initlen); } -/* Duplicate an sds string. */ -sds sdsdup(const sds s) { - return sdsnewlen(s, sdslen(s)); +/* Duplicate an hisds string. */ +hisds hi_sdsdup(const hisds s) { + return hi_sdsnewlen(s, hi_sdslen(s)); } -/* Free an sds string. No operation is performed if 's' is NULL. */ -void sdsfree(sds s) { +/* Free an hisds string. No operation is performed if 's' is NULL. */ +void hi_sdsfree(hisds s) { if (s == NULL) return; - s_free((char*)s-sdsHdrSize(s[-1])); + hi_s_free((char*)s-hi_sdsHdrSize(s[-1])); } -/* Set the sds string length to the length as obtained with strlen(), so +/* Set the hisds string length to the length as obtained with strlen(), so * considering as content only up to the first null term character. * - * This function is useful when the sds string is hacked manually in some + * This function is useful when the hisds string is hacked manually in some * way, like in the following example: * - * s = sdsnew("foobar"); + * s = hi_sdsnew("foobar"); * s[2] = '\0'; - * sdsupdatelen(s); - * printf("%d\n", sdslen(s)); + * hi_sdsupdatelen(s); + * printf("%d\n", hi_sdslen(s)); * - * The output will be "2", but if we comment out the call to sdsupdatelen() + * The output will be "2", but if we comment out the call to hi_sdsupdatelen() * the output will be "6" as the string was modified but the logical length * remains 6 bytes. */ -void sdsupdatelen(sds s) { +void hi_sdsupdatelen(hisds s) { int reallen = strlen(s); - sdssetlen(s, reallen); + hi_sdssetlen(s, reallen); } -/* Modify an sds string in-place to make it empty (zero length). +/* Modify an hisds string in-place to make it empty (zero length). * However all the existing buffer is not discarded but set as free space * so that next append operations will not require allocations up to the * number of bytes previously available. */ -void sdsclear(sds s) { - sdssetlen(s, 0); +void hi_sdsclear(hisds s) { + hi_sdssetlen(s, 0); s[0] = '\0'; } -/* Enlarge the free space at the end of the sds string so that the caller +/* Enlarge the free space at the end of the hisds string so that the caller * is sure that after calling this function can overwrite up to addlen * bytes after the end of the string, plus one more byte for nul term. * - * Note: this does not change the *length* of the sds string as returned - * by sdslen(), but only the free buffer space we have. */ -sds sdsMakeRoomFor(sds s, size_t addlen) { + * Note: this does not change the *length* of the hisds string as returned + * by hi_sdslen(), but only the free buffer space we have. */ +hisds hi_sdsMakeRoomFor(hisds s, size_t addlen) { void *sh, *newsh; - size_t avail = sdsavail(s); + size_t avail = hi_sdsavail(s); size_t len, newlen; - char type, oldtype = s[-1] & SDS_TYPE_MASK; + char type, oldtype = s[-1] & HI_SDS_TYPE_MASK; int hdrlen; /* Return ASAP if there is enough space left. */ if (avail >= addlen) return s; - len = sdslen(s); - sh = (char*)s-sdsHdrSize(oldtype); + len = hi_sdslen(s); + sh = (char*)s-hi_sdsHdrSize(oldtype); newlen = (len+addlen); - if (newlen < SDS_MAX_PREALLOC) + if (newlen < HI_SDS_MAX_PREALLOC) newlen *= 2; else - newlen += SDS_MAX_PREALLOC; + newlen += HI_SDS_MAX_PREALLOC; - type = sdsReqType(newlen); + type = hi_sdsReqType(newlen); /* Don't use type 5: the user is appending to the string and type 5 is - * not able to remember empty space, so sdsMakeRoomFor() must be called + * not able to remember empty space, so hi_sdsMakeRoomFor() must be called * at every appending operation. */ - if (type == SDS_TYPE_5) type = SDS_TYPE_8; + if (type == HI_SDS_TYPE_5) type = HI_SDS_TYPE_8; - hdrlen = sdsHdrSize(type); + hdrlen = hi_sdsHdrSize(type); if (oldtype==type) { - newsh = s_realloc(sh, hdrlen+newlen+1); + newsh = hi_s_realloc(sh, hdrlen+newlen+1); if (newsh == NULL) return NULL; s = (char*)newsh+hdrlen; } else { /* Since the header size changes, need to move the string forward, * and can't use realloc */ - newsh = s_malloc(hdrlen+newlen+1); + newsh = hi_s_malloc(hdrlen+newlen+1); if (newsh == NULL) return NULL; memcpy((char*)newsh+hdrlen, s, len+1); - s_free(sh); + hi_s_free(sh); s = (char*)newsh+hdrlen; s[-1] = type; - sdssetlen(s, len); + hi_sdssetlen(s, len); } - sdssetalloc(s, newlen); + hi_sdssetalloc(s, newlen); return s; } -/* Reallocate the sds string so that it has no free space at the end. The +/* Reallocate the hisds string so that it has no free space at the end. The * contained string remains not altered, but next concatenation operations * will require a reallocation. * - * After the call, the passed sds string is no longer valid and all the + * After the call, the passed hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ -sds sdsRemoveFreeSpace(sds s) { +hisds hi_sdsRemoveFreeSpace(hisds s) { void *sh, *newsh; - char type, oldtype = s[-1] & SDS_TYPE_MASK; + char type, oldtype = s[-1] & HI_SDS_TYPE_MASK; int hdrlen; - size_t len = sdslen(s); - sh = (char*)s-sdsHdrSize(oldtype); + size_t len = hi_sdslen(s); + sh = (char*)s-hi_sdsHdrSize(oldtype); - type = sdsReqType(len); - hdrlen = sdsHdrSize(type); + type = hi_sdsReqType(len); + hdrlen = hi_sdsHdrSize(type); if (oldtype==type) { - newsh = s_realloc(sh, hdrlen+len+1); + newsh = hi_s_realloc(sh, hdrlen+len+1); if (newsh == NULL) return NULL; s = (char*)newsh+hdrlen; } else { - newsh = s_malloc(hdrlen+len+1); + newsh = hi_s_malloc(hdrlen+len+1); if (newsh == NULL) return NULL; memcpy((char*)newsh+hdrlen, s, len+1); - s_free(sh); + hi_s_free(sh); s = (char*)newsh+hdrlen; s[-1] = type; - sdssetlen(s, len); + hi_sdssetlen(s, len); } - sdssetalloc(s, len); + hi_sdssetalloc(s, len); return s; } -/* Return the total size of the allocation of the specifed sds string, +/* Return the total size of the allocation of the specifed hisds string, * including: - * 1) The sds header before the pointer. + * 1) The hisds header before the pointer. * 2) The string. * 3) The free buffer at the end if any. * 4) The implicit null term. */ -size_t sdsAllocSize(sds s) { - size_t alloc = sdsalloc(s); - return sdsHdrSize(s[-1])+alloc+1; +size_t hi_sdsAllocSize(hisds s) { + size_t alloc = hi_sdsalloc(s); + return hi_sdsHdrSize(s[-1])+alloc+1; } /* Return the pointer of the actual SDS allocation (normally SDS strings * are referenced by the start of the string buffer). */ -void *sdsAllocPtr(sds s) { - return (void*) (s-sdsHdrSize(s[-1])); +void *hi_sdsAllocPtr(hisds s) { + return (void*) (s-hi_sdsHdrSize(s[-1])); } -/* Increment the sds length and decrements the left free space at the +/* Increment the hisds length and decrements the left free space at the * end of the string according to 'incr'. Also set the null term * in the new end of the string. * * This function is used in order to fix the string length after the - * user calls sdsMakeRoomFor(), writes something after the end of + * user calls hi_sdsMakeRoomFor(), writes something after the end of * the current string, and finally needs to set the new length. * * Note: it is possible to use a negative increment in order to @@ -299,48 +301,48 @@ void *sdsAllocPtr(sds s) { * * Usage example: * - * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the + * Using hi_sdsIncrLen() and hi_sdsMakeRoomFor() it is possible to mount the * following schema, to cat bytes coming from the kernel to the end of an - * sds string without copying into an intermediate buffer: + * hisds string without copying into an intermediate buffer: * - * oldlen = sdslen(s); - * s = sdsMakeRoomFor(s, BUFFER_SIZE); + * oldlen = hi_hi_sdslen(s); + * s = hi_sdsMakeRoomFor(s, BUFFER_SIZE); * nread = read(fd, s+oldlen, BUFFER_SIZE); * ... check for nread <= 0 and handle it ... - * sdsIncrLen(s, nread); + * hi_sdsIncrLen(s, nread); */ -void sdsIncrLen(sds s, int incr) { +void hi_sdsIncrLen(hisds s, int incr) { unsigned char flags = s[-1]; size_t len; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: { + switch(flags&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - unsigned char oldlen = SDS_TYPE_5_LEN(flags); + unsigned char oldlen = HI_SDS_TYPE_5_LEN(flags); assert((incr > 0 && oldlen+incr < 32) || (incr < 0 && oldlen >= (unsigned int)(-incr))); - *fp = SDS_TYPE_5 | ((oldlen+incr) << SDS_TYPE_BITS); + *fp = HI_SDS_TYPE_5 | ((oldlen+incr) << HI_SDS_TYPE_BITS); len = oldlen+incr; break; } - case SDS_TYPE_8: { - SDS_HDR_VAR(8,s); + case HI_SDS_TYPE_8: { + HI_SDS_HDR_VAR(8,s); assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr))); len = (sh->len += incr); break; } - case SDS_TYPE_16: { - SDS_HDR_VAR(16,s); + case HI_SDS_TYPE_16: { + HI_SDS_HDR_VAR(16,s); assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr))); len = (sh->len += incr); break; } - case SDS_TYPE_32: { - SDS_HDR_VAR(32,s); + case HI_SDS_TYPE_32: { + HI_SDS_HDR_VAR(32,s); assert((incr >= 0 && sh->alloc-sh->len >= (unsigned int)incr) || (incr < 0 && sh->len >= (unsigned int)(-incr))); len = (sh->len += incr); break; } - case SDS_TYPE_64: { - SDS_HDR_VAR(64,s); + case HI_SDS_TYPE_64: { + HI_SDS_HDR_VAR(64,s); assert((incr >= 0 && sh->alloc-sh->len >= (uint64_t)incr) || (incr < 0 && sh->len >= (uint64_t)(-incr))); len = (sh->len += incr); break; @@ -350,83 +352,83 @@ void sdsIncrLen(sds s, int incr) { s[len] = '\0'; } -/* Grow the sds to have the specified length. Bytes that were not part of - * the original length of the sds will be set to zero. +/* Grow the hisds to have the specified length. Bytes that were not part of + * the original length of the hisds will be set to zero. * * if the specified length is smaller than the current length, no operation * is performed. */ -sds sdsgrowzero(sds s, size_t len) { - size_t curlen = sdslen(s); +hisds hi_sdsgrowzero(hisds s, size_t len) { + size_t curlen = hi_sdslen(s); if (len <= curlen) return s; - s = sdsMakeRoomFor(s,len-curlen); + s = hi_sdsMakeRoomFor(s,len-curlen); if (s == NULL) return NULL; /* Make sure added region doesn't contain garbage */ memset(s+curlen,0,(len-curlen+1)); /* also set trailing \0 byte */ - sdssetlen(s, len); + hi_sdssetlen(s, len); return s; } /* Append the specified binary-safe string pointed by 't' of 'len' bytes to the - * end of the specified sds string 's'. + * end of the specified hisds string 's'. * - * After the call, the passed sds string is no longer valid and all the + * After the call, the passed hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ -sds sdscatlen(sds s, const void *t, size_t len) { - size_t curlen = sdslen(s); +hisds hi_sdscatlen(hisds s, const void *t, size_t len) { + size_t curlen = hi_sdslen(s); - s = sdsMakeRoomFor(s,len); + s = hi_sdsMakeRoomFor(s,len); if (s == NULL) return NULL; memcpy(s+curlen, t, len); - sdssetlen(s, curlen+len); + hi_sdssetlen(s, curlen+len); s[curlen+len] = '\0'; return s; } -/* Append the specified null termianted C string to the sds string 's'. +/* Append the specified null termianted C string to the hisds string 's'. * - * After the call, the passed sds string is no longer valid and all the + * After the call, the passed hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ -sds sdscat(sds s, const char *t) { - return sdscatlen(s, t, strlen(t)); +hisds hi_sdscat(hisds s, const char *t) { + return hi_sdscatlen(s, t, strlen(t)); } -/* Append the specified sds 't' to the existing sds 's'. +/* Append the specified hisds 't' to the existing hisds 's'. * - * After the call, the modified sds string is no longer valid and all the + * After the call, the modified hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ -sds sdscatsds(sds s, const sds t) { - return sdscatlen(s, t, sdslen(t)); +hisds hi_sdscatsds(hisds s, const hisds t) { + return hi_sdscatlen(s, t, hi_sdslen(t)); } -/* Destructively modify the sds string 's' to hold the specified binary +/* Destructively modify the hisds string 's' to hold the specified binary * safe string pointed by 't' of length 'len' bytes. */ -sds sdscpylen(sds s, const char *t, size_t len) { - if (sdsalloc(s) < len) { - s = sdsMakeRoomFor(s,len-sdslen(s)); +hisds hi_sdscpylen(hisds s, const char *t, size_t len) { + if (hi_sdsalloc(s) < len) { + s = hi_sdsMakeRoomFor(s,len-hi_sdslen(s)); if (s == NULL) return NULL; } memcpy(s, t, len); s[len] = '\0'; - sdssetlen(s, len); + hi_sdssetlen(s, len); return s; } -/* Like sdscpylen() but 't' must be a null-termined string so that the length +/* Like hi_sdscpylen() but 't' must be a null-termined string so that the length * of the string is obtained with strlen(). */ -sds sdscpy(sds s, const char *t) { - return sdscpylen(s, t, strlen(t)); +hisds hi_sdscpy(hisds s, const char *t) { + return hi_sdscpylen(s, t, strlen(t)); } -/* Helper for sdscatlonglong() doing the actual number -> string +/* Helper for hi_sdscatlonglong() doing the actual number -> string * conversion. 's' must point to a string with room for at least - * SDS_LLSTR_SIZE bytes. + * HI_SDS_LLSTR_SIZE bytes. * * The function returns the length of the null-terminated string * representation stored at 's'. */ -#define SDS_LLSTR_SIZE 21 -int sdsll2str(char *s, long long value) { +#define HI_SDS_LLSTR_SIZE 21 +int hi_sdsll2str(char *s, long long value) { char *p, aux; unsigned long long v; size_t l; @@ -457,8 +459,8 @@ int sdsll2str(char *s, long long value) { return l; } -/* Identical sdsll2str(), but for unsigned long long type. */ -int sdsull2str(char *s, unsigned long long v) { +/* Identical hi_sdsll2str(), but for unsigned long long type. */ +int hi_sdsull2str(char *s, unsigned long long v) { char *p, aux; size_t l; @@ -486,19 +488,19 @@ int sdsull2str(char *s, unsigned long long v) { return l; } -/* Create an sds string from a long long value. It is much faster than: +/* Create an hisds string from a long long value. It is much faster than: * - * sdscatprintf(sdsempty(),"%lld\n", value); + * hi_sdscatprintf(hi_sdsempty(),"%lld\n", value); */ -sds sdsfromlonglong(long long value) { - char buf[SDS_LLSTR_SIZE]; - int len = sdsll2str(buf,value); +hisds hi_sdsfromlonglong(long long value) { + char buf[HI_SDS_LLSTR_SIZE]; + int len = hi_sdsll2str(buf,value); - return sdsnewlen(buf,len); + return hi_sdsnewlen(buf,len); } -/* Like sdscatprintf() but gets va_list instead of being variadic. */ -sds sdscatvprintf(sds s, const char *fmt, va_list ap) { +/* Like hi_sdscatprintf() but gets va_list instead of being variadic. */ +hisds hi_sdscatvprintf(hisds s, const char *fmt, va_list ap) { va_list cpy; char staticbuf[1024], *buf = staticbuf, *t; size_t buflen = strlen(fmt)*2; @@ -506,7 +508,7 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { /* We try to start using a static buffer for speed. * If not possible we revert to heap allocation. */ if (buflen > sizeof(staticbuf)) { - buf = s_malloc(buflen); + buf = hi_s_malloc(buflen); if (buf == NULL) return NULL; } else { buflen = sizeof(staticbuf); @@ -520,9 +522,9 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { vsnprintf(buf, buflen, fmt, cpy); va_end(cpy); if (buf[buflen-2] != '\0') { - if (buf != staticbuf) s_free(buf); + if (buf != staticbuf) hi_s_free(buf); buflen *= 2; - buf = s_malloc(buflen); + buf = hi_s_malloc(buflen); if (buf == NULL) return NULL; continue; } @@ -530,39 +532,39 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { } /* Finally concat the obtained string to the SDS string and return it. */ - t = sdscat(s, buf); - if (buf != staticbuf) s_free(buf); + t = hi_sdscat(s, buf); + if (buf != staticbuf) hi_s_free(buf); return t; } -/* Append to the sds string 's' a string obtained using printf-alike format +/* Append to the hisds string 's' a string obtained using printf-alike format * specifier. * - * After the call, the modified sds string is no longer valid and all the + * After the call, the modified hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. * * Example: * - * s = sdsnew("Sum is: "); - * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b). + * s = hi_sdsnew("Sum is: "); + * s = hi_sdscatprintf(s,"%d+%d = %d",a,b,a+b). * * Often you need to create a string from scratch with the printf-alike - * format. When this is the need, just use sdsempty() as the target string: + * format. When this is the need, just use hi_sdsempty() as the target string: * - * s = sdscatprintf(sdsempty(), "... your format ...", args); + * s = hi_sdscatprintf(hi_sdsempty(), "... your format ...", args); */ -sds sdscatprintf(sds s, const char *fmt, ...) { +hisds hi_sdscatprintf(hisds s, const char *fmt, ...) { va_list ap; char *t; va_start(ap, fmt); - t = sdscatvprintf(s,fmt,ap); + t = hi_sdscatvprintf(s,fmt,ap); va_end(ap); return t; } -/* This function is similar to sdscatprintf, but much faster as it does +/* This function is similar to hi_sdscatprintf, but much faster as it does * not rely on sprintf() family functions implemented by the libc that - * are often very slow. Moreover directly handling the sds string as + * are often very slow. Moreover directly handling the hisds string as * new data is concatenated provides a performance improvement. * * However this function only handles an incompatible subset of printf-alike @@ -576,13 +578,13 @@ sds sdscatprintf(sds s, const char *fmt, ...) { * %U - 64 bit unsigned integer (unsigned long long, uint64_t) * %% - Verbatim "%" character. */ -sds sdscatfmt(sds s, char const *fmt, ...) { +hisds hi_sdscatfmt(hisds s, char const *fmt, ...) { const char *f = fmt; int i; va_list ap; va_start(ap,fmt); - i = sdslen(s); /* Position of the next byte to write to dest str. */ + i = hi_sdslen(s); /* Position of the next byte to write to dest str. */ while(*f) { char next, *str; size_t l; @@ -590,8 +592,9 @@ sds sdscatfmt(sds s, char const *fmt, ...) { unsigned long long unum; /* Make sure there is always space for at least 1 char. */ - if (sdsavail(s)==0) { - s = sdsMakeRoomFor(s,1); + if (hi_sdsavail(s)==0) { + s = hi_sdsMakeRoomFor(s,1); + if (s == NULL) goto fmt_error; } switch(*f) { @@ -602,12 +605,13 @@ sds sdscatfmt(sds s, char const *fmt, ...) { case 's': case 'S': str = va_arg(ap,char*); - l = (next == 's') ? strlen(str) : sdslen(str); - if (sdsavail(s) < l) { - s = sdsMakeRoomFor(s,l); + l = (next == 's') ? strlen(str) : hi_sdslen(str); + if (hi_sdsavail(s) < l) { + s = hi_sdsMakeRoomFor(s,l); + if (s == NULL) goto fmt_error; } memcpy(s+i,str,l); - sdsinclen(s,l); + hi_sdsinclen(s,l); i += l; break; case 'i': @@ -617,13 +621,14 @@ sds sdscatfmt(sds s, char const *fmt, ...) { else num = va_arg(ap,long long); { - char buf[SDS_LLSTR_SIZE]; - l = sdsll2str(buf,num); - if (sdsavail(s) < l) { - s = sdsMakeRoomFor(s,l); + char buf[HI_SDS_LLSTR_SIZE]; + l = hi_sdsll2str(buf,num); + if (hi_sdsavail(s) < l) { + s = hi_sdsMakeRoomFor(s,l); + if (s == NULL) goto fmt_error; } memcpy(s+i,buf,l); - sdsinclen(s,l); + hi_sdsinclen(s,l); i += l; } break; @@ -634,25 +639,26 @@ sds sdscatfmt(sds s, char const *fmt, ...) { else unum = va_arg(ap,unsigned long long); { - char buf[SDS_LLSTR_SIZE]; - l = sdsull2str(buf,unum); - if (sdsavail(s) < l) { - s = sdsMakeRoomFor(s,l); + char buf[HI_SDS_LLSTR_SIZE]; + l = hi_sdsull2str(buf,unum); + if (hi_sdsavail(s) < l) { + s = hi_sdsMakeRoomFor(s,l); + if (s == NULL) goto fmt_error; } memcpy(s+i,buf,l); - sdsinclen(s,l); + hi_sdsinclen(s,l); i += l; } break; default: /* Handle %% and generally %. */ s[i++] = next; - sdsinclen(s,1); + hi_sdsinclen(s,1); break; } break; default: s[i++] = *f; - sdsinclen(s,1); + hi_sdsinclen(s,1); break; } f++; @@ -662,34 +668,38 @@ sds sdscatfmt(sds s, char const *fmt, ...) { /* Add null-term */ s[i] = '\0'; return s; + +fmt_error: + va_end(ap); + return NULL; } /* Remove the part of the string from left and from right composed just of * contiguous characters found in 'cset', that is a null terminted C string. * - * After the call, the modified sds string is no longer valid and all the + * After the call, the modified hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. * * Example: * - * s = sdsnew("AA...AA.a.aa.aHelloWorld :::"); - * s = sdstrim(s,"Aa. :"); + * s = hi_sdsnew("AA...AA.a.aa.aHelloWorld :::"); + * s = hi_sdstrim(s,"Aa. :"); * printf("%s\n", s); * * Output will be just "Hello World". */ -sds sdstrim(sds s, const char *cset) { +hisds hi_sdstrim(hisds s, const char *cset) { char *start, *end, *sp, *ep; size_t len; sp = start = s; - ep = end = s+sdslen(s)-1; + ep = end = s+hi_sdslen(s)-1; while(sp <= end && strchr(cset, *sp)) sp++; while(ep > sp && strchr(cset, *ep)) ep--; len = (sp > ep) ? 0 : ((ep-sp)+1); if (s != sp) memmove(s, sp, len); s[len] = '\0'; - sdssetlen(s,len); + hi_sdssetlen(s,len); return s; } @@ -704,15 +714,20 @@ sds sdstrim(sds s, const char *cset) { * * The string is modified in-place. * + * Return value: + * -1 (error) if hi_sdslen(s) is larger than maximum positive ssize_t value. + * 0 on success. + * * Example: * - * s = sdsnew("Hello World"); - * sdsrange(s,1,-1); => "ello World" + * s = hi_sdsnew("Hello World"); + * hi_sdsrange(s,1,-1); => "ello World" */ -void sdsrange(sds s, int start, int end) { - size_t newlen, len = sdslen(s); +int hi_sdsrange(hisds s, ssize_t start, ssize_t end) { + size_t newlen, len = hi_sdslen(s); + if (len > SSIZE_MAX) return -1; - if (len == 0) return; + if (len == 0) return 0; if (start < 0) { start = len+start; if (start < 0) start = 0; @@ -723,9 +738,9 @@ void sdsrange(sds s, int start, int end) { } newlen = (start > end) ? 0 : (end-start)+1; if (newlen != 0) { - if (start >= (signed)len) { + if (start >= (ssize_t)len) { newlen = 0; - } else if (end >= (signed)len) { + } else if (end >= (ssize_t)len) { end = len-1; newlen = (start > end) ? 0 : (end-start)+1; } @@ -734,24 +749,25 @@ void sdsrange(sds s, int start, int end) { } if (start && newlen) memmove(s, s+start, newlen); s[newlen] = 0; - sdssetlen(s,newlen); + hi_sdssetlen(s,newlen); + return 0; } -/* Apply tolower() to every character of the sds string 's'. */ -void sdstolower(sds s) { - int len = sdslen(s), j; +/* Apply tolower() to every character of the hisds string 's'. */ +void hi_sdstolower(hisds s) { + int len = hi_sdslen(s), j; for (j = 0; j < len; j++) s[j] = tolower(s[j]); } -/* Apply toupper() to every character of the sds string 's'. */ -void sdstoupper(sds s) { - int len = sdslen(s), j; +/* Apply toupper() to every character of the hisds string 's'. */ +void hi_sdstoupper(hisds s) { + int len = hi_sdslen(s), j; for (j = 0; j < len; j++) s[j] = toupper(s[j]); } -/* Compare two sds strings s1 and s2 with memcmp(). +/* Compare two hisds strings s1 and s2 with memcmp(). * * Return value: * @@ -762,12 +778,12 @@ void sdstoupper(sds s) { * If two strings share exactly the same prefix, but one of the two has * additional characters, the longer string is considered to be greater than * the smaller one. */ -int sdscmp(const sds s1, const sds s2) { +int hi_sdscmp(const hisds s1, const hisds s2) { size_t l1, l2, minlen; int cmp; - l1 = sdslen(s1); - l2 = sdslen(s2); + l1 = hi_sdslen(s1); + l2 = hi_sdslen(s2); minlen = (l1 < l2) ? l1 : l2; cmp = memcmp(s1,s2,minlen); if (cmp == 0) return l1-l2; @@ -775,7 +791,7 @@ int sdscmp(const sds s1, const sds s2) { } /* Split 's' with separator in 'sep'. An array - * of sds strings is returned. *count will be set + * of hisds strings is returned. *count will be set * by reference to the number of tokens returned. * * On out of memory, zero length string, zero length @@ -783,20 +799,20 @@ int sdscmp(const sds s1, const sds s2) { * * Note that 'sep' is able to split a string using * a multi-character separator. For example - * sdssplit("foo_-_bar","_-_"); will return two + * hi_sdssplit("foo_-_bar","_-_"); will return two * elements "foo" and "bar". * * This version of the function is binary-safe but - * requires length arguments. sdssplit() is just the + * requires length arguments. hi_sdssplit() is just the * same function but for zero-terminated strings. */ -sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) { +hisds *hi_sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) { int elements = 0, slots = 5, start = 0, j; - sds *tokens; + hisds *tokens; if (seplen < 1 || len < 0) return NULL; - tokens = s_malloc(sizeof(sds)*slots); + tokens = hi_s_malloc(sizeof(hisds)*slots); if (tokens == NULL) return NULL; if (len == 0) { @@ -806,16 +822,16 @@ sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count for (j = 0; j < (len-(seplen-1)); j++) { /* make sure there is room for the next element and the final one */ if (slots < elements+2) { - sds *newtokens; + hisds *newtokens; slots *= 2; - newtokens = s_realloc(tokens,sizeof(sds)*slots); + newtokens = hi_s_realloc(tokens,sizeof(hisds)*slots); if (newtokens == NULL) goto cleanup; tokens = newtokens; } /* search the separator */ if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) { - tokens[elements] = sdsnewlen(s+start,j-start); + tokens[elements] = hi_sdsnewlen(s+start,j-start); if (tokens[elements] == NULL) goto cleanup; elements++; start = j+seplen; @@ -823,7 +839,7 @@ sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count } } /* Add the final element. We are sure there is room in the tokens array. */ - tokens[elements] = sdsnewlen(s+start,len-start); + tokens[elements] = hi_sdsnewlen(s+start,len-start); if (tokens[elements] == NULL) goto cleanup; elements++; *count = elements; @@ -832,62 +848,55 @@ sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count cleanup: { int i; - for (i = 0; i < elements; i++) sdsfree(tokens[i]); - s_free(tokens); + for (i = 0; i < elements; i++) hi_sdsfree(tokens[i]); + hi_s_free(tokens); *count = 0; return NULL; } } -/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */ -void sdsfreesplitres(sds *tokens, int count) { +/* Free the result returned by hi_sdssplitlen(), or do nothing if 'tokens' is NULL. */ +void hi_sdsfreesplitres(hisds *tokens, int count) { if (!tokens) return; while(count--) - sdsfree(tokens[count]); - s_free(tokens); + hi_sdsfree(tokens[count]); + hi_s_free(tokens); } -/* Append to the sds string "s" an escaped string representation where +/* Append to the hisds string "s" an escaped string representation where * all the non-printable characters (tested with isprint()) are turned into * escapes in the form "\n\r\a...." or "\x". * - * After the call, the modified sds string is no longer valid and all the + * After the call, the modified hisds string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ -sds sdscatrepr(sds s, const char *p, size_t len) { - s = sdscatlen(s,"\"",1); +hisds hi_sdscatrepr(hisds s, const char *p, size_t len) { + s = hi_sdscatlen(s,"\"",1); while(len--) { switch(*p) { case '\\': case '"': - s = sdscatprintf(s,"\\%c",*p); + s = hi_sdscatprintf(s,"\\%c",*p); break; - case '\n': s = sdscatlen(s,"\\n",2); break; - case '\r': s = sdscatlen(s,"\\r",2); break; - case '\t': s = sdscatlen(s,"\\t",2); break; - case '\a': s = sdscatlen(s,"\\a",2); break; - case '\b': s = sdscatlen(s,"\\b",2); break; + case '\n': s = hi_sdscatlen(s,"\\n",2); break; + case '\r': s = hi_sdscatlen(s,"\\r",2); break; + case '\t': s = hi_sdscatlen(s,"\\t",2); break; + case '\a': s = hi_sdscatlen(s,"\\a",2); break; + case '\b': s = hi_sdscatlen(s,"\\b",2); break; default: if (isprint(*p)) - s = sdscatprintf(s,"%c",*p); + s = hi_sdscatprintf(s,"%c",*p); else - s = sdscatprintf(s,"\\x%02x",(unsigned char)*p); + s = hi_sdscatprintf(s,"\\x%02x",(unsigned char)*p); break; } p++; } - return sdscatlen(s,"\"",1); + return hi_sdscatlen(s,"\"",1); } -/* Helper function for sdssplitargs() that returns non zero if 'c' - * is a valid hex digit. */ -int is_hex_digit(char c) { - return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || - (c >= 'A' && c <= 'F'); -} - -/* Helper function for sdssplitargs() that converts a hex digit into an +/* Helper function for hi_sdssplitargs() that converts a hex digit into an * integer from 0 to 15 */ -int hex_digit_to_int(char c) { +static int hi_hex_digit_to_int(char c) { switch(c) { case '0': return 0; case '1': return 1; @@ -915,20 +924,20 @@ int hex_digit_to_int(char c) { * foo bar "newline are supported\n" and "\xff\x00otherstuff" * * The number of arguments is stored into *argc, and an array - * of sds is returned. + * of hisds is returned. * - * The caller should free the resulting array of sds strings with - * sdsfreesplitres(). + * The caller should free the resulting array of hisds strings with + * hi_sdsfreesplitres(). * - * Note that sdscatrepr() is able to convert back a string into - * a quoted string in the same format sdssplitargs() is able to parse. + * Note that hi_sdscatrepr() is able to convert back a string into + * a quoted string in the same format hi_sdssplitargs() is able to parse. * * The function returns the allocated tokens on success, even when the * input string is empty, or NULL if the input contains unbalanced * quotes or closed quotes followed by non space characters * as in: "foo"bar or "foo' */ -sds *sdssplitargs(const char *line, int *argc) { +hisds *hi_sdssplitargs(const char *line, int *argc) { const char *p = line; char *current = NULL; char **vector = NULL; @@ -943,18 +952,18 @@ sds *sdssplitargs(const char *line, int *argc) { int insq=0; /* set to 1 if we are in 'single quotes' */ int done=0; - if (current == NULL) current = sdsempty(); + if (current == NULL) current = hi_sdsempty(); while(!done) { if (inq) { if (*p == '\\' && *(p+1) == 'x' && - is_hex_digit(*(p+2)) && - is_hex_digit(*(p+3))) + isxdigit(*(p+2)) && + isxdigit(*(p+3))) { unsigned char byte; - byte = (hex_digit_to_int(*(p+2))*16)+ - hex_digit_to_int(*(p+3)); - current = sdscatlen(current,(char*)&byte,1); + byte = (hi_hex_digit_to_int(*(p+2))*16)+ + hi_hex_digit_to_int(*(p+3)); + current = hi_sdscatlen(current,(char*)&byte,1); p += 3; } else if (*p == '\\' && *(p+1)) { char c; @@ -968,7 +977,7 @@ sds *sdssplitargs(const char *line, int *argc) { case 'a': c = '\a'; break; default: c = *p; break; } - current = sdscatlen(current,&c,1); + current = hi_sdscatlen(current,&c,1); } else if (*p == '"') { /* closing quote must be followed by a space or * nothing at all. */ @@ -978,12 +987,12 @@ sds *sdssplitargs(const char *line, int *argc) { /* unterminated quotes */ goto err; } else { - current = sdscatlen(current,p,1); + current = hi_sdscatlen(current,p,1); } } else if (insq) { if (*p == '\\' && *(p+1) == '\'') { p++; - current = sdscatlen(current,"'",1); + current = hi_sdscatlen(current,"'",1); } else if (*p == '\'') { /* closing quote must be followed by a space or * nothing at all. */ @@ -993,7 +1002,7 @@ sds *sdssplitargs(const char *line, int *argc) { /* unterminated quotes */ goto err; } else { - current = sdscatlen(current,p,1); + current = hi_sdscatlen(current,p,1); } } else { switch(*p) { @@ -1011,29 +1020,37 @@ sds *sdssplitargs(const char *line, int *argc) { insq=1; break; default: - current = sdscatlen(current,p,1); + current = hi_sdscatlen(current,p,1); break; } } if (*p) p++; } /* add the token to the vector */ - vector = s_realloc(vector,((*argc)+1)*sizeof(char*)); - vector[*argc] = current; - (*argc)++; - current = NULL; + { + char **new_vector = hi_s_realloc(vector,((*argc)+1)*sizeof(char*)); + if (new_vector == NULL) { + hi_s_free(vector); + return NULL; + } + + vector = new_vector; + vector[*argc] = current; + (*argc)++; + current = NULL; + } } else { /* Even on empty input string return something not NULL. */ - if (vector == NULL) vector = s_malloc(sizeof(void*)); + if (vector == NULL) vector = hi_s_malloc(sizeof(void*)); return vector; } } err: while((*argc)--) - sdsfree(vector[*argc]); - s_free(vector); - if (current) sdsfree(current); + hi_sdsfree(vector[*argc]); + hi_s_free(vector); + if (current) hi_sdsfree(current); *argc = 0; return NULL; } @@ -1042,13 +1059,13 @@ sds *sdssplitargs(const char *line, int *argc) { * characters specified in the 'from' string to the corresponding character * in the 'to' array. * - * For instance: sdsmapchars(mystring, "ho", "01", 2) + * For instance: hi_sdsmapchars(mystring, "ho", "01", 2) * will have the effect of turning the string "hello" into "0ell1". * - * The function returns the sds string pointer, that is always the same + * The function returns the hisds string pointer, that is always the same * as the input pointer since no resize is needed. */ -sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) { - size_t j, i, l = sdslen(s); +hisds hi_sdsmapchars(hisds s, const char *from, const char *to, size_t setlen) { + size_t j, i, l = hi_sdslen(s); for (j = 0; j < l; j++) { for (i = 0; i < setlen; i++) { @@ -1062,26 +1079,26 @@ sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) { } /* Join an array of C strings using the specified separator (also a C string). - * Returns the result as an sds string. */ -sds sdsjoin(char **argv, int argc, char *sep) { - sds join = sdsempty(); + * Returns the result as an hisds string. */ +hisds hi_sdsjoin(char **argv, int argc, char *sep) { + hisds join = hi_sdsempty(); int j; for (j = 0; j < argc; j++) { - join = sdscat(join, argv[j]); - if (j != argc-1) join = sdscat(join,sep); + join = hi_sdscat(join, argv[j]); + if (j != argc-1) join = hi_sdscat(join,sep); } return join; } -/* Like sdsjoin, but joins an array of SDS strings. */ -sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) { - sds join = sdsempty(); +/* Like hi_sdsjoin, but joins an array of SDS strings. */ +hisds hi_sdsjoinsds(hisds *argv, int argc, const char *sep, size_t seplen) { + hisds join = hi_sdsempty(); int j; for (j = 0; j < argc; j++) { - join = sdscatsds(join, argv[j]); - if (j != argc-1) join = sdscatlen(join,sep,seplen); + join = hi_sdscatsds(join, argv[j]); + if (j != argc-1) join = hi_sdscatlen(join,sep,seplen); } return join; } @@ -1091,138 +1108,138 @@ sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) { * the overhead of function calls. Here we define these wrappers only for * the programs SDS is linked to, if they want to touch the SDS internals * even if they use a different allocator. */ -void *sds_malloc(size_t size) { return s_malloc(size); } -void *sds_realloc(void *ptr, size_t size) { return s_realloc(ptr,size); } -void sds_free(void *ptr) { s_free(ptr); } +void *hi_sds_malloc(size_t size) { return hi_s_malloc(size); } +void *hi_sds_realloc(void *ptr, size_t size) { return hi_s_realloc(ptr,size); } +void hi_sds_free(void *ptr) { hi_s_free(ptr); } -#if defined(SDS_TEST_MAIN) +#if defined(HI_SDS_TEST_MAIN) #include #include "testhelp.h" #include "limits.h" #define UNUSED(x) (void)(x) -int sdsTest(void) { +int hi_sdsTest(void) { { - sds x = sdsnew("foo"), y; + hisds x = hi_sdsnew("foo"), y; test_cond("Create a string and obtain the length", - sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0) + hi_sdslen(x) == 3 && memcmp(x,"foo\0",4) == 0) - sdsfree(x); - x = sdsnewlen("foo",2); + hi_sdsfree(x); + x = hi_sdsnewlen("foo",2); test_cond("Create a string with specified length", - sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0) + hi_sdslen(x) == 2 && memcmp(x,"fo\0",3) == 0) - x = sdscat(x,"bar"); + x = hi_sdscat(x,"bar"); test_cond("Strings concatenation", - sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0); + hi_sdslen(x) == 5 && memcmp(x,"fobar\0",6) == 0); - x = sdscpy(x,"a"); - test_cond("sdscpy() against an originally longer string", - sdslen(x) == 1 && memcmp(x,"a\0",2) == 0) + x = hi_sdscpy(x,"a"); + test_cond("hi_sdscpy() against an originally longer string", + hi_sdslen(x) == 1 && memcmp(x,"a\0",2) == 0) - x = sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk"); - test_cond("sdscpy() against an originally shorter string", - sdslen(x) == 33 && + x = hi_sdscpy(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk"); + test_cond("hi_sdscpy() against an originally shorter string", + hi_sdslen(x) == 33 && memcmp(x,"xyzxxxxxxxxxxyyyyyyyyyykkkkkkkkkk\0",33) == 0) - sdsfree(x); - x = sdscatprintf(sdsempty(),"%d",123); - test_cond("sdscatprintf() seems working in the base case", - sdslen(x) == 3 && memcmp(x,"123\0",4) == 0) + hi_sdsfree(x); + x = hi_sdscatprintf(hi_sdsempty(),"%d",123); + test_cond("hi_sdscatprintf() seems working in the base case", + hi_sdslen(x) == 3 && memcmp(x,"123\0",4) == 0) - sdsfree(x); - x = sdsnew("--"); - x = sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX); - test_cond("sdscatfmt() seems working in the base case", - sdslen(x) == 60 && + hi_sdsfree(x); + x = hi_sdsnew("--"); + x = hi_sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX); + test_cond("hi_sdscatfmt() seems working in the base case", + hi_sdslen(x) == 60 && memcmp(x,"--Hello Hi! World -9223372036854775808," "9223372036854775807--",60) == 0) printf("[%s]\n",x); - sdsfree(x); - x = sdsnew("--"); - x = sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX); - test_cond("sdscatfmt() seems working with unsigned numbers", - sdslen(x) == 35 && + hi_sdsfree(x); + x = hi_sdsnew("--"); + x = hi_sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX); + test_cond("hi_sdscatfmt() seems working with unsigned numbers", + hi_sdslen(x) == 35 && memcmp(x,"--4294967295,18446744073709551615--",35) == 0) - sdsfree(x); - x = sdsnew(" x "); - sdstrim(x," x"); - test_cond("sdstrim() works when all chars match", - sdslen(x) == 0) - - sdsfree(x); - x = sdsnew(" x "); - sdstrim(x," "); - test_cond("sdstrim() works when a single char remains", - sdslen(x) == 1 && x[0] == 'x') - - sdsfree(x); - x = sdsnew("xxciaoyyy"); - sdstrim(x,"xy"); - test_cond("sdstrim() correctly trims characters", - sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0) - - y = sdsdup(x); - sdsrange(y,1,1); - test_cond("sdsrange(...,1,1)", - sdslen(y) == 1 && memcmp(y,"i\0",2) == 0) - - sdsfree(y); - y = sdsdup(x); - sdsrange(y,1,-1); - test_cond("sdsrange(...,1,-1)", - sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) - - sdsfree(y); - y = sdsdup(x); - sdsrange(y,-2,-1); - test_cond("sdsrange(...,-2,-1)", - sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0) - - sdsfree(y); - y = sdsdup(x); - sdsrange(y,2,1); - test_cond("sdsrange(...,2,1)", - sdslen(y) == 0 && memcmp(y,"\0",1) == 0) - - sdsfree(y); - y = sdsdup(x); - sdsrange(y,1,100); - test_cond("sdsrange(...,1,100)", - sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) - - sdsfree(y); - y = sdsdup(x); - sdsrange(y,100,100); - test_cond("sdsrange(...,100,100)", - sdslen(y) == 0 && memcmp(y,"\0",1) == 0) - - sdsfree(y); - sdsfree(x); - x = sdsnew("foo"); - y = sdsnew("foa"); - test_cond("sdscmp(foo,foa)", sdscmp(x,y) > 0) - - sdsfree(y); - sdsfree(x); - x = sdsnew("bar"); - y = sdsnew("bar"); - test_cond("sdscmp(bar,bar)", sdscmp(x,y) == 0) - - sdsfree(y); - sdsfree(x); - x = sdsnew("aar"); - y = sdsnew("bar"); - test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0) - - sdsfree(y); - sdsfree(x); - x = sdsnewlen("\a\n\0foo\r",7); - y = sdscatrepr(sdsempty(),x,sdslen(x)); - test_cond("sdscatrepr(...data...)", + hi_sdsfree(x); + x = hi_sdsnew(" x "); + hi_sdstrim(x," x"); + test_cond("hi_sdstrim() works when all chars match", + hi_sdslen(x) == 0) + + hi_sdsfree(x); + x = hi_sdsnew(" x "); + hi_sdstrim(x," "); + test_cond("hi_sdstrim() works when a single char remains", + hi_sdslen(x) == 1 && x[0] == 'x') + + hi_sdsfree(x); + x = hi_sdsnew("xxciaoyyy"); + hi_sdstrim(x,"xy"); + test_cond("hi_sdstrim() correctly trims characters", + hi_sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0) + + y = hi_sdsdup(x); + hi_sdsrange(y,1,1); + test_cond("hi_sdsrange(...,1,1)", + hi_sdslen(y) == 1 && memcmp(y,"i\0",2) == 0) + + hi_sdsfree(y); + y = hi_sdsdup(x); + hi_sdsrange(y,1,-1); + test_cond("hi_sdsrange(...,1,-1)", + hi_sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) + + hi_sdsfree(y); + y = hi_sdsdup(x); + hi_sdsrange(y,-2,-1); + test_cond("hi_sdsrange(...,-2,-1)", + hi_sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0) + + hi_sdsfree(y); + y = hi_sdsdup(x); + hi_sdsrange(y,2,1); + test_cond("hi_sdsrange(...,2,1)", + hi_sdslen(y) == 0 && memcmp(y,"\0",1) == 0) + + hi_sdsfree(y); + y = hi_sdsdup(x); + hi_sdsrange(y,1,100); + test_cond("hi_sdsrange(...,1,100)", + hi_sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) + + hi_sdsfree(y); + y = hi_sdsdup(x); + hi_sdsrange(y,100,100); + test_cond("hi_sdsrange(...,100,100)", + hi_sdslen(y) == 0 && memcmp(y,"\0",1) == 0) + + hi_sdsfree(y); + hi_sdsfree(x); + x = hi_sdsnew("foo"); + y = hi_sdsnew("foa"); + test_cond("hi_sdscmp(foo,foa)", hi_sdscmp(x,y) > 0) + + hi_sdsfree(y); + hi_sdsfree(x); + x = hi_sdsnew("bar"); + y = hi_sdsnew("bar"); + test_cond("hi_sdscmp(bar,bar)", hi_sdscmp(x,y) == 0) + + hi_sdsfree(y); + hi_sdsfree(x); + x = hi_sdsnew("aar"); + y = hi_sdsnew("bar"); + test_cond("hi_sdscmp(bar,bar)", hi_sdscmp(x,y) < 0) + + hi_sdsfree(y); + hi_sdsfree(x); + x = hi_sdsnewlen("\a\n\0foo\r",7); + y = hi_sdscatrepr(hi_sdsempty(),x,hi_sdslen(x)); + test_cond("hi_sdscatrepr(...data...)", memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0) { @@ -1230,43 +1247,43 @@ int sdsTest(void) { char *p; int step = 10, j, i; - sdsfree(x); - sdsfree(y); - x = sdsnew("0"); - test_cond("sdsnew() free/len buffers", sdslen(x) == 1 && sdsavail(x) == 0); + hi_sdsfree(x); + hi_sdsfree(y); + x = hi_sdsnew("0"); + test_cond("hi_sdsnew() free/len buffers", hi_sdslen(x) == 1 && hi_sdsavail(x) == 0); /* Run the test a few times in order to hit the first two * SDS header types. */ for (i = 0; i < 10; i++) { - int oldlen = sdslen(x); - x = sdsMakeRoomFor(x,step); - int type = x[-1]&SDS_TYPE_MASK; - - test_cond("sdsMakeRoomFor() len", sdslen(x) == oldlen); - if (type != SDS_TYPE_5) { - test_cond("sdsMakeRoomFor() free", sdsavail(x) >= step); - oldfree = sdsavail(x); + int oldlen = hi_sdslen(x); + x = hi_sdsMakeRoomFor(x,step); + int type = x[-1]&HI_SDS_TYPE_MASK; + + test_cond("sdsMakeRoomFor() len", hi_sdslen(x) == oldlen); + if (type != HI_SDS_TYPE_5) { + test_cond("hi_sdsMakeRoomFor() free", hi_sdsavail(x) >= step); + oldfree = hi_sdsavail(x); } p = x+oldlen; for (j = 0; j < step; j++) { p[j] = 'A'+j; } - sdsIncrLen(x,step); + hi_sdsIncrLen(x,step); } - test_cond("sdsMakeRoomFor() content", + test_cond("hi_sdsMakeRoomFor() content", memcmp("0ABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJ",x,101) == 0); - test_cond("sdsMakeRoomFor() final length",sdslen(x)==101); + test_cond("sdsMakeRoomFor() final length",hi_sdslen(x)==101); - sdsfree(x); + hi_sdsfree(x); } } - test_report() + test_report(); return 0; } #endif -#ifdef SDS_TEST_MAIN +#ifdef HI_SDS_TEST_MAIN int main(void) { - return sdsTest(); + return hi_sdsTest(); } #endif diff --git a/redis.submodule/deps/hiredis/sds.h b/redis.submodule/deps/hiredis/sds.h index 13be75a..573d6dd 100644 --- a/redis.submodule/deps/hiredis/sds.h +++ b/redis.submodule/deps/hiredis/sds.h @@ -30,244 +30,249 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#ifndef __SDS_H -#define __SDS_H +#ifndef HIREDIS_SDS_H +#define HIREDIS_SDS_H -#define SDS_MAX_PREALLOC (1024*1024) +#define HI_SDS_MAX_PREALLOC (1024*1024) +#ifdef _MSC_VER +#define __attribute__(x) +typedef long long ssize_t; +#define SSIZE_MAX (LLONG_MAX >> 1) +#endif #include #include #include -typedef char *sds; +typedef char *hisds; /* Note: sdshdr5 is never used, we just access the flags byte directly. * However is here to document the layout of type 5 SDS strings. */ -struct __attribute__ ((__packed__)) sdshdr5 { +struct __attribute__ ((__packed__)) hisdshdr5 { unsigned char flags; /* 3 lsb of type, and 5 msb of string length */ char buf[]; }; -struct __attribute__ ((__packed__)) sdshdr8 { +struct __attribute__ ((__packed__)) hisdshdr8 { uint8_t len; /* used */ uint8_t alloc; /* excluding the header and null terminator */ unsigned char flags; /* 3 lsb of type, 5 unused bits */ char buf[]; }; -struct __attribute__ ((__packed__)) sdshdr16 { +struct __attribute__ ((__packed__)) hisdshdr16 { uint16_t len; /* used */ uint16_t alloc; /* excluding the header and null terminator */ unsigned char flags; /* 3 lsb of type, 5 unused bits */ char buf[]; }; -struct __attribute__ ((__packed__)) sdshdr32 { +struct __attribute__ ((__packed__)) hisdshdr32 { uint32_t len; /* used */ uint32_t alloc; /* excluding the header and null terminator */ unsigned char flags; /* 3 lsb of type, 5 unused bits */ char buf[]; }; -struct __attribute__ ((__packed__)) sdshdr64 { +struct __attribute__ ((__packed__)) hisdshdr64 { uint64_t len; /* used */ uint64_t alloc; /* excluding the header and null terminator */ unsigned char flags; /* 3 lsb of type, 5 unused bits */ char buf[]; }; -#define SDS_TYPE_5 0 -#define SDS_TYPE_8 1 -#define SDS_TYPE_16 2 -#define SDS_TYPE_32 3 -#define SDS_TYPE_64 4 -#define SDS_TYPE_MASK 7 -#define SDS_TYPE_BITS 3 -#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))); -#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T)))) -#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS) +#define HI_SDS_TYPE_5 0 +#define HI_SDS_TYPE_8 1 +#define HI_SDS_TYPE_16 2 +#define HI_SDS_TYPE_32 3 +#define HI_SDS_TYPE_64 4 +#define HI_SDS_TYPE_MASK 7 +#define HI_SDS_TYPE_BITS 3 +#define HI_SDS_HDR_VAR(T,s) struct hisdshdr##T *sh = (struct hisdshdr##T *)((s)-(sizeof(struct hisdshdr##T))); +#define HI_SDS_HDR(T,s) ((struct hisdshdr##T *)((s)-(sizeof(struct hisdshdr##T)))) +#define HI_SDS_TYPE_5_LEN(f) ((f)>>HI_SDS_TYPE_BITS) -static inline size_t sdslen(const sds s) { +static inline size_t hi_sdslen(const hisds s) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: - return SDS_TYPE_5_LEN(flags); - case SDS_TYPE_8: - return SDS_HDR(8,s)->len; - case SDS_TYPE_16: - return SDS_HDR(16,s)->len; - case SDS_TYPE_32: - return SDS_HDR(32,s)->len; - case SDS_TYPE_64: - return SDS_HDR(64,s)->len; + switch(flags & HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: + return HI_SDS_TYPE_5_LEN(flags); + case HI_SDS_TYPE_8: + return HI_SDS_HDR(8,s)->len; + case HI_SDS_TYPE_16: + return HI_SDS_HDR(16,s)->len; + case HI_SDS_TYPE_32: + return HI_SDS_HDR(32,s)->len; + case HI_SDS_TYPE_64: + return HI_SDS_HDR(64,s)->len; } return 0; } -static inline size_t sdsavail(const sds s) { +static inline size_t hi_sdsavail(const hisds s) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: { + switch(flags&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: { return 0; } - case SDS_TYPE_8: { - SDS_HDR_VAR(8,s); + case HI_SDS_TYPE_8: { + HI_SDS_HDR_VAR(8,s); return sh->alloc - sh->len; } - case SDS_TYPE_16: { - SDS_HDR_VAR(16,s); + case HI_SDS_TYPE_16: { + HI_SDS_HDR_VAR(16,s); return sh->alloc - sh->len; } - case SDS_TYPE_32: { - SDS_HDR_VAR(32,s); + case HI_SDS_TYPE_32: { + HI_SDS_HDR_VAR(32,s); return sh->alloc - sh->len; } - case SDS_TYPE_64: { - SDS_HDR_VAR(64,s); + case HI_SDS_TYPE_64: { + HI_SDS_HDR_VAR(64,s); return sh->alloc - sh->len; } } return 0; } -static inline void sdssetlen(sds s, size_t newlen) { +static inline void hi_sdssetlen(hisds s, size_t newlen) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: + switch(flags&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); + *fp = (unsigned char)(HI_SDS_TYPE_5 | (newlen << HI_SDS_TYPE_BITS)); } break; - case SDS_TYPE_8: - SDS_HDR(8,s)->len = newlen; + case HI_SDS_TYPE_8: + HI_SDS_HDR(8,s)->len = (uint8_t)newlen; break; - case SDS_TYPE_16: - SDS_HDR(16,s)->len = newlen; + case HI_SDS_TYPE_16: + HI_SDS_HDR(16,s)->len = (uint16_t)newlen; break; - case SDS_TYPE_32: - SDS_HDR(32,s)->len = newlen; + case HI_SDS_TYPE_32: + HI_SDS_HDR(32,s)->len = (uint32_t)newlen; break; - case SDS_TYPE_64: - SDS_HDR(64,s)->len = newlen; + case HI_SDS_TYPE_64: + HI_SDS_HDR(64,s)->len = (uint64_t)newlen; break; } } -static inline void sdsinclen(sds s, size_t inc) { +static inline void hi_sdsinclen(hisds s, size_t inc) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: + switch(flags&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - unsigned char newlen = SDS_TYPE_5_LEN(flags)+inc; - *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); + unsigned char newlen = HI_SDS_TYPE_5_LEN(flags)+(unsigned char)inc; + *fp = HI_SDS_TYPE_5 | (newlen << HI_SDS_TYPE_BITS); } break; - case SDS_TYPE_8: - SDS_HDR(8,s)->len += inc; + case HI_SDS_TYPE_8: + HI_SDS_HDR(8,s)->len += (uint8_t)inc; break; - case SDS_TYPE_16: - SDS_HDR(16,s)->len += inc; + case HI_SDS_TYPE_16: + HI_SDS_HDR(16,s)->len += (uint16_t)inc; break; - case SDS_TYPE_32: - SDS_HDR(32,s)->len += inc; + case HI_SDS_TYPE_32: + HI_SDS_HDR(32,s)->len += (uint32_t)inc; break; - case SDS_TYPE_64: - SDS_HDR(64,s)->len += inc; + case HI_SDS_TYPE_64: + HI_SDS_HDR(64,s)->len += (uint64_t)inc; break; } } -/* sdsalloc() = sdsavail() + sdslen() */ -static inline size_t sdsalloc(const sds s) { +/* hi_sdsalloc() = hi_sdsavail() + hi_sdslen() */ +static inline size_t hi_sdsalloc(const hisds s) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: - return SDS_TYPE_5_LEN(flags); - case SDS_TYPE_8: - return SDS_HDR(8,s)->alloc; - case SDS_TYPE_16: - return SDS_HDR(16,s)->alloc; - case SDS_TYPE_32: - return SDS_HDR(32,s)->alloc; - case SDS_TYPE_64: - return SDS_HDR(64,s)->alloc; + switch(flags & HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: + return HI_SDS_TYPE_5_LEN(flags); + case HI_SDS_TYPE_8: + return HI_SDS_HDR(8,s)->alloc; + case HI_SDS_TYPE_16: + return HI_SDS_HDR(16,s)->alloc; + case HI_SDS_TYPE_32: + return HI_SDS_HDR(32,s)->alloc; + case HI_SDS_TYPE_64: + return HI_SDS_HDR(64,s)->alloc; } return 0; } -static inline void sdssetalloc(sds s, size_t newlen) { +static inline void hi_sdssetalloc(hisds s, size_t newlen) { unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: + switch(flags&HI_SDS_TYPE_MASK) { + case HI_SDS_TYPE_5: /* Nothing to do, this type has no total allocation info. */ break; - case SDS_TYPE_8: - SDS_HDR(8,s)->alloc = newlen; + case HI_SDS_TYPE_8: + HI_SDS_HDR(8,s)->alloc = (uint8_t)newlen; break; - case SDS_TYPE_16: - SDS_HDR(16,s)->alloc = newlen; + case HI_SDS_TYPE_16: + HI_SDS_HDR(16,s)->alloc = (uint16_t)newlen; break; - case SDS_TYPE_32: - SDS_HDR(32,s)->alloc = newlen; + case HI_SDS_TYPE_32: + HI_SDS_HDR(32,s)->alloc = (uint32_t)newlen; break; - case SDS_TYPE_64: - SDS_HDR(64,s)->alloc = newlen; + case HI_SDS_TYPE_64: + HI_SDS_HDR(64,s)->alloc = (uint64_t)newlen; break; } } -sds sdsnewlen(const void *init, size_t initlen); -sds sdsnew(const char *init); -sds sdsempty(void); -sds sdsdup(const sds s); -void sdsfree(sds s); -sds sdsgrowzero(sds s, size_t len); -sds sdscatlen(sds s, const void *t, size_t len); -sds sdscat(sds s, const char *t); -sds sdscatsds(sds s, const sds t); -sds sdscpylen(sds s, const char *t, size_t len); -sds sdscpy(sds s, const char *t); +hisds hi_sdsnewlen(const void *init, size_t initlen); +hisds hi_sdsnew(const char *init); +hisds hi_sdsempty(void); +hisds hi_sdsdup(const hisds s); +void hi_sdsfree(hisds s); +hisds hi_sdsgrowzero(hisds s, size_t len); +hisds hi_sdscatlen(hisds s, const void *t, size_t len); +hisds hi_sdscat(hisds s, const char *t); +hisds hi_sdscatsds(hisds s, const hisds t); +hisds hi_sdscpylen(hisds s, const char *t, size_t len); +hisds hi_sdscpy(hisds s, const char *t); -sds sdscatvprintf(sds s, const char *fmt, va_list ap); +hisds hi_sdscatvprintf(hisds s, const char *fmt, va_list ap); #ifdef __GNUC__ -sds sdscatprintf(sds s, const char *fmt, ...) +hisds hi_sdscatprintf(hisds s, const char *fmt, ...) __attribute__((format(printf, 2, 3))); #else -sds sdscatprintf(sds s, const char *fmt, ...); +hisds hi_sdscatprintf(hisds s, const char *fmt, ...); #endif -sds sdscatfmt(sds s, char const *fmt, ...); -sds sdstrim(sds s, const char *cset); -void sdsrange(sds s, int start, int end); -void sdsupdatelen(sds s); -void sdsclear(sds s); -int sdscmp(const sds s1, const sds s2); -sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count); -void sdsfreesplitres(sds *tokens, int count); -void sdstolower(sds s); -void sdstoupper(sds s); -sds sdsfromlonglong(long long value); -sds sdscatrepr(sds s, const char *p, size_t len); -sds *sdssplitargs(const char *line, int *argc); -sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen); -sds sdsjoin(char **argv, int argc, char *sep); -sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen); +hisds hi_sdscatfmt(hisds s, char const *fmt, ...); +hisds hi_sdstrim(hisds s, const char *cset); +int hi_sdsrange(hisds s, ssize_t start, ssize_t end); +void hi_sdsupdatelen(hisds s); +void hi_sdsclear(hisds s); +int hi_sdscmp(const hisds s1, const hisds s2); +hisds *hi_sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count); +void hi_sdsfreesplitres(hisds *tokens, int count); +void hi_sdstolower(hisds s); +void hi_sdstoupper(hisds s); +hisds hi_sdsfromlonglong(long long value); +hisds hi_sdscatrepr(hisds s, const char *p, size_t len); +hisds *hi_sdssplitargs(const char *line, int *argc); +hisds hi_sdsmapchars(hisds s, const char *from, const char *to, size_t setlen); +hisds hi_sdsjoin(char **argv, int argc, char *sep); +hisds hi_sdsjoinsds(hisds *argv, int argc, const char *sep, size_t seplen); /* Low level functions exposed to the user API */ -sds sdsMakeRoomFor(sds s, size_t addlen); -void sdsIncrLen(sds s, int incr); -sds sdsRemoveFreeSpace(sds s); -size_t sdsAllocSize(sds s); -void *sdsAllocPtr(sds s); +hisds hi_sdsMakeRoomFor(hisds s, size_t addlen); +void hi_sdsIncrLen(hisds s, int incr); +hisds hi_sdsRemoveFreeSpace(hisds s); +size_t hi_sdsAllocSize(hisds s); +void *hi_sdsAllocPtr(hisds s); /* Export the allocator used by SDS to the program using SDS. * Sometimes the program SDS is linked to, may use a different set of * allocators, but may want to allocate or free things that SDS will * respectively free or allocate. */ -void *sds_malloc(size_t size); -void *sds_realloc(void *ptr, size_t size); -void sds_free(void *ptr); +void *hi_sds_malloc(size_t size); +void *hi_sds_realloc(void *ptr, size_t size); +void hi_sds_free(void *ptr); #ifdef REDIS_TEST -int sdsTest(int argc, char *argv[]); +int hi_sdsTest(int argc, char *argv[]); #endif -#endif +#endif /* HIREDIS_SDS_H */ diff --git a/redis.submodule/deps/hiredis/sdsalloc.h b/redis.submodule/deps/hiredis/sdsalloc.h index f43023c..c9dcc3d 100644 --- a/redis.submodule/deps/hiredis/sdsalloc.h +++ b/redis.submodule/deps/hiredis/sdsalloc.h @@ -37,6 +37,8 @@ * the include of your alternate allocator if needed (not needed in order * to use the default libc allocator). */ -#define s_malloc malloc -#define s_realloc realloc -#define s_free free +#include "alloc.h" + +#define hi_s_malloc hi_malloc +#define hi_s_realloc hi_realloc +#define hi_s_free hi_free diff --git a/redis.submodule/deps/hiredis/sdscompat.h b/redis.submodule/deps/hiredis/sdscompat.h new file mode 100644 index 0000000..e5a2574 --- /dev/null +++ b/redis.submodule/deps/hiredis/sdscompat.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2020, Michael Grunder + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * SDS compatibility header. + * + * This simple file maps sds types and calls to their unique hiredis symbol names. + * It's useful when we build Hiredis as a dependency of Redis and want to call + * Hiredis' sds symbols rather than the ones built into Redis, as the libraries + * have slightly diverged and could cause hard to track down ABI incompatibility + * bugs. + * + */ + +#ifndef HIREDIS_SDS_COMPAT +#define HIREDIS_SDS_COMPAT + +#define sds hisds + +#define sdslen hi_sdslen +#define sdsavail hi_sdsavail +#define sdssetlen hi_sdssetlen +#define sdsinclen hi_sdsinclen +#define sdsalloc hi_sdsalloc +#define sdssetalloc hi_sdssetalloc + +#define sdsAllocPtr hi_sdsAllocPtr +#define sdsAllocSize hi_sdsAllocSize +#define sdscat hi_sdscat +#define sdscatfmt hi_sdscatfmt +#define sdscatlen hi_sdscatlen +#define sdscatprintf hi_sdscatprintf +#define sdscatrepr hi_sdscatrepr +#define sdscatsds hi_sdscatsds +#define sdscatvprintf hi_sdscatvprintf +#define sdsclear hi_sdsclear +#define sdscmp hi_sdscmp +#define sdscpy hi_sdscpy +#define sdscpylen hi_sdscpylen +#define sdsdup hi_sdsdup +#define sdsempty hi_sdsempty +#define sds_free hi_sds_free +#define sdsfree hi_sdsfree +#define sdsfreesplitres hi_sdsfreesplitres +#define sdsfromlonglong hi_sdsfromlonglong +#define sdsgrowzero hi_sdsgrowzero +#define sdsIncrLen hi_sdsIncrLen +#define sdsjoin hi_sdsjoin +#define sdsjoinsds hi_sdsjoinsds +#define sdsll2str hi_sdsll2str +#define sdsMakeRoomFor hi_sdsMakeRoomFor +#define sds_malloc hi_sds_malloc +#define sdsmapchars hi_sdsmapchars +#define sdsnew hi_sdsnew +#define sdsnewlen hi_sdsnewlen +#define sdsrange hi_sdsrange +#define sds_realloc hi_sds_realloc +#define sdsRemoveFreeSpace hi_sdsRemoveFreeSpace +#define sdssplitargs hi_sdssplitargs +#define sdssplitlen hi_sdssplitlen +#define sdstolower hi_sdstolower +#define sdstoupper hi_sdstoupper +#define sdstrim hi_sdstrim +#define sdsull2str hi_sdsull2str +#define sdsupdatelen hi_sdsupdatelen + +#endif /* HIREDIS_SDS_COMPAT */ diff --git a/redis.submodule/deps/hiredis/sockcompat.c b/redis.submodule/deps/hiredis/sockcompat.c new file mode 100644 index 0000000..f99d14b --- /dev/null +++ b/redis.submodule/deps/hiredis/sockcompat.c @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2019, Marcus Geelnard + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define REDIS_SOCKCOMPAT_IMPLEMENTATION +#include "sockcompat.h" + +#ifdef _WIN32 +static int _wsaErrorToErrno(int err) { + switch (err) { + case WSAEWOULDBLOCK: + return EWOULDBLOCK; + case WSAEINPROGRESS: + return EINPROGRESS; + case WSAEALREADY: + return EALREADY; + case WSAENOTSOCK: + return ENOTSOCK; + case WSAEDESTADDRREQ: + return EDESTADDRREQ; + case WSAEMSGSIZE: + return EMSGSIZE; + case WSAEPROTOTYPE: + return EPROTOTYPE; + case WSAENOPROTOOPT: + return ENOPROTOOPT; + case WSAEPROTONOSUPPORT: + return EPROTONOSUPPORT; + case WSAEOPNOTSUPP: + return EOPNOTSUPP; + case WSAEAFNOSUPPORT: + return EAFNOSUPPORT; + case WSAEADDRINUSE: + return EADDRINUSE; + case WSAEADDRNOTAVAIL: + return EADDRNOTAVAIL; + case WSAENETDOWN: + return ENETDOWN; + case WSAENETUNREACH: + return ENETUNREACH; + case WSAENETRESET: + return ENETRESET; + case WSAECONNABORTED: + return ECONNABORTED; + case WSAECONNRESET: + return ECONNRESET; + case WSAENOBUFS: + return ENOBUFS; + case WSAEISCONN: + return EISCONN; + case WSAENOTCONN: + return ENOTCONN; + case WSAETIMEDOUT: + return ETIMEDOUT; + case WSAECONNREFUSED: + return ECONNREFUSED; + case WSAELOOP: + return ELOOP; + case WSAENAMETOOLONG: + return ENAMETOOLONG; + case WSAEHOSTUNREACH: + return EHOSTUNREACH; + case WSAENOTEMPTY: + return ENOTEMPTY; + default: + /* We just return a generic I/O error if we could not find a relevant error. */ + return EIO; + } +} + +static void _updateErrno(int success) { + errno = success ? 0 : _wsaErrorToErrno(WSAGetLastError()); +} + +static int _initWinsock() { + static int s_initialized = 0; + if (!s_initialized) { + static WSADATA wsadata; + int err = WSAStartup(MAKEWORD(2,2), &wsadata); + if (err != 0) { + errno = _wsaErrorToErrno(err); + return 0; + } + s_initialized = 1; + } + return 1; +} + +int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res) { + /* Note: This function is likely to be called before other functions, so run init here. */ + if (!_initWinsock()) { + return EAI_FAIL; + } + + switch (getaddrinfo(node, service, hints, res)) { + case 0: return 0; + case WSATRY_AGAIN: return EAI_AGAIN; + case WSAEINVAL: return EAI_BADFLAGS; + case WSAEAFNOSUPPORT: return EAI_FAMILY; + case WSA_NOT_ENOUGH_MEMORY: return EAI_MEMORY; + case WSAHOST_NOT_FOUND: return EAI_NONAME; + case WSATYPE_NOT_FOUND: return EAI_SERVICE; + case WSAESOCKTNOSUPPORT: return EAI_SOCKTYPE; + default: return EAI_FAIL; /* Including WSANO_RECOVERY */ + } +} + +const char *win32_gai_strerror(int errcode) { + switch (errcode) { + case 0: errcode = 0; break; + case EAI_AGAIN: errcode = WSATRY_AGAIN; break; + case EAI_BADFLAGS: errcode = WSAEINVAL; break; + case EAI_FAMILY: errcode = WSAEAFNOSUPPORT; break; + case EAI_MEMORY: errcode = WSA_NOT_ENOUGH_MEMORY; break; + case EAI_NONAME: errcode = WSAHOST_NOT_FOUND; break; + case EAI_SERVICE: errcode = WSATYPE_NOT_FOUND; break; + case EAI_SOCKTYPE: errcode = WSAESOCKTNOSUPPORT; break; + default: errcode = WSANO_RECOVERY; break; /* Including EAI_FAIL */ + } + return gai_strerror(errcode); +} + +void win32_freeaddrinfo(struct addrinfo *res) { + freeaddrinfo(res); +} + +SOCKET win32_socket(int domain, int type, int protocol) { + SOCKET s; + + /* Note: This function is likely to be called before other functions, so run init here. */ + if (!_initWinsock()) { + return INVALID_SOCKET; + } + + _updateErrno((s = socket(domain, type, protocol)) != INVALID_SOCKET); + return s; +} + +int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp) { + int ret = ioctlsocket(fd, (long)request, argp); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) { + int ret = bind(sockfd, addr, addrlen); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) { + int ret = connect(sockfd, addr, addrlen); + _updateErrno(ret != SOCKET_ERROR); + + /* For Winsock connect(), the WSAEWOULDBLOCK error means the same thing as + * EINPROGRESS for POSIX connect(), so we do that translation to keep POSIX + * logic consistent. */ + if (errno == EWOULDBLOCK) { + errno = EINPROGRESS; + } + + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen) { + int ret = 0; + if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) { + if (*optlen >= sizeof (struct timeval)) { + struct timeval *tv = optval; + DWORD timeout = 0; + socklen_t dwlen = 0; + ret = getsockopt(sockfd, level, optname, (char *)&timeout, &dwlen); + tv->tv_sec = timeout / 1000; + tv->tv_usec = (timeout * 1000) % 1000000; + } else { + ret = WSAEFAULT; + } + *optlen = sizeof (struct timeval); + } else { + ret = getsockopt(sockfd, level, optname, (char*)optval, optlen); + } + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen) { + int ret = 0; + if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) { + const struct timeval *tv = optval; + DWORD timeout = tv->tv_sec * 1000 + tv->tv_usec / 1000; + ret = setsockopt(sockfd, level, optname, (const char*)&timeout, sizeof(DWORD)); + } else { + ret = setsockopt(sockfd, level, optname, (const char*)optval, optlen); + } + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_close(SOCKET fd) { + int ret = closesocket(fd); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags) { + int ret = recv(sockfd, (char*)buf, (int)len, flags); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags) { + int ret = send(sockfd, (const char*)buf, (int)len, flags); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout) { + int ret = WSAPoll(fds, nfds, timeout); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} +#endif /* _WIN32 */ diff --git a/redis.submodule/deps/hiredis/sockcompat.h b/redis.submodule/deps/hiredis/sockcompat.h new file mode 100644 index 0000000..85810e8 --- /dev/null +++ b/redis.submodule/deps/hiredis/sockcompat.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2019, Marcus Geelnard + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SOCKCOMPAT_H +#define __SOCKCOMPAT_H + +#ifndef _WIN32 +/* For POSIX systems we use the standard BSD socket API. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#else +/* For Windows we use winsock. */ +#undef _WIN32_WINNT +#define _WIN32_WINNT 0x0600 /* To get WSAPoll etc. */ +#include +#include +#include +#include + +#ifdef _MSC_VER +typedef long long ssize_t; +#endif + +/* Emulate the parts of the BSD socket API that we need (override the winsock signatures). */ +int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res); +const char *win32_gai_strerror(int errcode); +void win32_freeaddrinfo(struct addrinfo *res); +SOCKET win32_socket(int domain, int type, int protocol); +int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp); +int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen); +int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen); +int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen); +int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen); +int win32_close(SOCKET fd); +ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags); +ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags); +typedef ULONG nfds_t; +int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout); + +#ifndef REDIS_SOCKCOMPAT_IMPLEMENTATION +#define getaddrinfo(node, service, hints, res) win32_getaddrinfo(node, service, hints, res) +#undef gai_strerror +#define gai_strerror(errcode) win32_gai_strerror(errcode) +#define freeaddrinfo(res) win32_freeaddrinfo(res) +#define socket(domain, type, protocol) win32_socket(domain, type, protocol) +#define ioctl(fd, request, argp) win32_ioctl(fd, request, argp) +#define bind(sockfd, addr, addrlen) win32_bind(sockfd, addr, addrlen) +#define connect(sockfd, addr, addrlen) win32_connect(sockfd, addr, addrlen) +#define getsockopt(sockfd, level, optname, optval, optlen) win32_getsockopt(sockfd, level, optname, optval, optlen) +#define setsockopt(sockfd, level, optname, optval, optlen) win32_setsockopt(sockfd, level, optname, optval, optlen) +#define close(fd) win32_close(fd) +#define recv(sockfd, buf, len, flags) win32_recv(sockfd, buf, len, flags) +#define send(sockfd, buf, len, flags) win32_send(sockfd, buf, len, flags) +#define poll(fds, nfds, timeout) win32_poll(fds, nfds, timeout) +#endif /* REDIS_SOCKCOMPAT_IMPLEMENTATION */ +#endif /* _WIN32 */ + +#endif /* __SOCKCOMPAT_H */ diff --git a/redis.submodule/deps/hiredis/ssl.c b/redis.submodule/deps/hiredis/ssl.c new file mode 100644 index 0000000..fe9a2fd --- /dev/null +++ b/redis.submodule/deps/hiredis/ssl.c @@ -0,0 +1,526 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * Copyright (c) 2019, Redis Labs + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "hiredis.h" +#include "async.h" + +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif + +#include +#include + +#include "win32.h" +#include "async_private.h" +#include "hiredis_ssl.h" + +void __redisSetError(redisContext *c, int type, const char *str); + +struct redisSSLContext { + /* Associated OpenSSL SSL_CTX as created by redisCreateSSLContext() */ + SSL_CTX *ssl_ctx; + + /* Requested SNI, or NULL */ + char *server_name; +}; + +/* The SSL connection context is attached to SSL/TLS connections as a privdata. */ +typedef struct redisSSL { + /** + * OpenSSL SSL object. + */ + SSL *ssl; + + /** + * SSL_write() requires to be called again with the same arguments it was + * previously called with in the event of an SSL_read/SSL_write situation + */ + size_t lastLen; + + /** Whether the SSL layer requires read (possibly before a write) */ + int wantRead; + + /** + * Whether a write was requested prior to a read. If set, the write() + * should resume whenever a read takes place, if possible + */ + int pendingWrite; +} redisSSL; + +/* Forward declaration */ +redisContextFuncs redisContextSSLFuncs; + +/** + * OpenSSL global initialization and locking handling callbacks. + * Note that this is only required for OpenSSL < 1.1.0. + */ + +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#define HIREDIS_USE_CRYPTO_LOCKS +#endif + +#ifdef HIREDIS_USE_CRYPTO_LOCKS +#ifdef _WIN32 +typedef CRITICAL_SECTION sslLockType; +static void sslLockInit(sslLockType* l) { + InitializeCriticalSection(l); +} +static void sslLockAcquire(sslLockType* l) { + EnterCriticalSection(l); +} +static void sslLockRelease(sslLockType* l) { + LeaveCriticalSection(l); +} +#else +typedef pthread_mutex_t sslLockType; +static void sslLockInit(sslLockType *l) { + pthread_mutex_init(l, NULL); +} +static void sslLockAcquire(sslLockType *l) { + pthread_mutex_lock(l); +} +static void sslLockRelease(sslLockType *l) { + pthread_mutex_unlock(l); +} +#endif + +static sslLockType* ossl_locks; + +static void opensslDoLock(int mode, int lkid, const char *f, int line) { + sslLockType *l = ossl_locks + lkid; + + if (mode & CRYPTO_LOCK) { + sslLockAcquire(l); + } else { + sslLockRelease(l); + } + + (void)f; + (void)line; +} + +static int initOpensslLocks(void) { + unsigned ii, nlocks; + if (CRYPTO_get_locking_callback() != NULL) { + /* Someone already set the callback before us. Don't destroy it! */ + return REDIS_OK; + } + nlocks = CRYPTO_num_locks(); + ossl_locks = hi_malloc(sizeof(*ossl_locks) * nlocks); + if (ossl_locks == NULL) + return REDIS_ERR; + + for (ii = 0; ii < nlocks; ii++) { + sslLockInit(ossl_locks + ii); + } + CRYPTO_set_locking_callback(opensslDoLock); + return REDIS_OK; +} +#endif /* HIREDIS_USE_CRYPTO_LOCKS */ + +int redisInitOpenSSL(void) +{ + SSL_library_init(); +#ifdef HIREDIS_USE_CRYPTO_LOCKS + initOpensslLocks(); +#endif + + return REDIS_OK; +} + +/** + * redisSSLContext helper context destruction. + */ + +const char *redisSSLContextGetError(redisSSLContextError error) +{ + switch (error) { + case REDIS_SSL_CTX_NONE: + return "No Error"; + case REDIS_SSL_CTX_CREATE_FAILED: + return "Failed to create OpenSSL SSL_CTX"; + case REDIS_SSL_CTX_CERT_KEY_REQUIRED: + return "Client cert and key must both be specified or skipped"; + case REDIS_SSL_CTX_CA_CERT_LOAD_FAILED: + return "Failed to load CA Certificate or CA Path"; + case REDIS_SSL_CTX_CLIENT_CERT_LOAD_FAILED: + return "Failed to load client certificate"; + case REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED: + return "Failed to load private key"; + default: + return "Unknown error code"; + } +} + +void redisFreeSSLContext(redisSSLContext *ctx) +{ + if (!ctx) + return; + + if (ctx->server_name) { + hi_free(ctx->server_name); + ctx->server_name = NULL; + } + + if (ctx->ssl_ctx) { + SSL_CTX_free(ctx->ssl_ctx); + ctx->ssl_ctx = NULL; + } + + hi_free(ctx); +} + + +/** + * redisSSLContext helper context initialization. + */ + +redisSSLContext *redisCreateSSLContext(const char *cacert_filename, const char *capath, + const char *cert_filename, const char *private_key_filename, + const char *server_name, redisSSLContextError *error) +{ + redisSSLContext *ctx = hi_calloc(1, sizeof(redisSSLContext)); + if (ctx == NULL) + goto error; + + ctx->ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + if (!ctx->ssl_ctx) { + if (error) *error = REDIS_SSL_CTX_CREATE_FAILED; + goto error; + } + + SSL_CTX_set_options(ctx->ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); + SSL_CTX_set_verify(ctx->ssl_ctx, SSL_VERIFY_PEER, NULL); + + if ((cert_filename != NULL && private_key_filename == NULL) || + (private_key_filename != NULL && cert_filename == NULL)) { + if (error) *error = REDIS_SSL_CTX_CERT_KEY_REQUIRED; + goto error; + } + + if (capath || cacert_filename) { + if (!SSL_CTX_load_verify_locations(ctx->ssl_ctx, cacert_filename, capath)) { + if (error) *error = REDIS_SSL_CTX_CA_CERT_LOAD_FAILED; + goto error; + } + } + + if (cert_filename) { + if (!SSL_CTX_use_certificate_chain_file(ctx->ssl_ctx, cert_filename)) { + if (error) *error = REDIS_SSL_CTX_CLIENT_CERT_LOAD_FAILED; + goto error; + } + if (!SSL_CTX_use_PrivateKey_file(ctx->ssl_ctx, private_key_filename, SSL_FILETYPE_PEM)) { + if (error) *error = REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED; + goto error; + } + } + + if (server_name) + ctx->server_name = hi_strdup(server_name); + + return ctx; + +error: + redisFreeSSLContext(ctx); + return NULL; +} + +/** + * SSL Connection initialization. + */ + + +static int redisSSLConnect(redisContext *c, SSL *ssl) { + if (c->privctx) { + __redisSetError(c, REDIS_ERR_OTHER, "redisContext was already associated"); + return REDIS_ERR; + } + + redisSSL *rssl = hi_calloc(1, sizeof(redisSSL)); + if (rssl == NULL) { + __redisSetError(c, REDIS_ERR_OOM, "Out of memory"); + return REDIS_ERR; + } + + c->funcs = &redisContextSSLFuncs; + rssl->ssl = ssl; + + SSL_set_mode(rssl->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); + SSL_set_fd(rssl->ssl, c->fd); + SSL_set_connect_state(rssl->ssl); + + ERR_clear_error(); + int rv = SSL_connect(rssl->ssl); + if (rv == 1) { + c->privctx = rssl; + return REDIS_OK; + } + + rv = SSL_get_error(rssl->ssl, rv); + if (((c->flags & REDIS_BLOCK) == 0) && + (rv == SSL_ERROR_WANT_READ || rv == SSL_ERROR_WANT_WRITE)) { + c->privctx = rssl; + return REDIS_OK; + } + + if (c->err == 0) { + char err[512]; + if (rv == SSL_ERROR_SYSCALL) + snprintf(err,sizeof(err)-1,"SSL_connect failed: %s",strerror(errno)); + else { + unsigned long e = ERR_peek_last_error(); + snprintf(err,sizeof(err)-1,"SSL_connect failed: %s", + ERR_reason_error_string(e)); + } + __redisSetError(c, REDIS_ERR_IO, err); + } + + hi_free(rssl); + return REDIS_ERR; +} + +/** + * A wrapper around redisSSLConnect() for users who manage their own context and + * create their own SSL object. + */ + +int redisInitiateSSL(redisContext *c, SSL *ssl) { + return redisSSLConnect(c, ssl); +} + +/** + * A wrapper around redisSSLConnect() for users who use redisSSLContext and don't + * manage their own SSL objects. + */ + +int redisInitiateSSLWithContext(redisContext *c, redisSSLContext *redis_ssl_ctx) +{ + if (!c || !redis_ssl_ctx) + return REDIS_ERR; + + /* We want to verify that redisSSLConnect() won't fail on this, as it will + * not own the SSL object in that case and we'll end up leaking. + */ + if (c->privctx) + return REDIS_ERR; + + SSL *ssl = SSL_new(redis_ssl_ctx->ssl_ctx); + if (!ssl) { + __redisSetError(c, REDIS_ERR_OTHER, "Couldn't create new SSL instance"); + goto error; + } + + if (redis_ssl_ctx->server_name) { + if (!SSL_set_tlsext_host_name(ssl, redis_ssl_ctx->server_name)) { + __redisSetError(c, REDIS_ERR_OTHER, "Failed to set server_name/SNI"); + goto error; + } + } + + return redisSSLConnect(c, ssl); + +error: + if (ssl) + SSL_free(ssl); + return REDIS_ERR; +} + +static int maybeCheckWant(redisSSL *rssl, int rv) { + /** + * If the error is WANT_READ or WANT_WRITE, the appropriate flags are set + * and true is returned. False is returned otherwise + */ + if (rv == SSL_ERROR_WANT_READ) { + rssl->wantRead = 1; + return 1; + } else if (rv == SSL_ERROR_WANT_WRITE) { + rssl->pendingWrite = 1; + return 1; + } else { + return 0; + } +} + +/** + * Implementation of redisContextFuncs for SSL connections. + */ + +static void redisSSLFree(void *privctx){ + redisSSL *rsc = privctx; + + if (!rsc) return; + if (rsc->ssl) { + SSL_free(rsc->ssl); + rsc->ssl = NULL; + } + hi_free(rsc); +} + +static ssize_t redisSSLRead(redisContext *c, char *buf, size_t bufcap) { + redisSSL *rssl = c->privctx; + + int nread = SSL_read(rssl->ssl, buf, bufcap); + if (nread > 0) { + return nread; + } else if (nread == 0) { + __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection"); + return -1; + } else { + int err = SSL_get_error(rssl->ssl, nread); + if (c->flags & REDIS_BLOCK) { + /** + * In blocking mode, we should never end up in a situation where + * we get an error without it being an actual error, except + * in the case of EINTR, which can be spuriously received from + * debuggers or whatever. + */ + if (errno == EINTR) { + return 0; + } else { + const char *msg = NULL; + if (errno == EAGAIN) { + msg = "Resource temporarily unavailable"; + } + __redisSetError(c, REDIS_ERR_IO, msg); + return -1; + } + } + + /** + * We can very well get an EWOULDBLOCK/EAGAIN, however + */ + if (maybeCheckWant(rssl, err)) { + return 0; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } +} + +static ssize_t redisSSLWrite(redisContext *c) { + redisSSL *rssl = c->privctx; + + size_t len = rssl->lastLen ? rssl->lastLen : hi_sdslen(c->obuf); + int rv = SSL_write(rssl->ssl, c->obuf, len); + + if (rv > 0) { + rssl->lastLen = 0; + } else if (rv < 0) { + rssl->lastLen = len; + + int err = SSL_get_error(rssl->ssl, rv); + if ((c->flags & REDIS_BLOCK) == 0 && maybeCheckWant(rssl, err)) { + return 0; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } + return rv; +} + +static void redisSSLAsyncRead(redisAsyncContext *ac) { + int rv; + redisSSL *rssl = ac->c.privctx; + redisContext *c = &ac->c; + + rssl->wantRead = 0; + + if (rssl->pendingWrite) { + int done; + + /* This is probably just a write event */ + rssl->pendingWrite = 0; + rv = redisBufferWrite(c, &done); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + return; + } else if (!done) { + _EL_ADD_WRITE(ac); + } + } + + rv = redisBufferRead(c); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + } else { + _EL_ADD_READ(ac); + redisProcessCallbacks(ac); + } +} + +static void redisSSLAsyncWrite(redisAsyncContext *ac) { + int rv, done = 0; + redisSSL *rssl = ac->c.privctx; + redisContext *c = &ac->c; + + rssl->pendingWrite = 0; + rv = redisBufferWrite(c, &done); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + return; + } + + if (!done) { + if (rssl->wantRead) { + /* Need to read-before-write */ + rssl->pendingWrite = 1; + _EL_DEL_WRITE(ac); + } else { + /* No extra reads needed, just need to write more */ + _EL_ADD_WRITE(ac); + } + } else { + /* Already done! */ + _EL_DEL_WRITE(ac); + } + + /* Always reschedule a read */ + _EL_ADD_READ(ac); +} + +redisContextFuncs redisContextSSLFuncs = { + .free_privctx = redisSSLFree, + .async_read = redisSSLAsyncRead, + .async_write = redisSSLAsyncWrite, + .read = redisSSLRead, + .write = redisSSLWrite +}; + diff --git a/redis.submodule/deps/hiredis/test.c b/redis.submodule/deps/hiredis/test.c index a23d606..bdff74e 100644 --- a/redis.submodule/deps/hiredis/test.c +++ b/redis.submodule/deps/hiredis/test.c @@ -1,22 +1,30 @@ #include "fmacros.h" +#include "sockcompat.h" #include #include #include +#ifndef _WIN32 #include #include +#endif #include -#include #include #include #include #include "hiredis.h" +#include "async.h" +#ifdef HIREDIS_TEST_SSL +#include "hiredis_ssl.h" +#endif #include "net.h" +#include "win32.h" enum connection_type { CONN_TCP, CONN_UNIX, - CONN_FD + CONN_FD, + CONN_SSL }; struct config { @@ -31,17 +39,45 @@ struct config { struct { const char *path; } unix_sock; + + struct { + const char *host; + int port; + const char *ca_cert; + const char *cert; + const char *key; + } ssl; +}; + +struct privdata { + int dtor_counter; +}; + +struct pushCounters { + int nil; + int str; }; +#ifdef HIREDIS_TEST_SSL +redisSSLContext *_ssl_ctx = NULL; +#endif + /* The following lines make up our testing "framework" :) */ -static int tests = 0, fails = 0; +static int tests = 0, fails = 0, skips = 0; #define test(_s) { printf("#%02d ", ++tests); printf(_s); } #define test_cond(_c) if(_c) printf("\033[0;32mPASSED\033[0;0m\n"); else {printf("\033[0;31mFAILED\033[0;0m\n"); fails++;} +#define test_skipped() { printf("\033[01;33mSKIPPED\033[0;0m\n"); skips++; } static long long usec(void) { +#ifndef _MSC_VER struct timeval tv; gettimeofday(&tv,NULL); return (((long long)tv.tv_sec)*1000000)+tv.tv_usec; +#else + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + return (((long long)ft.dwHighDateTime << 32) | ft.dwLowDateTime) / 10; +#endif } /* The assert() calls below have side effects, so we need assert() @@ -51,6 +87,43 @@ static long long usec(void) { #define assert(e) (void)(e) #endif +/* Helper to extract Redis version information. Aborts on any failure. */ +#define REDIS_VERSION_FIELD "redis_version:" +void get_redis_version(redisContext *c, int *majorptr, int *minorptr) { + redisReply *reply; + char *eptr, *s, *e; + int major, minor; + + reply = redisCommand(c, "INFO"); + if (reply == NULL || c->err || reply->type != REDIS_REPLY_STRING) + goto abort; + if ((s = strstr(reply->str, REDIS_VERSION_FIELD)) == NULL) + goto abort; + + s += strlen(REDIS_VERSION_FIELD); + + /* We need a field terminator and at least 'x.y.z' (5) bytes of data */ + if ((e = strstr(s, "\r\n")) == NULL || (e - s) < 5) + goto abort; + + /* Extract version info */ + major = strtol(s, &eptr, 10); + if (*eptr != '.') goto abort; + minor = strtol(eptr+1, NULL, 10); + + /* Push info the caller wants */ + if (majorptr) *majorptr = major; + if (minorptr) *minorptr = minor; + + freeReplyObject(reply); + return; + +abort: + freeReplyObject(reply); + fprintf(stderr, "Error: Cannot determine Redis version, aborting\n"); + exit(1); +} + static redisContext *select_database(redisContext *c) { redisReply *reply; @@ -73,6 +146,26 @@ static redisContext *select_database(redisContext *c) { return c; } +/* Switch protocol */ +static void send_hello(redisContext *c, int version) { + redisReply *reply; + int expected; + + reply = redisCommand(c, "HELLO %d", version); + expected = version == 3 ? REDIS_REPLY_MAP : REDIS_REPLY_ARRAY; + assert(reply != NULL && reply->type == expected); + freeReplyObject(reply); +} + +/* Togggle client tracking */ +static void send_client_tracking(redisContext *c, const char *str) { + redisReply *reply; + + reply = redisCommand(c, "CLIENT TRACKING %s", str); + assert(reply != NULL && reply->type == REDIS_REPLY_STATUS); + freeReplyObject(reply); +} + static int disconnect(redisContext *c, int keep_fd) { redisReply *reply; @@ -91,11 +184,26 @@ static int disconnect(redisContext *c, int keep_fd) { return -1; } -static redisContext *connect(struct config config) { +static void do_ssl_handshake(redisContext *c) { +#ifdef HIREDIS_TEST_SSL + redisInitiateSSLWithContext(c, _ssl_ctx); + if (c->err) { + printf("SSL error: %s\n", c->errstr); + redisFree(c); + exit(1); + } +#else + (void) c; +#endif +} + +static redisContext *do_connect(struct config config) { redisContext *c = NULL; if (config.type == CONN_TCP) { c = redisConnect(config.tcp.host, config.tcp.port); + } else if (config.type == CONN_SSL) { + c = redisConnect(config.ssl.host, config.ssl.port); } else if (config.type == CONN_UNIX) { c = redisConnectUnix(config.unix_sock.path); } else if (config.type == CONN_FD) { @@ -119,9 +227,21 @@ static redisContext *connect(struct config config) { exit(1); } + if (config.type == CONN_SSL) { + do_ssl_handshake(c); + } + return select_database(c); } +static void do_reconnect(redisContext *c, struct config config) { + redisReconnect(c); + + if (config.type == CONN_SSL) { + do_ssl_handshake(c); + } +} + static void test_format_commands(void) { char *cmd; int len; @@ -130,43 +250,43 @@ static void test_format_commands(void) { len = redisFormatCommand(&cmd,"SET foo bar"); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - free(cmd); + hi_free(cmd); test("Format command with %%s string interpolation: "); len = redisFormatCommand(&cmd,"SET %s %s","foo","bar"); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - free(cmd); + hi_free(cmd); test("Format command with %%s and an empty string: "); len = redisFormatCommand(&cmd,"SET %s %s","foo",""); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(0+2)); - free(cmd); + hi_free(cmd); test("Format command with an empty string in between proper interpolations: "); len = redisFormatCommand(&cmd,"SET %s %s","","foo"); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$0\r\n\r\n$3\r\nfoo\r\n",len) == 0 && len == 4+4+(3+2)+4+(0+2)+4+(3+2)); - free(cmd); + hi_free(cmd); test("Format command with %%b string interpolation: "); len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"b\0r",(size_t)3); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nb\0r\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - free(cmd); + hi_free(cmd); test("Format command with %%b and an empty string: "); len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"",(size_t)0); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(0+2)); - free(cmd); + hi_free(cmd); test("Format command with literal %%: "); len = redisFormatCommand(&cmd,"SET %% %%"); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$1\r\n%\r\n$1\r\n%\r\n",len) == 0 && len == 4+4+(3+2)+4+(1+2)+4+(1+2)); - free(cmd); + hi_free(cmd); /* Vararg width depends on the type. These tests make sure that the * width is correctly determined using the format and subsequent varargs @@ -177,7 +297,7 @@ static void test_format_commands(void) { len = redisFormatCommand(&cmd,"key:%08" fmt " str:%s", value, "hello"); \ test_cond(strncmp(cmd,"*2\r\n$12\r\nkey:00000123\r\n$9\r\nstr:hello\r\n",len) == 0 && \ len == 4+5+(12+2)+4+(9+2)); \ - free(cmd); \ + hi_free(cmd); \ } while(0) #define FLOAT_WIDTH_TEST(type) do { \ @@ -186,7 +306,7 @@ static void test_format_commands(void) { len = redisFormatCommand(&cmd,"key:%08.3f str:%s", value, "hello"); \ test_cond(strncmp(cmd,"*2\r\n$12\r\nkey:0123.000\r\n$9\r\nstr:hello\r\n",len) == 0 && \ len == 4+5+(12+2)+4+(9+2)); \ - free(cmd); \ + hi_free(cmd); \ } while(0) INTEGER_WIDTH_TEST("d", int); @@ -217,29 +337,29 @@ static void test_format_commands(void) { len = redisFormatCommandArgv(&cmd,argc,argv,NULL); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - free(cmd); + hi_free(cmd); test("Format command by passing argc/argv with lengths: "); len = redisFormatCommandArgv(&cmd,argc,argv,lens); test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(7+2)+4+(3+2)); - free(cmd); + hi_free(cmd); - sds sds_cmd; + hisds sds_cmd; - sds_cmd = sdsempty(); - test("Format command into sds by passing argc/argv without lengths: "); + sds_cmd = NULL; + test("Format command into hisds by passing argc/argv without lengths: "); len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,NULL); test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(3+2)+4+(3+2)); - sdsfree(sds_cmd); + hi_sdsfree(sds_cmd); - sds_cmd = sdsempty(); - test("Format command into sds by passing argc/argv with lengths: "); + sds_cmd = NULL; + test("Format command into hisds by passing argc/argv with lengths: "); len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,lens); test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 && len == 4+4+(3+2)+4+(7+2)+4+(3+2)); - sdsfree(sds_cmd); + hi_sdsfree(sds_cmd); } static void test_append_formatted_commands(struct config config) { @@ -248,7 +368,7 @@ static void test_append_formatted_commands(struct config config) { char *cmd; int len; - c = connect(config); + c = do_connect(config); test("Append format command: "); @@ -258,7 +378,7 @@ static void test_append_formatted_commands(struct config config) { assert(redisGetReply(c, (void*)&reply) == REDIS_OK); - free(cmd); + hi_free(cmd); freeReplyObject(reply); disconnect(c, 0); @@ -266,7 +386,7 @@ static void test_append_formatted_commands(struct config config) { static void test_reply_reader(void) { redisReader *reader; - void *reply; + void *reply, *root; int ret; int i; @@ -290,18 +410,128 @@ static void test_reply_reader(void) { strcasecmp(reader->errstr,"Protocol error, got \"@\" as reply type byte") == 0); redisReaderFree(reader); - test("Set error on nested multi bulks with depth > 7: "); reader = redisReaderCreate(); + test("Can handle arbitrarily nested multi-bulks: "); + for (i = 0; i < 128; i++) { + redisReaderFeed(reader,(char*)"*1\r\n", 4); + } + redisReaderFeed(reader,(char*)"$6\r\nLOLWUT\r\n",12); + ret = redisReaderGetReply(reader,&reply); + root = reply; /* Keep track of the root reply */ + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_ARRAY && + ((redisReply*)reply)->elements == 1); - for (i = 0; i < 9; i++) { - redisReaderFeed(reader,(char*)"*1\r\n",4); + test("Can parse arbitrarily nested multi-bulks correctly: "); + while(i--) { + assert(reply != NULL && ((redisReply*)reply)->type == REDIS_REPLY_ARRAY); + reply = ((redisReply*)reply)->element[0]; } + test_cond(((redisReply*)reply)->type == REDIS_REPLY_STRING && + !memcmp(((redisReply*)reply)->str, "LOLWUT", 6)); + freeReplyObject(root); + redisReaderFree(reader); - ret = redisReaderGetReply(reader,NULL); + test("Correctly parses LLONG_MAX: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, ":9223372036854775807\r\n",22); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_INTEGER && + ((redisReply*)reply)->integer == LLONG_MAX); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Set error when > LLONG_MAX: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, ":9223372036854775808\r\n",22); + ret = redisReaderGetReply(reader,&reply); test_cond(ret == REDIS_ERR && - strncasecmp(reader->errstr,"No support for",14) == 0); + strcasecmp(reader->errstr,"Bad integer value") == 0); + freeReplyObject(reply); redisReaderFree(reader); + test("Correctly parses LLONG_MIN: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, ":-9223372036854775808\r\n",23); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_INTEGER && + ((redisReply*)reply)->integer == LLONG_MIN); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Set error when < LLONG_MIN: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, ":-9223372036854775809\r\n",23); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Bad integer value") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Set error when array < -1: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, "*-2\r\n+asdf\r\n",12); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Multi-bulk length out of range") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Set error when bulk < -1: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, "$-2\r\nasdf\r\n",11); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Bulk string length out of range") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Can configure maximum multi-bulk elements: "); + reader = redisReaderCreate(); + reader->maxelements = 1024; + redisReaderFeed(reader, "*1025\r\n", 7); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr, "Multi-bulk length out of range") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Multi-bulk never overflows regardless of maxelements: "); + size_t bad_mbulk_len = (SIZE_MAX / sizeof(void *)) + 3; + char bad_mbulk_reply[100]; + snprintf(bad_mbulk_reply, sizeof(bad_mbulk_reply), "*%llu\r\n+asdf\r\n", + (unsigned long long) bad_mbulk_len); + + reader = redisReaderCreate(); + reader->maxelements = 0; /* Don't rely on default limit */ + redisReaderFeed(reader, bad_mbulk_reply, strlen(bad_mbulk_reply)); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && strcasecmp(reader->errstr, "Out of memory") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + +#if LLONG_MAX > SIZE_MAX + test("Set error when array > SIZE_MAX: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, "*9223372036854775807\r\n+asdf\r\n",29); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Multi-bulk length out of range") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + + test("Set error when bulk > SIZE_MAX: "); + reader = redisReaderCreate(); + redisReaderFeed(reader, "$9223372036854775807\r\nasdf\r\n",28); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && + strcasecmp(reader->errstr,"Bulk string length out of range") == 0); + freeReplyObject(reply); + redisReaderFree(reader); +#endif + test("Works with NULL functions for reply: "); reader = redisReaderCreate(); reader->fn = NULL; @@ -341,6 +571,32 @@ static void test_reply_reader(void) { ((redisReply*)reply)->elements == 0); freeReplyObject(reply); redisReaderFree(reader); + + /* RESP3 verbatim strings (GitHub issue #802) */ + test("Can parse RESP3 verbatim strings: "); + reader = redisReaderCreate(); + redisReaderFeed(reader,(char*)"=10\r\ntxt:LOLWUT\r\n",17); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_VERB && + !memcmp(((redisReply*)reply)->str,"LOLWUT", 6)); + freeReplyObject(reply); + redisReaderFree(reader); + + /* RESP3 push messages (Github issue #815) */ + test("Can parse RESP3 push messages: "); + reader = redisReaderCreate(); + redisReaderFeed(reader,(char*)">2\r\n$6\r\nLOLWUT\r\n:42\r\n",21); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_OK && + ((redisReply*)reply)->type == REDIS_REPLY_PUSH && + ((redisReply*)reply)->elements == 2 && + ((redisReply*)reply)->element[0]->type == REDIS_REPLY_STRING && + !memcmp(((redisReply*)reply)->element[0]->str,"LOLWUT",6) && + ((redisReply*)reply)->element[1]->type == REDIS_REPLY_INTEGER && + ((redisReply*)reply)->element[1]->integer == 42); + freeReplyObject(reply); + redisReaderFree(reader); } static void test_free_null(void) { @@ -356,21 +612,77 @@ static void test_free_null(void) { test_cond(reply == NULL); } +static void *hi_malloc_fail(size_t size) { + (void)size; + return NULL; +} + +static void *hi_calloc_fail(size_t nmemb, size_t size) { + (void)nmemb; + (void)size; + return NULL; +} + +static void *hi_realloc_fail(void *ptr, size_t size) { + (void)ptr; + (void)size; + return NULL; +} + +static void test_allocator_injection(void) { + hiredisAllocFuncs ha = { + .mallocFn = hi_malloc_fail, + .callocFn = hi_calloc_fail, + .reallocFn = hi_realloc_fail, + .strdupFn = strdup, + .freeFn = free, + }; + + // Override hiredis allocators + hiredisSetAllocators(&ha); + + test("redisContext uses injected allocators: "); + redisContext *c = redisConnect("localhost", 6379); + test_cond(c == NULL); + + test("redisReader uses injected allocators: "); + redisReader *reader = redisReaderCreate(); + test_cond(reader == NULL); + + // Return allocators to default + hiredisResetAllocators(); +} + +#define HIREDIS_BAD_DOMAIN "idontexist-noreally.com" static void test_blocking_connection_errors(void) { redisContext *c; + struct addrinfo hints = {.ai_family = AF_INET}; + struct addrinfo *ai_tmp = NULL; + + int rv = getaddrinfo(HIREDIS_BAD_DOMAIN, "6379", &hints, &ai_tmp); + if (rv != 0) { + // Address does *not* exist + test("Returns error when host cannot be resolved: "); + // First see if this domain name *actually* resolves to NXDOMAIN + c = redisConnect(HIREDIS_BAD_DOMAIN, 6379); + test_cond( + c->err == REDIS_ERR_OTHER && + (strcmp(c->errstr, "Name or service not known") == 0 || + strcmp(c->errstr, "Can't resolve: " HIREDIS_BAD_DOMAIN) == 0 || + strcmp(c->errstr, "Name does not resolve") == 0 || + strcmp(c->errstr, "nodename nor servname provided, or not known") == 0 || + strcmp(c->errstr, "No address associated with hostname") == 0 || + strcmp(c->errstr, "Temporary failure in name resolution") == 0 || + strcmp(c->errstr, "hostname nor servname provided, or not known") == 0 || + strcmp(c->errstr, "no address associated with name") == 0 || + strcmp(c->errstr, "No such host is known. ") == 0)); + redisFree(c); + } else { + printf("Skipping NXDOMAIN test. Found evil ISP!\n"); + freeaddrinfo(ai_tmp); + } - test("Returns error when host cannot be resolved: "); - c = redisConnect((char*)"idontexist.test", 6379); - test_cond(c->err == REDIS_ERR_OTHER && - (strcmp(c->errstr,"Name or service not known") == 0 || - strcmp(c->errstr,"Can't resolve: idontexist.test") == 0 || - strcmp(c->errstr,"nodename nor servname provided, or not known") == 0 || - strcmp(c->errstr,"No address associated with hostname") == 0 || - strcmp(c->errstr,"Temporary failure in name resolution") == 0 || - strcmp(c->errstr,"hostname nor servname provided, or not known") == 0 || - strcmp(c->errstr,"no address associated with name") == 0)); - redisFree(c); - +#ifndef _WIN32 test("Returns error when the port is not open: "); c = redisConnect((char*)"localhost", 1); test_cond(c->err == REDIS_ERR_IO && @@ -381,13 +693,168 @@ static void test_blocking_connection_errors(void) { c = redisConnectUnix((char*)"/tmp/idontexist.sock"); test_cond(c->err == REDIS_ERR_IO); /* Don't care about the message... */ redisFree(c); +#endif +} + +/* Test push handler */ +void push_handler(void *privdata, void *r) { + struct pushCounters *pcounts = privdata; + redisReply *reply = r, *payload; + + assert(reply && reply->type == REDIS_REPLY_PUSH && reply->elements == 2); + + payload = reply->element[1]; + if (payload->type == REDIS_REPLY_ARRAY) { + payload = payload->element[0]; + } + + if (payload->type == REDIS_REPLY_STRING) { + pcounts->str++; + } else if (payload->type == REDIS_REPLY_NIL) { + pcounts->nil++; + } + + freeReplyObject(reply); +} + +/* Dummy function just to test setting a callback with redisOptions */ +void push_handler_async(redisAsyncContext *ac, void *reply) { + (void)ac; + (void)reply; +} + +static void test_resp3_push_handler(redisContext *c) { + struct pushCounters pc = {0}; + redisPushFn *old = NULL; + redisReply *reply; + void *privdata; + + /* Switch to RESP3 and turn on client tracking */ + send_hello(c, 3); + send_client_tracking(c, "ON"); + privdata = c->privdata; + c->privdata = &pc; + + reply = redisCommand(c, "GET key:0"); + assert(reply != NULL); + freeReplyObject(reply); + + test("RESP3 PUSH messages are handled out of band by default: "); + reply = redisCommand(c, "SET key:0 val:0"); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS); + freeReplyObject(reply); + + assert((reply = redisCommand(c, "GET key:0")) != NULL); + freeReplyObject(reply); + + old = redisSetPushCallback(c, push_handler); + test("We can set a custom RESP3 PUSH handler: "); + reply = redisCommand(c, "SET key:0 val:0"); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && pc.str == 1); + freeReplyObject(reply); + + test("We properly handle a NIL invalidation payload: "); + reply = redisCommand(c, "FLUSHDB"); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && pc.nil == 1); + freeReplyObject(reply); + + /* Unset the push callback and generate an invalidate message making + * sure it is not handled out of band. */ + test("With no handler, PUSH replies come in-band: "); + redisSetPushCallback(c, NULL); + assert((reply = redisCommand(c, "GET key:0")) != NULL); + freeReplyObject(reply); + assert((reply = redisCommand(c, "SET key:0 invalid")) != NULL); + test_cond(reply->type == REDIS_REPLY_PUSH); + freeReplyObject(reply); + + test("With no PUSH handler, no replies are lost: "); + assert(redisGetReply(c, (void**)&reply) == REDIS_OK); + test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS); + freeReplyObject(reply); + + /* Return to the originally set PUSH handler */ + assert(old != NULL); + redisSetPushCallback(c, old); + + /* Switch back to RESP2 and disable tracking */ + c->privdata = privdata; + send_client_tracking(c, "OFF"); + send_hello(c, 2); +} + +redisOptions get_redis_tcp_options(struct config config) { + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, config.tcp.host, config.tcp.port); + return options; +} + +static void test_resp3_push_options(struct config config) { + redisAsyncContext *ac; + redisContext *c; + redisOptions options; + + test("We set a default RESP3 handler for redisContext: "); + options = get_redis_tcp_options(config); + assert((c = redisConnectWithOptions(&options)) != NULL); + test_cond(c->push_cb != NULL); + redisFree(c); + + test("We don't set a default RESP3 push handler for redisAsyncContext: "); + options = get_redis_tcp_options(config); + assert((ac = redisAsyncConnectWithOptions(&options)) != NULL); + test_cond(ac->c.push_cb == NULL); + redisAsyncFree(ac); + + test("Our REDIS_OPT_NO_PUSH_AUTOFREE flag works: "); + options = get_redis_tcp_options(config); + options.options |= REDIS_OPT_NO_PUSH_AUTOFREE; + assert((c = redisConnectWithOptions(&options)) != NULL); + test_cond(c->push_cb == NULL); + redisFree(c); + + test("We can use redisOptions to set a custom PUSH handler for redisContext: "); + options = get_redis_tcp_options(config); + options.push_cb = push_handler; + assert((c = redisConnectWithOptions(&options)) != NULL); + test_cond(c->push_cb == push_handler); + redisFree(c); + + test("We can use redisOptions to set a custom PUSH handler for redisAsyncContext: "); + options = get_redis_tcp_options(config); + options.async_push_cb = push_handler_async; + assert((ac = redisAsyncConnectWithOptions(&options)) != NULL); + test_cond(ac->push_cb == push_handler_async); + redisAsyncFree(ac); +} + +void free_privdata(void *privdata) { + struct privdata *data = privdata; + data->dtor_counter++; +} + +static void test_privdata_hooks(struct config config) { + struct privdata data = {0}; + redisOptions options; + redisContext *c; + + test("We can use redisOptions to set privdata: "); + options = get_redis_tcp_options(config); + REDIS_OPTIONS_SET_PRIVDATA(&options, &data, free_privdata); + assert((c = redisConnectWithOptions(&options)) != NULL); + test_cond(c->privdata == &data); + + test("Our privdata destructor fires when we free the context: "); + redisFree(c); + test_cond(data.dtor_counter == 1); } static void test_blocking_connection(struct config config) { redisContext *c; redisReply *reply; + int major; - c = connect(config); + c = do_connect(config); test("Is able to deliver commands: "); reply = redisCommand(c,"PING"); @@ -458,17 +925,45 @@ static void test_blocking_connection(struct config config) { strcasecmp(reply->element[1]->str,"pong") == 0); freeReplyObject(reply); + /* Make sure passing NULL to redisGetReply is safe */ + test("Can pass NULL to redisGetReply: "); + assert(redisAppendCommand(c, "PING") == REDIS_OK); + test_cond(redisGetReply(c, NULL) == REDIS_OK); + + get_redis_version(c, &major, NULL); + if (major >= 6) test_resp3_push_handler(c); + test_resp3_push_options(config); + + test_privdata_hooks(config); + disconnect(c, 0); } +/* Send DEBUG SLEEP 0 to detect if we have this command */ +static int detect_debug_sleep(redisContext *c) { + int detected; + redisReply *reply = redisCommand(c, "DEBUG SLEEP 0\r\n"); + + if (reply == NULL || c->err) { + const char *cause = c->err ? c->errstr : "(none)"; + fprintf(stderr, "Error testing for DEBUG SLEEP (Redis error: %s), exiting\n", cause); + exit(-1); + } + + detected = reply->type == REDIS_REPLY_STATUS; + freeReplyObject(reply); + + return detected; +} + static void test_blocking_connection_timeouts(struct config config) { redisContext *c; redisReply *reply; ssize_t s; - const char *cmd = "DEBUG SLEEP 3\r\n"; + const char *sleep_cmd = "DEBUG SLEEP 3\r\n"; struct timeval tv; - c = connect(config); + c = do_connect(config); test("Successfully completes a command when the timeout is not exceeded: "); reply = redisCommand(c,"SET foo fast"); freeReplyObject(reply); @@ -480,18 +975,29 @@ static void test_blocking_connection_timeouts(struct config config) { freeReplyObject(reply); disconnect(c, 0); - c = connect(config); + c = do_connect(config); test("Does not return a reply when the command times out: "); - s = write(c->fd, cmd, strlen(cmd)); - tv.tv_sec = 0; - tv.tv_usec = 10000; - redisSetTimeout(c, tv); - reply = redisCommand(c, "GET foo"); - test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_IO && strcmp(c->errstr, "Resource temporarily unavailable") == 0); - freeReplyObject(reply); + if (detect_debug_sleep(c)) { + redisAppendFormattedCommand(c, sleep_cmd, strlen(sleep_cmd)); + s = c->funcs->write(c); + tv.tv_sec = 0; + tv.tv_usec = 10000; + redisSetTimeout(c, tv); + reply = redisCommand(c, "GET foo"); +#ifndef _WIN32 + test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_IO && + strcmp(c->errstr, "Resource temporarily unavailable") == 0); +#else + test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_TIMEOUT && + strcmp(c->errstr, "recv timeout") == 0); +#endif + freeReplyObject(reply); + } else { + test_skipped(); + } test("Reconnect properly reconnects after a timeout: "); - redisReconnect(c); + do_reconnect(c, config); reply = redisCommand(c, "PING"); test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); freeReplyObject(reply); @@ -499,7 +1005,7 @@ static void test_blocking_connection_timeouts(struct config config) { test("Reconnect properly uses owned parameters: "); config.tcp.host = "foo"; config.unix_sock.path = "foo"; - redisReconnect(c); + do_reconnect(c, config); reply = redisCommand(c, "PING"); test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); freeReplyObject(reply); @@ -514,19 +1020,8 @@ static void test_blocking_io_errors(struct config config) { int major, minor; /* Connect to target given by config. */ - c = connect(config); - { - /* Find out Redis version to determine the path for the next test */ - const char *field = "redis_version:"; - char *p, *eptr; - - reply = redisCommand(c,"INFO"); - p = strstr(reply->str,field); - major = strtol(p+strlen(field),&eptr,10); - p = eptr+1; /* char next to the first "." */ - minor = strtol(p,&eptr,10); - freeReplyObject(reply); - } + c = do_connect(config); + get_redis_version(c, &major, &minor); test("Returns I/O error when the connection is lost: "); reply = redisCommand(c,"QUIT"); @@ -540,6 +1035,7 @@ static void test_blocking_io_errors(struct config config) { test_cond(reply == NULL); } +#ifndef _WIN32 /* On 2.0, QUIT will cause the connection to be closed immediately and * the read(2) for the reply on QUIT will set the error to EOF. * On >2.0, QUIT will return with OK and another read(2) needed to be @@ -547,14 +1043,19 @@ static void test_blocking_io_errors(struct config config) { * conditions, the error will be set to EOF. */ assert(c->err == REDIS_ERR_EOF && strcmp(c->errstr,"Server closed the connection") == 0); +#endif redisFree(c); - c = connect(config); + c = do_connect(config); test("Returns I/O error on socket timeout: "); struct timeval tv = { 0, 1000 }; assert(redisSetTimeout(c,tv) == REDIS_OK); - test_cond(redisGetReply(c,&_reply) == REDIS_ERR && - c->err == REDIS_ERR_IO && errno == EAGAIN); + int respcode = redisGetReply(c,&_reply); +#ifndef _WIN32 + test_cond(respcode == REDIS_ERR && c->err == REDIS_ERR_IO && errno == EAGAIN); +#else + test_cond(respcode == REDIS_ERR && c->err == REDIS_ERR_TIMEOUT); +#endif redisFree(c); } @@ -582,8 +1083,20 @@ static void test_invalid_timeout_errors(struct config config) { redisFree(c); } +/* Wrap malloc to abort on failure so OOM checks don't make the test logic + * harder to follow. */ +void *hi_malloc_safe(size_t size) { + void *ptr = hi_malloc(size); + if (ptr == NULL) { + fprintf(stderr, "Error: Out of memory\n"); + exit(-1); + } + + return ptr; +} + static void test_throughput(struct config config) { - redisContext *c = connect(config); + redisContext *c = do_connect(config); redisReply **replies; int i, num; long long t1, t2; @@ -593,7 +1106,7 @@ static void test_throughput(struct config config) { freeReplyObject(redisCommand(c,"LPUSH mylist foo")); num = 1000; - replies = malloc(sizeof(redisReply*)*num); + replies = hi_malloc_safe(sizeof(redisReply*)*num); t1 = usec(); for (i = 0; i < num; i++) { replies[i] = redisCommand(c,"PING"); @@ -601,10 +1114,10 @@ static void test_throughput(struct config config) { } t2 = usec(); for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); + hi_free(replies); printf("\t(%dx PING: %.3fs)\n", num, (t2-t1)/1000000.0); - replies = malloc(sizeof(redisReply*)*num); + replies = hi_malloc_safe(sizeof(redisReply*)*num); t1 = usec(); for (i = 0; i < num; i++) { replies[i] = redisCommand(c,"LRANGE mylist 0 499"); @@ -613,11 +1126,22 @@ static void test_throughput(struct config config) { } t2 = usec(); for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); + hi_free(replies); printf("\t(%dx LRANGE with 500 elements: %.3fs)\n", num, (t2-t1)/1000000.0); + replies = hi_malloc_safe(sizeof(redisReply*)*num); + t1 = usec(); + for (i = 0; i < num; i++) { + replies[i] = redisCommand(c, "INCRBY incrkey %d", 1000000); + assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_INTEGER); + } + t2 = usec(); + for (i = 0; i < num; i++) freeReplyObject(replies[i]); + hi_free(replies); + printf("\t(%dx INCRBY: %.3fs)\n", num, (t2-t1)/1000000.0); + num = 10000; - replies = malloc(sizeof(redisReply*)*num); + replies = hi_malloc_safe(sizeof(redisReply*)*num); for (i = 0; i < num; i++) redisAppendCommand(c,"PING"); t1 = usec(); @@ -627,10 +1151,10 @@ static void test_throughput(struct config config) { } t2 = usec(); for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); + hi_free(replies); printf("\t(%dx PING (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); - replies = malloc(sizeof(redisReply*)*num); + replies = hi_malloc_safe(sizeof(redisReply*)*num); for (i = 0; i < num; i++) redisAppendCommand(c,"LRANGE mylist 0 499"); t1 = usec(); @@ -641,9 +1165,22 @@ static void test_throughput(struct config config) { } t2 = usec(); for (i = 0; i < num; i++) freeReplyObject(replies[i]); - free(replies); + hi_free(replies); printf("\t(%dx LRANGE with 500 elements (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); + replies = hi_malloc_safe(sizeof(redisReply*)*num); + for (i = 0; i < num; i++) + redisAppendCommand(c,"INCRBY incrkey %d", 1000000); + t1 = usec(); + for (i = 0; i < num; i++) { + assert(redisGetReply(c, (void*)&replies[i]) == REDIS_OK); + assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_INTEGER); + } + t2 = usec(); + for (i = 0; i < num; i++) freeReplyObject(replies[i]); + hi_free(replies); + printf("\t(%dx INCRBY (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); + disconnect(c, 0); } @@ -758,9 +1295,8 @@ int main(int argc, char **argv) { }; int throughput = 1; int test_inherit_fd = 1; - - /* Ignore broken pipe signal (for I/O error tests). */ - signal(SIGPIPE, SIG_IGN); + int skips_as_fails = 0; + int test_unix_socket; /* Parse command line options. */ argv++; argc--; @@ -778,6 +1314,25 @@ int main(int argc, char **argv) { throughput = 0; } else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) { test_inherit_fd = 0; + } else if (argc >= 1 && !strcmp(argv[0],"--skips-as-fails")) { + skips_as_fails = 1; +#ifdef HIREDIS_TEST_SSL + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-port")) { + argv++; argc--; + cfg.ssl.port = atoi(argv[0]); + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-host")) { + argv++; argc--; + cfg.ssl.host = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-ca-cert")) { + argv++; argc--; + cfg.ssl.ca_cert = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-cert")) { + argv++; argc--; + cfg.ssl.cert = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-key")) { + argv++; argc--; + cfg.ssl.key = argv[0]; +#endif } else { fprintf(stderr, "Invalid argument: %s\n", argv[0]); exit(1); @@ -785,6 +1340,19 @@ int main(int argc, char **argv) { argv++; argc--; } +#ifndef _WIN32 + /* Ignore broken pipe signal (for I/O error tests). */ + signal(SIGPIPE, SIG_IGN); + + test_unix_socket = access(cfg.unix_sock.path, F_OK) == 0; + +#else + /* Unix sockets don't exist in Windows */ + test_unix_socket = 0; +#endif + + test_allocator_injection(); + test_format_commands(); test_reply_reader(); test_blocking_connection_errors(); @@ -799,25 +1367,59 @@ int main(int argc, char **argv) { test_append_formatted_commands(cfg); if (throughput) test_throughput(cfg); - printf("\nTesting against Unix socket connection (%s):\n", cfg.unix_sock.path); - cfg.type = CONN_UNIX; - test_blocking_connection(cfg); - test_blocking_connection_timeouts(cfg); - test_blocking_io_errors(cfg); - if (throughput) test_throughput(cfg); + printf("\nTesting against Unix socket connection (%s): ", cfg.unix_sock.path); + if (test_unix_socket) { + printf("\n"); + cfg.type = CONN_UNIX; + test_blocking_connection(cfg); + test_blocking_connection_timeouts(cfg); + test_blocking_io_errors(cfg); + if (throughput) test_throughput(cfg); + } else { + test_skipped(); + } + +#ifdef HIREDIS_TEST_SSL + if (cfg.ssl.port && cfg.ssl.host) { + + redisInitOpenSSL(); + _ssl_ctx = redisCreateSSLContext(cfg.ssl.ca_cert, NULL, cfg.ssl.cert, cfg.ssl.key, NULL, NULL); + assert(_ssl_ctx != NULL); + + printf("\nTesting against SSL connection (%s:%d):\n", cfg.ssl.host, cfg.ssl.port); + cfg.type = CONN_SSL; - if (test_inherit_fd) { - printf("\nTesting against inherited fd (%s):\n", cfg.unix_sock.path); - cfg.type = CONN_FD; test_blocking_connection(cfg); + test_blocking_connection_timeouts(cfg); + test_blocking_io_errors(cfg); + test_invalid_timeout_errors(cfg); + test_append_formatted_commands(cfg); + if (throughput) test_throughput(cfg); + + redisFreeSSLContext(_ssl_ctx); + _ssl_ctx = NULL; } +#endif + if (test_inherit_fd) { + printf("\nTesting against inherited fd (%s): ", cfg.unix_sock.path); + if (test_unix_socket) { + printf("\n"); + cfg.type = CONN_FD; + test_blocking_connection(cfg); + } else { + test_skipped(); + } + } - if (fails) { + if (fails || (skips_as_fails && skips)) { printf("*** %d TESTS FAILED ***\n", fails); + if (skips) { + printf("*** %d TESTS SKIPPED ***\n", skips); + } return 1; } - printf("ALL TESTS PASSED\n"); + printf("ALL TESTS PASSED (%d skipped)\n", skips); return 0; } diff --git a/redis.submodule/deps/hiredis/test.sh b/redis.submodule/deps/hiredis/test.sh new file mode 100755 index 0000000..c72bcb0 --- /dev/null +++ b/redis.submodule/deps/hiredis/test.sh @@ -0,0 +1,78 @@ +#!/bin/sh -ue + +REDIS_SERVER=${REDIS_SERVER:-redis-server} +REDIS_PORT=${REDIS_PORT:-56379} +REDIS_SSL_PORT=${REDIS_SSL_PORT:-56443} +TEST_SSL=${TEST_SSL:-0} +SKIPS_AS_FAILS=${SKIPS_AS_FAILS-:0} +SSL_TEST_ARGS= +SKIPS_ARG= + +tmpdir=$(mktemp -d) +PID_FILE=${tmpdir}/hiredis-test-redis.pid +SOCK_FILE=${tmpdir}/hiredis-test-redis.sock + +if [ "$TEST_SSL" = "1" ]; then + SSL_CA_CERT=${tmpdir}/ca.crt + SSL_CA_KEY=${tmpdir}/ca.key + SSL_CERT=${tmpdir}/redis.crt + SSL_KEY=${tmpdir}/redis.key + + openssl genrsa -out ${tmpdir}/ca.key 4096 + openssl req \ + -x509 -new -nodes -sha256 \ + -key ${SSL_CA_KEY} \ + -days 3650 \ + -subj '/CN=Hiredis Test CA' \ + -out ${SSL_CA_CERT} + openssl genrsa -out ${SSL_KEY} 2048 + openssl req \ + -new -sha256 \ + -key ${SSL_KEY} \ + -subj '/CN=Hiredis Test Cert' | \ + openssl x509 \ + -req -sha256 \ + -CA ${SSL_CA_CERT} \ + -CAkey ${SSL_CA_KEY} \ + -CAserial ${tmpdir}/ca.txt \ + -CAcreateserial \ + -days 365 \ + -out ${SSL_CERT} + + SSL_TEST_ARGS="--ssl-host 127.0.0.1 --ssl-port ${REDIS_SSL_PORT} --ssl-ca-cert ${SSL_CA_CERT} --ssl-cert ${SSL_CERT} --ssl-key ${SSL_KEY}" +fi + +cleanup() { + set +e + kill $(cat ${PID_FILE}) + rm -rf ${tmpdir} +} +trap cleanup INT TERM EXIT + +cat > ${tmpdir}/redis.conf <> ${tmpdir}/redis.conf < /* for struct timeval */ + #ifndef inline #define inline __inline #endif +#ifndef strcasecmp +#define strcasecmp stricmp +#endif + +#ifndef strncasecmp +#define strncasecmp strnicmp +#endif + #ifndef va_copy #define va_copy(d,s) ((d) = (s)) #endif @@ -37,6 +47,10 @@ __inline int c99_snprintf(char* str, size_t size, const char* format, ...) return count; } #endif +#endif /* _MSC_VER */ -#endif -#endif \ No newline at end of file +#ifdef _WIN32 +#define strerror_r(errno,buf,len) strerror_s(buf,len,errno) +#endif /* _WIN32 */ + +#endif /* _WIN32_HELPER_INCLUDE */ diff --git a/redis.submodule/deps/jemalloc/configure.ac b/redis.submodule/deps/jemalloc/configure.ac index a6a08db..afa5df8 100644 --- a/redis.submodule/deps/jemalloc/configure.ac +++ b/redis.submodule/deps/jemalloc/configure.ac @@ -512,7 +512,7 @@ CTARGET='-o $@' LDTARGET='-o $@' TEST_LD_MODE= EXTRA_LDFLAGS= -ARFLAGS='crus' +ARFLAGS='crs' AROUT=' $@' CC_MM=1 diff --git a/redis.submodule/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h b/redis.submodule/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h index 290e5cf..b19a942 100644 --- a/redis.submodule/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h +++ b/redis.submodule/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h @@ -216,7 +216,7 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, } JEMALLOC_ALWAYS_INLINE int -iget_defrag_hint(tsdn_t *tsdn, void* ptr, int *bin_util, int *run_util) { +iget_defrag_hint(tsdn_t *tsdn, void* ptr) { int defrag = 0; rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); @@ -232,11 +232,22 @@ iget_defrag_hint(tsdn_t *tsdn, void* ptr, int *bin_util, int *run_util) { malloc_mutex_lock(tsdn, &bin->lock); /* don't bother moving allocations from the slab currently used for new allocations */ if (slab != bin->slabcur) { - const bin_info_t *bin_info = &bin_infos[binind]; - size_t availregs = bin_info->nregs * bin->stats.curslabs; - *bin_util = ((long long)bin->stats.curregs<<16) / availregs; - *run_util = ((long long)(bin_info->nregs - extent_nfree_get(slab))<<16) / bin_info->nregs; - defrag = 1; + int free_in_slab = extent_nfree_get(slab); + if (free_in_slab) { + const bin_info_t *bin_info = &bin_infos[binind]; + unsigned long curslabs = bin->stats.curslabs; + size_t curregs = bin->stats.curregs; + if (bin->slabcur) { + /* remove slabcur from the overall utilization */ + curregs -= bin_info->nregs - extent_nfree_get(bin->slabcur); + curslabs -= 1; + } + /* Compare the utilization ratio of the slab in question to the total average, + * to avoid precision lost and division, we do that by extrapolating the usage + * of the slab as if all slabs have the same usage. If this slab is less used + * than the average, we'll prefer to evict the data to hopefully more used ones */ + defrag = (bin_info->nregs - free_in_slab) * curslabs <= curregs; + } } malloc_mutex_unlock(tsdn, &bin->lock); } diff --git a/redis.submodule/deps/jemalloc/src/background_thread.c b/redis.submodule/deps/jemalloc/src/background_thread.c index 3517a3b..457669c 100644 --- a/redis.submodule/deps/jemalloc/src/background_thread.c +++ b/redis.submodule/deps/jemalloc/src/background_thread.c @@ -787,7 +787,13 @@ background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) { nstime_init(&stats->run_interval, 0); for (unsigned i = 0; i < max_background_threads; i++) { background_thread_info_t *info = &background_thread_info[i]; - malloc_mutex_lock(tsdn, &info->mtx); + if (malloc_mutex_trylock(tsdn, &info->mtx)) { + /* + * Each background thread run may take a long time; + * avoid waiting on the stats if the thread is active. + */ + continue; + } if (info->state != background_thread_stopped) { num_runs += info->tot_n_runs; nstime_add(&stats->run_interval, &info->tot_sleep_time); diff --git a/redis.submodule/deps/jemalloc/src/jemalloc.c b/redis.submodule/deps/jemalloc/src/jemalloc.c index 5b936cb..585645a 100644 --- a/redis.submodule/deps/jemalloc/src/jemalloc.c +++ b/redis.submodule/deps/jemalloc/src/jemalloc.c @@ -3326,12 +3326,10 @@ jemalloc_postfork_child(void) { /******************************************************************************/ /* Helps the application decide if a pointer is worth re-allocating in order to reduce fragmentation. - * returns 0 if the allocation is in the currently active run, - * or when it is not causing any frag issue (large or huge bin) - * returns the bin utilization and run utilization both in fixed point 16:16. + * returns 1 if the allocation should be moved, and 0 if the allocation be kept. * If the application decides to re-allocate it should use MALLOCX_TCACHE_NONE when doing so. */ JEMALLOC_EXPORT int JEMALLOC_NOTHROW -get_defrag_hint(void* ptr, int *bin_util, int *run_util) { +get_defrag_hint(void* ptr) { assert(ptr != NULL); - return iget_defrag_hint(TSDN_NULL, ptr, bin_util, run_util); + return iget_defrag_hint(TSDN_NULL, ptr); } diff --git a/redis.submodule/deps/jemalloc/src/malloc_io.c b/redis.submodule/deps/jemalloc/src/malloc_io.c index 7bdc13f..dd88265 100644 --- a/redis.submodule/deps/jemalloc/src/malloc_io.c +++ b/redis.submodule/deps/jemalloc/src/malloc_io.c @@ -362,7 +362,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { } \ } while (0) #define GET_ARG_NUMERIC(val, len) do { \ - switch (len) { \ + switch ((unsigned char)len) { \ case '?': \ val = va_arg(ap, int); \ break; \ diff --git a/redis.submodule/deps/linenoise/README.markdown b/redis.submodule/deps/linenoise/README.markdown index e01642c..1afea2a 100644 --- a/redis.submodule/deps/linenoise/README.markdown +++ b/redis.submodule/deps/linenoise/README.markdown @@ -21,7 +21,7 @@ So what usually happens is either: The result is a pollution of binaries without line editing support. -So I spent more or less two hours doing a reality check resulting in this little library: is it *really* needed for a line editing library to be 20k lines of code? Apparently not, it is possibe to get a very small, zero configuration, trivial to embed library, that solves the problem. Smaller programs will just include this, supporing line editing out of the box. Larger programs may use this little library or just checking with configure if readline/libedit is available and resorting to Linenoise if not. +So I spent more or less two hours doing a reality check resulting in this little library: is it *really* needed for a line editing library to be 20k lines of code? Apparently not, it is possibe to get a very small, zero configuration, trivial to embed library, that solves the problem. Smaller programs will just include this, supporting line editing out of the box. Larger programs may use this little library or just checking with configure if readline/libedit is available and resorting to Linenoise if not. ## Terminals, in 2010. @@ -126,6 +126,24 @@ Linenoise has direct support for persisting the history into an history file. The functions `linenoiseHistorySave` and `linenoiseHistoryLoad` do just that. Both functions return -1 on error and 0 on success. +## Mask mode + +Sometimes it is useful to allow the user to type passwords or other +secrets that should not be displayed. For such situations linenoise supports +a "mask mode" that will just replace the characters the user is typing +with `*` characters, like in the following example: + + $ ./linenoise_example + hello> get mykey + echo: 'get mykey' + hello> /mask + hello> ********* + +You can enable and disable mask mode using the following two functions: + + void linenoiseMaskModeEnable(void); + void linenoiseMaskModeDisable(void); + ## Completion Linenoise supports completion, which is the ability to complete the user @@ -222,3 +240,8 @@ Sometimes you may want to clear the screen as a result of something the user typed. You can do this by calling the following function: void linenoiseClearScreen(void); + +## Related projects + +* [Linenoise NG](https://github.com/arangodb/linenoise-ng) is a fork of Linenoise that aims to add more advanced features like UTF-8 support, Windows support and other features. Uses C++ instead of C as development language. +* [Linenoise-swift](https://github.com/andybest/linenoise-swift) is a reimplementation of Linenoise written in Swift. diff --git a/redis.submodule/deps/linenoise/example.c b/redis.submodule/deps/linenoise/example.c index 3a544d3..74358c3 100644 --- a/redis.submodule/deps/linenoise/example.c +++ b/redis.submodule/deps/linenoise/example.c @@ -55,6 +55,7 @@ int main(int argc, char **argv) { * * The typed string is returned as a malloc() allocated string by * linenoise, so the user needs to free() it. */ + while((line = linenoise("hello> ")) != NULL) { /* Do something with the string. */ if (line[0] != '\0' && line[0] != '/') { @@ -65,6 +66,10 @@ int main(int argc, char **argv) { /* The "/historylen" command will change the history len. */ int len = atoi(line+11); linenoiseHistorySetMaxLen(len); + } else if (!strncmp(line, "/mask", 5)) { + linenoiseMaskModeEnable(); + } else if (!strncmp(line, "/unmask", 7)) { + linenoiseMaskModeDisable(); } else if (line[0] == '/') { printf("Unreconized command: %s\n", line); } diff --git a/redis.submodule/deps/linenoise/linenoise.c b/redis.submodule/deps/linenoise/linenoise.c index fce14a7..ccf5c55 100644 --- a/redis.submodule/deps/linenoise/linenoise.c +++ b/redis.submodule/deps/linenoise/linenoise.c @@ -125,6 +125,7 @@ static linenoiseHintsCallback *hintsCallback = NULL; static linenoiseFreeHintsCallback *freeHintsCallback = NULL; static struct termios orig_termios; /* In order to restore at exit.*/ +static int maskmode = 0; /* Show "***" instead of input. For passwords. */ static int rawmode = 0; /* For atexit() function to check if restore is needed*/ static int mlmode = 0; /* Multi line mode. Default is single line. */ static int atexit_registered = 0; /* Register atexit just 1 time. */ @@ -197,6 +198,19 @@ FILE *lndebug_fp = NULL; /* ======================= Low level terminal handling ====================== */ +/* Enable "mask mode". When it is enabled, instead of the input that + * the user is typing, the terminal will just display a corresponding + * number of asterisks, like "****". This is useful for passwords and other + * secrets that should not be displayed. */ +void linenoiseMaskModeEnable(void) { + maskmode = 1; +} + +/* Disable mask mode. */ +void linenoiseMaskModeDisable(void) { + maskmode = 0; +} + /* Set if to use or not the multi line mode. */ void linenoiseSetMultiLine(int ml) { mlmode = ml; @@ -485,6 +499,8 @@ void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) { if (bold == 1 && color == -1) color = 37; if (color != -1 || bold != 0) snprintf(seq,64,"\033[%d;%d;49m",bold,color); + else + seq[0] = '\0'; abAppend(ab,seq,strlen(seq)); abAppend(ab,hint,hintlen); if (color != -1 || bold != 0) @@ -523,7 +539,11 @@ static void refreshSingleLine(struct linenoiseState *l) { abAppend(&ab,seq,strlen(seq)); /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); - abAppend(&ab,buf,len); + if (maskmode == 1) { + while (len--) abAppend(&ab,"*",1); + } else { + abAppend(&ab,buf,len); + } /* Show hits if any. */ refreshShowHints(&ab,l,plen); /* Erase to right */ @@ -577,7 +597,12 @@ static void refreshMultiLine(struct linenoiseState *l) { /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); - abAppend(&ab,l->buf,l->len); + if (maskmode == 1) { + unsigned int i; + for (i = 0; i < l->len; i++) abAppend(&ab,"*",1); + } else { + abAppend(&ab,l->buf,l->len); + } /* Show hits if any. */ refreshShowHints(&ab,l,plen); @@ -600,7 +625,7 @@ static void refreshMultiLine(struct linenoiseState *l) { rpos2 = (plen+l->pos+l->cols)/l->cols; /* current cursor relative row. */ lndebug("rpos2 %d", rpos2); - /* Go up till we reach the expected positon. */ + /* Go up till we reach the expected position. */ if (rows-rpos2 > 0) { lndebug("go-up %d", rows-rpos2); snprintf(seq,64,"\x1b[%dA", rows-rpos2); @@ -645,7 +670,8 @@ int linenoiseEditInsert(struct linenoiseState *l, char c) { if ((!mlmode && l->plen+l->len < l->cols && !hintsCallback)) { /* Avoid a full update of the line in the * trivial case. */ - if (write(l->ofd,&c,1) == -1) return -1; + char d = (maskmode==1) ? '*' : c; + if (write(l->ofd,&d,1) == -1) return -1; } else { refreshLine(l); } @@ -741,7 +767,7 @@ void linenoiseEditBackspace(struct linenoiseState *l) { } } -/* Delete the previosu word, maintaining the cursor at the start of the +/* Delete the previous word, maintaining the cursor at the start of the * current word. */ void linenoiseEditDeletePrevWord(struct linenoiseState *l) { size_t old_pos = l->pos; diff --git a/redis.submodule/deps/linenoise/linenoise.h b/redis.submodule/deps/linenoise/linenoise.h index ed20232..6dfee73 100644 --- a/redis.submodule/deps/linenoise/linenoise.h +++ b/redis.submodule/deps/linenoise/linenoise.h @@ -65,6 +65,8 @@ int linenoiseHistoryLoad(const char *filename); void linenoiseClearScreen(void); void linenoiseSetMultiLine(int ml); void linenoisePrintKeyCodes(void); +void linenoiseMaskModeEnable(void); +void linenoiseMaskModeDisable(void); #ifdef __cplusplus } diff --git a/redis.submodule/deps/lua/src/lauxlib.c b/redis.submodule/deps/lua/src/lauxlib.c index 10f14e2..751f1e8 100644 --- a/redis.submodule/deps/lua/src/lauxlib.c +++ b/redis.submodule/deps/lua/src/lauxlib.c @@ -575,7 +575,7 @@ LUALIB_API int luaL_loadfile (lua_State *L, const char *filename) { if (lf.f == NULL) return errfile(L, "reopen", fnameindex); /* skip eventual `#!...' */ while ((c = getc(lf.f)) != EOF && c != LUA_SIGNATURE[0]) ; - lf.extraline = 0; + lf.extraline = 0; } ungetc(c, lf.f); status = lua_load(L, getF, &lf, lua_tostring(L, -1)); diff --git a/redis.submodule/deps/lua/src/ldo.c b/redis.submodule/deps/lua/src/ldo.c index 514f7a2..e641dfc 100644 --- a/redis.submodule/deps/lua/src/ldo.c +++ b/redis.submodule/deps/lua/src/ldo.c @@ -274,7 +274,7 @@ int luaD_precall (lua_State *L, StkId func, int nresults) { CallInfo *ci; StkId st, base; Proto *p = cl->p; - luaD_checkstack(L, p->maxstacksize); + luaD_checkstack(L, p->maxstacksize + p->numparams); func = restorestack(L, funcr); if (!p->is_vararg) { /* no varargs? */ base = func + 1; @@ -493,7 +493,7 @@ static void f_parser (lua_State *L, void *ud) { Proto *tf; Closure *cl; struct SParser *p = cast(struct SParser *, ud); - int c = luaZ_lookahead(p->z); + luaZ_lookahead(p->z); luaC_checkGC(L); tf = (luaY_parser)(L, p->z, &p->buff, p->name); diff --git a/redis.submodule/deps/lua/src/ltablib.c b/redis.submodule/deps/lua/src/ltablib.c index b6d9cb4..0bdac7f 100644 --- a/redis.submodule/deps/lua/src/ltablib.c +++ b/redis.submodule/deps/lua/src/ltablib.c @@ -137,7 +137,7 @@ static void addfield (lua_State *L, luaL_Buffer *b, int i) { if (!lua_isstring(L, -1)) luaL_error(L, "invalid value (%s) at index %d in table for " LUA_QL("concat"), luaL_typename(L, -1), i); - luaL_addvalue(b); + luaL_addvalue(b); } diff --git a/redis.submodule/deps/lua/src/lua_struct.c b/redis.submodule/deps/lua/src/lua_struct.c index 4d5f027..c58c8e7 100644 --- a/redis.submodule/deps/lua/src/lua_struct.c +++ b/redis.submodule/deps/lua/src/lua_struct.c @@ -89,12 +89,14 @@ typedef struct Header { } Header; -static int getnum (const char **fmt, int df) { +static int getnum (lua_State *L, const char **fmt, int df) { if (!isdigit(**fmt)) /* no number? */ return df; /* return default value */ else { int a = 0; do { + if (a > (INT_MAX / 10) || a * 10 > (INT_MAX - (**fmt - '0'))) + luaL_error(L, "integral size overflow"); a = a*10 + *((*fmt)++) - '0'; } while (isdigit(**fmt)); return a; @@ -115,9 +117,9 @@ static size_t optsize (lua_State *L, char opt, const char **fmt) { case 'f': return sizeof(float); case 'd': return sizeof(double); case 'x': return 1; - case 'c': return getnum(fmt, 1); + case 'c': return getnum(L, fmt, 1); case 'i': case 'I': { - int sz = getnum(fmt, sizeof(int)); + int sz = getnum(L, fmt, sizeof(int)); if (sz > MAXINTSIZE) luaL_error(L, "integral size %d is larger than limit of %d", sz, MAXINTSIZE); @@ -150,7 +152,7 @@ static void controloptions (lua_State *L, int opt, const char **fmt, case '>': h->endian = BIG; return; case '<': h->endian = LITTLE; return; case '!': { - int a = getnum(fmt, MAXALIGN); + int a = getnum(L, fmt, MAXALIGN); if (!isp2(a)) luaL_error(L, "alignment %d is not a power of 2", a); h->align = a; diff --git a/redis.submodule/redis.conf b/redis.submodule/redis.conf index d74ff98..e8eff27 100644 --- a/redis.submodule/redis.conf +++ b/redis.submodule/redis.conf @@ -24,7 +24,7 @@ # to customize a few per-server settings. Include files can include # other files, so use this wisely. # -# Notice option "include" won't be rewritten by command "CONFIG REWRITE" +# Note that option "include" won't be rewritten by command "CONFIG REWRITE" # from admin or Redis Sentinel. Since Redis always uses the last processed # line as value of a configuration directive, you'd better put includes # at the beginning of this file to avoid overwriting config change at runtime. @@ -46,27 +46,33 @@ ################################## NETWORK ##################################### # By default, if no "bind" configuration directive is specified, Redis listens -# for connections from all the network interfaces available on the server. +# for connections from all available network interfaces on the host machine. # It is possible to listen to just one or multiple selected interfaces using # the "bind" configuration directive, followed by one or more IP addresses. +# Each address can be prefixed by "-", which means that redis will not fail to +# start if the address is not available. Being not available only refers to +# addresses that does not correspond to any network interfece. Addresses that +# are already in use will always fail, and unsupported protocols will always BE +# silently skipped. # # Examples: # -# bind 192.168.1.100 10.0.0.1 -# bind 127.0.0.1 ::1 +# bind 192.168.1.100 10.0.0.1 # listens on two specific IPv4 addresses +# bind 127.0.0.1 ::1 # listens on loopback IPv4 and IPv6 +# bind * -::* # like the default, all available interfaces # # ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the # internet, binding to all the interfaces is dangerous and will expose the # instance to everybody on the internet. So by default we uncomment the -# following bind directive, that will force Redis to listen only into -# the IPv4 loopback interface address (this means Redis will be able to -# accept connections only from clients running into the same computer it -# is running). +# following bind directive, that will force Redis to listen only on the +# IPv4 and IPv6 (if available) loopback interface addresses (this means Redis +# will only be able to accept client connections from the same host that it is +# running on). # # IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES -# JUST COMMENT THE FOLLOWING LINE. +# JUST COMMENT OUT THE FOLLOWING LINE. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -bind 127.0.0.1 +bind 127.0.0.1 -::1 # Protected mode is a layer of security protection, in order to avoid that # Redis instances left open on the internet are accessed and exploited. @@ -93,8 +99,8 @@ port 6379 # TCP listen() backlog. # -# In high requests-per-second environments you need an high backlog in order -# to avoid slow clients connections issues. Note that the Linux kernel +# In high requests-per-second environments you need a high backlog in order +# to avoid slow clients connection issues. Note that the Linux kernel # will silently truncate it to the value of /proc/sys/net/core/somaxconn so # make sure to raise both the value of somaxconn and tcp_max_syn_backlog # in order to get the desired effect. @@ -106,7 +112,7 @@ tcp-backlog 511 # incoming connections. There is no default, so Redis will not listen # on a unix socket when not specified. # -# unixsocket /tmp/redis.sock +# unixsocket /run/redis.sock # unixsocketperm 700 # Close the connection after a client is idle for N seconds (0 to disable) @@ -118,8 +124,8 @@ timeout 0 # of communication. This is useful for two reasons: # # 1) Detect dead peers. -# 2) Take the connection alive from the point of view of network -# equipment in the middle. +# 2) Force network equipment in the middle to consider the connection to be +# alive. # # On Linux, the specified value (in seconds) is the period used to send ACKs. # Note that to close the connection the double of the time is needed. @@ -129,22 +135,144 @@ timeout 0 # Redis default starting with Redis 3.2.1. tcp-keepalive 300 +################################# TLS/SSL ##################################### + +# By default, TLS/SSL is disabled. To enable it, the "tls-port" configuration +# directive can be used to define TLS-listening ports. To enable TLS on the +# default port, use: +# +# port 0 +# tls-port 6379 + +# Configure a X.509 certificate and private key to use for authenticating the +# server to connected clients, masters or cluster peers. These files should be +# PEM formatted. +# +# tls-cert-file redis.crt +# tls-key-file redis.key +# +# If the key file is encrypted using a passphrase, it can be included here +# as well. +# +# tls-key-file-pass secret + +# Normally Redis uses the same certificate for both server functions (accepting +# connections) and client functions (replicating from a master, establishing +# cluster bus connections, etc.). +# +# Sometimes certificates are issued with attributes that designate them as +# client-only or server-only certificates. In that case it may be desired to use +# different certificates for incoming (server) and outgoing (client) +# connections. To do that, use the following directives: +# +# tls-client-cert-file client.crt +# tls-client-key-file client.key +# +# If the key file is encrypted using a passphrase, it can be included here +# as well. +# +# tls-client-key-file-pass secret + +# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: +# +# tls-dh-params-file redis.dh + +# Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL +# clients and peers. Redis requires an explicit configuration of at least one +# of these, and will not implicitly use the system wide configuration. +# +# tls-ca-cert-file ca.crt +# tls-ca-cert-dir /etc/ssl/certs + +# By default, clients (including replica servers) on a TLS port are required +# to authenticate using valid client side certificates. +# +# If "no" is specified, client certificates are not required and not accepted. +# If "optional" is specified, client certificates are accepted and must be +# valid if provided, but are not required. +# +# tls-auth-clients no +# tls-auth-clients optional + +# By default, a Redis replica does not attempt to establish a TLS connection +# with its master. +# +# Use the following directive to enable TLS on replication links. +# +# tls-replication yes + +# By default, the Redis Cluster bus uses a plain TCP connection. To enable +# TLS for the bus protocol, use the following directive: +# +# tls-cluster yes + +# By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended +# that older formally deprecated versions are kept disabled to reduce the attack surface. +# You can explicitly specify TLS versions to support. +# Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2", +# "TLSv1.3" (OpenSSL >= 1.1.1) or any combination. +# To enable only TLSv1.2 and TLSv1.3, use: +# +# tls-protocols "TLSv1.2 TLSv1.3" + +# Configure allowed ciphers. See the ciphers(1ssl) manpage for more information +# about the syntax of this string. +# +# Note: this configuration applies only to <= TLSv1.2. +# +# tls-ciphers DEFAULT:!MEDIUM + +# Configure allowed TLSv1.3 ciphersuites. See the ciphers(1ssl) manpage for more +# information about the syntax of this string, and specifically for TLSv1.3 +# ciphersuites. +# +# tls-ciphersuites TLS_CHACHA20_POLY1305_SHA256 + +# When choosing a cipher, use the server's preference instead of the client +# preference. By default, the server follows the client's preference. +# +# tls-prefer-server-ciphers yes + +# By default, TLS session caching is enabled to allow faster and less expensive +# reconnections by clients that support it. Use the following directive to disable +# caching. +# +# tls-session-caching no + +# Change the default number of TLS sessions cached. A zero value sets the cache +# to unlimited size. The default size is 20480. +# +# tls-session-cache-size 5000 + +# Change the default timeout of cached TLS sessions. The default timeout is 300 +# seconds. +# +# tls-session-cache-timeout 60 + ################################# GENERAL ##################################### # By default Redis does not run as a daemon. Use 'yes' if you need it. # Note that Redis will write a pid file in /var/run/redis.pid when daemonized. +# When Redis is supervised by upstart or systemd, this parameter has no impact. daemonize no # If you run Redis from upstart or systemd, Redis can interact with your # supervision tree. Options: # supervised no - no supervision interaction # supervised upstart - signal upstart by putting Redis into SIGSTOP mode +# requires "expect stop" in your upstart job config # supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET +# on startup, and updating Redis status on a regular +# basis. # supervised auto - detect upstart or systemd method based on # UPSTART_JOB or NOTIFY_SOCKET environment variables # Note: these supervision methods only signal "process is ready." -# They do not enable continuous liveness pings back to your supervisor. -supervised no +# They do not enable continuous pings back to your supervisor. +# +# The default is "no". To run under upstart/systemd, you can simply uncomment +# the line below: +# +# supervised auto # If a pid file is specified, Redis writes it where specified at startup # and removes it at exit. @@ -155,6 +283,9 @@ supervised no # # Creating a pid file is best effort: if Redis is not able to create it # nothing bad happens, the server will start and run normally. +# +# Note that on modern Linux systems "/run/redis.pid" is more conforming +# and should be used instead. pidfile /var/run/redis_6379.pid # Specify the server verbosity level. @@ -180,44 +311,76 @@ logfile "" # Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. # syslog-facility local0 +# To disable the built in crash log, which will possibly produce cleaner core +# dumps when they are needed, uncomment the following: +# +# crash-log-enabled no + +# To disable the fast memory check that's run as part of the crash log, which +# will possibly let redis terminate sooner, uncomment the following: +# +# crash-memcheck-enabled no + # Set the number of databases. The default database is DB 0, you can select # a different one on a per-connection basis using SELECT where # dbid is a number between 0 and 'databases'-1 databases 16 # By default Redis shows an ASCII art logo only when started to log to the -# standard output and if the standard output is a TTY. Basically this means -# that normally a logo is displayed only in interactive sessions. +# standard output and if the standard output is a TTY and syslog logging is +# disabled. Basically this means that normally a logo is displayed only in +# interactive sessions. # # However it is possible to force the pre-4.0 behavior and always show a # ASCII art logo in startup logs by setting the following option to yes. -always-show-logo yes +always-show-logo no + +# By default, Redis modifies the process title (as seen in 'top' and 'ps') to +# provide some runtime information. It is possible to disable this and leave +# the process name as executed by setting the following to no. +set-proc-title yes + +# When changing the process title, Redis uses the following template to construct +# the modified title. +# +# Template variables are specified in curly brackets. The following variables are +# supported: +# +# {title} Name of process as executed if parent, or type of child process. +# {listen-addr} Bind address or '*' followed by TCP or TLS port listening on, or +# Unix socket if only that's available. +# {server-mode} Special mode, i.e. "[sentinel]" or "[cluster]". +# {port} TCP port listening on, or 0. +# {tls-port} TLS port listening on, or 0. +# {unixsocket} Unix domain socket listening on, or "". +# {config-file} Name of configuration file used. +# +proc-title-template "{title} {listen-addr} {server-mode}" ################################ SNAPSHOTTING ################################ + +# Save the DB to disk. # -# Save the DB on disk: +# save # -# save +# Redis will save the DB if both the given number of seconds and the given +# number of write operations against the DB occurred. # -# Will save the DB if both the given number of seconds and the given -# number of write operations against the DB occurred. +# Snapshotting can be completely disabled with a single empty string argument +# as in following example: # -# In the example below the behaviour will be to save: -# after 900 sec (15 min) if at least 1 key changed -# after 300 sec (5 min) if at least 10 keys changed -# after 60 sec if at least 10000 keys changed +# save "" # -# Note: you can disable saving completely by commenting out all "save" lines. +# Unless specified otherwise, by default Redis will save the DB: +# * After 3600 seconds (an hour) if at least 1 key changed +# * After 300 seconds (5 minutes) if at least 100 keys changed +# * After 60 seconds if at least 10000 keys changed # -# It is also possible to remove all the previously configured save -# points by adding a save directive with a single empty string argument -# like in the following example: +# You can set these explicitly by uncommenting the three following lines. # -# save "" - -save 900 1 -save 300 10 -save 60 10000 +# save 3600 1 +# save 300 100 +# save 60 10000 # By default Redis will stop accepting writes if RDB snapshots are enabled # (at least one save point) and the latest background save failed. @@ -235,7 +398,7 @@ save 60 10000 stop-writes-on-bgsave-error yes # Compress string objects using LZF when dump .rdb databases? -# For default that's set to 'yes' as it's almost always a win. +# By default compression is enabled as it's almost always a win. # If you want to save some CPU in the saving child set it to 'no' but # the dataset will likely be bigger if you have compressible values or keys. rdbcompression yes @@ -249,9 +412,37 @@ rdbcompression yes # tell the loading code to skip the check. rdbchecksum yes +# Enables or disables full sanitation checks for ziplist and listpack etc when +# loading an RDB or RESTORE payload. This reduces the chances of a assertion or +# crash later on while processing commands. +# Options: +# no - Never perform full sanitation +# yes - Always perform full sanitation +# clients - Perform full sanitation only for user connections. +# Excludes: RDB files, RESTORE commands received from the master +# connection, and client connections which have the +# skip-sanitize-payload ACL flag. +# The default should be 'clients' but since it currently affects cluster +# resharding via MIGRATE, it is temporarily set to 'no' by default. +# +# sanitize-dump-payload no + # The filename where to dump the DB dbfilename dump.rdb +# Remove RDB files used by replication in instances without persistence +# enabled. By default this option is disabled, however there are environments +# where for regulations or other security concerns, RDB files persisted on +# disk by masters in order to feed replicas, or stored on disk by replicas +# in order to load them for the initial synchronization, should be deleted +# ASAP. Note that this option ONLY WORKS in instances that have both AOF +# and RDB persistence disabled, otherwise is completely ignored. +# +# An alternative (and sometimes better) way to obtain the same effect is +# to use diskless replication on both master and replicas instances. However +# in the case of replicas, diskless is not always an option. +rdb-del-sync-files no + # The working directory. # # The DB will be written inside this directory, with the filename specified @@ -291,6 +482,17 @@ dir ./ # refuse the replica request. # # masterauth +# +# However this is not enough if you are using Redis ACLs (for Redis version +# 6 or greater), and the default user is not capable of running the PSYNC +# command and/or other commands needed for replication. In this case it's +# better to configure a special user to use with replication, and specify the +# masteruser configuration as such: +# +# masteruser +# +# When masteruser is specified, the replica will authenticate against its +# master using the new AUTH form: AUTH . # When a replica loses its connection with the master, or when the replication # is still in progress, the replica can act in two different ways: @@ -299,11 +501,11 @@ dir ./ # still reply to client requests, possibly with out of date data, or the # data set may just be empty if this is the first synchronization. # -# 2) if replica-serve-stale-data is set to 'no' the replica will reply with -# an error "SYNC with master in progress" to all the kind of commands -# but to INFO, replicaOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, -# SUBSCRIBE, UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, -# COMMAND, POST, HOST: and LATENCY. +# 2) If replica-serve-stale-data is set to 'no' the replica will reply with +# an error "SYNC with master in progress" to all commands except: +# INFO, REPLICAOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, SUBSCRIBE, +# UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, COMMAND, POST, +# HOST and LATENCY. # replica-serve-stale-data yes @@ -325,13 +527,11 @@ replica-read-only yes # Replication SYNC strategy: disk or socket. # -# ------------------------------------------------------- -# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY -# ------------------------------------------------------- +# New replicas and reconnecting replicas that are not able to continue the +# replication process just receiving differences, need to do what is called a +# "full synchronization". An RDB file is transmitted from the master to the +# replicas. # -# New replicas and reconnecting replicas that are not able to continue the replication -# process just receiving differences, need to do what is called a "full -# synchronization". An RDB file is transmitted from the master to the replicas. # The transmission can happen in two different ways: # # 1) Disk-backed: The Redis master creates a new process that writes the RDB @@ -341,14 +541,14 @@ replica-read-only yes # RDB file to replica sockets, without touching the disk at all. # # With disk-backed replication, while the RDB file is generated, more replicas -# can be queued and served with the RDB file as soon as the current child producing -# the RDB file finishes its work. With diskless replication instead once -# the transfer starts, new replicas arriving will be queued and a new transfer -# will start when the current one terminates. +# can be queued and served with the RDB file as soon as the current child +# producing the RDB file finishes its work. With diskless replication instead +# once the transfer starts, new replicas arriving will be queued and a new +# transfer will start when the current one terminates. # # When diskless replication is used, the master waits a configurable amount of -# time (in seconds) before starting the transfer in the hope that multiple replicas -# will arrive and the transfer can be parallelized. +# time (in seconds) before starting the transfer in the hope that multiple +# replicas will arrive and the transfer can be parallelized. # # With slow disks and fast (large bandwidth) networks, diskless replication # works better. @@ -359,16 +559,42 @@ repl-diskless-sync no # to the replicas. # # This is important since once the transfer starts, it is not possible to serve -# new replicas arriving, that will be queued for the next RDB transfer, so the server -# waits a delay in order to let more replicas arrive. +# new replicas arriving, that will be queued for the next RDB transfer, so the +# server waits a delay in order to let more replicas arrive. # # The delay is specified in seconds, and by default is 5 seconds. To disable # it entirely just set it to 0 seconds and the transfer will start ASAP. repl-diskless-sync-delay 5 -# Replicas send PINGs to server in a predefined interval. It's possible to change -# this interval with the repl_ping_replica_period option. The default value is 10 -# seconds. +# ----------------------------------------------------------------------------- +# WARNING: RDB diskless load is experimental. Since in this setup the replica +# does not immediately store an RDB on disk, it may cause data loss during +# failovers. RDB diskless load + Redis modules not handling I/O reads may also +# cause Redis to abort in case of I/O errors during the initial synchronization +# stage with the master. Use only if you know what you are doing. +# ----------------------------------------------------------------------------- +# +# Replica can load the RDB it reads from the replication link directly from the +# socket, or store the RDB to a file and read that file after it was completely +# received from the master. +# +# In many cases the disk is slower than the network, and storing and loading +# the RDB file may increase replication time (and even increase the master's +# Copy on Write memory and salve buffers). +# However, parsing the RDB file directly from the socket may mean that we have +# to flush the contents of the current database before the full rdb was +# received. For this reason we have the following options: +# +# "disabled" - Don't use diskless load (store the rdb file to the disk first) +# "on-empty-db" - Use diskless load only when it is completely safe. +# "swapdb" - Keep a copy of the current db contents in RAM while parsing +# the data directly from the socket. note that this requires +# sufficient memory, if you don't have it, you risk an OOM kill. +repl-diskless-load disabled + +# Replicas send PINGs to server in a predefined interval. It's possible to +# change this interval with the repl_ping_replica_period option. The default +# value is 10 seconds. # # repl-ping-replica-period 10 @@ -380,7 +606,8 @@ repl-diskless-sync-delay 5 # # It is important to make sure that this value is greater than the value # specified for repl-ping-replica-period otherwise a timeout will be detected -# every time there is low traffic between the master and the replica. +# every time there is low traffic between the master and the replica. The default +# value is 60 seconds. # # repl-timeout 60 @@ -400,38 +627,38 @@ repl-diskless-sync-delay 5 repl-disable-tcp-nodelay no # Set the replication backlog size. The backlog is a buffer that accumulates -# replica data when replicas are disconnected for some time, so that when a replica -# wants to reconnect again, often a full resync is not needed, but a partial -# resync is enough, just passing the portion of data the replica missed while -# disconnected. +# replica data when replicas are disconnected for some time, so that when a +# replica wants to reconnect again, often a full resync is not needed, but a +# partial resync is enough, just passing the portion of data the replica +# missed while disconnected. # -# The bigger the replication backlog, the longer the time the replica can be -# disconnected and later be able to perform a partial resynchronization. +# The bigger the replication backlog, the longer the replica can endure the +# disconnect and later be able to perform a partial resynchronization. # -# The backlog is only allocated once there is at least a replica connected. +# The backlog is only allocated if there is at least one replica connected. # # repl-backlog-size 1mb -# After a master has no longer connected replicas for some time, the backlog -# will be freed. The following option configures the amount of seconds that -# need to elapse, starting from the time the last replica disconnected, for -# the backlog buffer to be freed. +# After a master has no connected replicas for some time, the backlog will be +# freed. The following option configures the amount of seconds that need to +# elapse, starting from the time the last replica disconnected, for the backlog +# buffer to be freed. # # Note that replicas never free the backlog for timeout, since they may be # promoted to masters later, and should be able to correctly "partially -# resynchronize" with the replicas: hence they should always accumulate backlog. +# resynchronize" with other replicas: hence they should always accumulate backlog. # # A value of 0 means to never release the backlog. # # repl-backlog-ttl 3600 -# The replica priority is an integer number published by Redis in the INFO output. -# It is used by Redis Sentinel in order to select a replica to promote into a -# master if the master is no longer working correctly. +# The replica priority is an integer number published by Redis in the INFO +# output. It is used by Redis Sentinel in order to select a replica to promote +# into a master if the master is no longer working correctly. # # A replica with a low priority number is considered better for promotion, so -# for instance if there are three replicas with priority 10, 100, 25 Sentinel will -# pick the one with priority 10, that is the lowest. +# for instance if there are three replicas with priority 10, 100, 25 Sentinel +# will pick the one with priority 10, that is the lowest. # # However a special priority of 0 marks the replica as not able to perform the # role of master, so a replica with priority of 0 will never be selected by @@ -440,6 +667,18 @@ repl-disable-tcp-nodelay no # By default the priority is 100. replica-priority 100 +# ----------------------------------------------------------------------------- +# By default, Redis Sentinel includes all replicas in its reports. A replica +# can be excluded from Redis Sentinel's announcements. An unannounced replica +# will be ignored by the 'sentinel replicas ' command and won't be +# exposed to Redis Sentinel's clients. +# +# This option does not change the behavior of replica-priority. Even with +# replica-announced set to 'no', the replica can be promoted to master. To +# prevent this behavior, set replica-priority to 0. +# +# replica-announced yes + # It is possible for a master to stop accepting writes if there are less than # N replicas connected, having a lag less or equal than M seconds. # @@ -469,8 +708,8 @@ replica-priority 100 # Another place where this info is available is in the output of the # "ROLE" command of a master. # -# The listed IP and address normally reported by a replica is obtained -# in the following way: +# The listed IP address and port normally reported by a replica is +# obtained in the following way: # # IP: The address is auto detected by checking the peer address # of the socket used by the replica to connect with the master. @@ -480,7 +719,7 @@ replica-priority 100 # listen for connections. # # However when port forwarding or Network Address Translation (NAT) is -# used, the replica may be actually reachable via different IP and port +# used, the replica may actually be reachable via different IP and port # pairs. The following two options can be used by a replica in order to # report to its master a specific set of IP and port, so that both INFO # and ROLE will report those values. @@ -491,22 +730,204 @@ replica-priority 100 # replica-announce-ip 5.5.5.5 # replica-announce-port 1234 -################################## SECURITY ################################### +############################### KEYS TRACKING ################################# -# Require clients to issue AUTH before processing any other -# commands. This might be useful in environments in which you do not trust -# others with access to the host running redis-server. +# Redis implements server assisted support for client side caching of values. +# This is implemented using an invalidation table that remembers, using +# a radix key indexed by key name, what clients have which keys. In turn +# this is used in order to send invalidation messages to clients. Please +# check this page to understand more about the feature: +# +# https://redis.io/topics/client-side-caching +# +# When tracking is enabled for a client, all the read only queries are assumed +# to be cached: this will force Redis to store information in the invalidation +# table. When keys are modified, such information is flushed away, and +# invalidation messages are sent to the clients. However if the workload is +# heavily dominated by reads, Redis could use more and more memory in order +# to track the keys fetched by many clients. # -# This should stay commented out for backward compatibility and because most -# people do not need auth (e.g. they run their own servers). +# For this reason it is possible to configure a maximum fill value for the +# invalidation table. By default it is set to 1M of keys, and once this limit +# is reached, Redis will start to evict keys in the invalidation table +# even if they were not modified, just to reclaim memory: this will in turn +# force the clients to invalidate the cached values. Basically the table +# maximum size is a trade off between the memory you want to spend server +# side to track information about who cached what, and the ability of clients +# to retain cached objects in memory. # -# Warning: since Redis is pretty fast an outside user can try up to -# 150k passwords per second against a good box. This means that you should -# use a very strong password otherwise it will be very easy to break. +# If you set the value to 0, it means there are no limits, and Redis will +# retain as many keys as needed in the invalidation table. +# In the "stats" INFO section, you can find information about the number of +# keys in the invalidation table at every given moment. +# +# Note: when key tracking is used in broadcasting mode, no memory is used +# in the server side so this setting is useless. +# +# tracking-table-max-keys 1000000 + +################################## SECURITY ################################### + +# Warning: since Redis is pretty fast, an outside user can try up to +# 1 million passwords per second against a modern box. This means that you +# should use very strong passwords, otherwise they will be very easy to break. +# Note that because the password is really a shared secret between the client +# and the server, and should not be memorized by any human, the password +# can be easily a long string from /dev/urandom or whatever, so by using a +# long and unguessable password no brute force attack will be possible. + +# Redis ACL users are defined in the following format: +# +# user ... acl rules ... +# +# For example: +# +# user worker +@list +@connection ~jobs:* on >ffa9203c493aa99 +# +# The special username "default" is used for new connections. If this user +# has the "nopass" rule, then new connections will be immediately authenticated +# as the "default" user without the need of any password provided via the +# AUTH command. Otherwise if the "default" user is not flagged with "nopass" +# the connections will start in not authenticated state, and will require +# AUTH (or the HELLO command AUTH option) in order to be authenticated and +# start to work. +# +# The ACL rules that describe what a user can do are the following: +# +# on Enable the user: it is possible to authenticate as this user. +# off Disable the user: it's no longer possible to authenticate +# with this user, however the already authenticated connections +# will still work. +# skip-sanitize-payload RESTORE dump-payload sanitation is skipped. +# sanitize-payload RESTORE dump-payload is sanitized (default). +# + Allow the execution of that command +# - Disallow the execution of that command +# +@ Allow the execution of all the commands in such category +# with valid categories are like @admin, @set, @sortedset, ... +# and so forth, see the full list in the server.c file where +# the Redis command table is described and defined. +# The special category @all means all the commands, but currently +# present in the server, and that will be loaded in the future +# via modules. +# +|subcommand Allow a specific subcommand of an otherwise +# disabled command. Note that this form is not +# allowed as negative like -DEBUG|SEGFAULT, but +# only additive starting with "+". +# allcommands Alias for +@all. Note that it implies the ability to execute +# all the future commands loaded via the modules system. +# nocommands Alias for -@all. +# ~ Add a pattern of keys that can be mentioned as part of +# commands. For instance ~* allows all the keys. The pattern +# is a glob-style pattern like the one of KEYS. +# It is possible to specify multiple patterns. +# allkeys Alias for ~* +# resetkeys Flush the list of allowed keys patterns. +# & Add a glob-style pattern of Pub/Sub channels that can be +# accessed by the user. It is possible to specify multiple channel +# patterns. +# allchannels Alias for &* +# resetchannels Flush the list of allowed channel patterns. +# > Add this password to the list of valid password for the user. +# For example >mypass will add "mypass" to the list. +# This directive clears the "nopass" flag (see later). +# < Remove this password from the list of valid passwords. +# nopass All the set passwords of the user are removed, and the user +# is flagged as requiring no password: it means that every +# password will work against this user. If this directive is +# used for the default user, every new connection will be +# immediately authenticated with the default user without +# any explicit AUTH command required. Note that the "resetpass" +# directive will clear this condition. +# resetpass Flush the list of allowed passwords. Moreover removes the +# "nopass" status. After "resetpass" the user has no associated +# passwords and there is no way to authenticate without adding +# some password (or setting it as "nopass" later). +# reset Performs the following actions: resetpass, resetkeys, off, +# -@all. The user returns to the same state it has immediately +# after its creation. +# +# ACL rules can be specified in any order: for instance you can start with +# passwords, then flags, or key patterns. However note that the additive +# and subtractive rules will CHANGE MEANING depending on the ordering. +# For instance see the following example: +# +# user alice on +@all -DEBUG ~* >somepassword +# +# This will allow "alice" to use all the commands with the exception of the +# DEBUG command, since +@all added all the commands to the set of the commands +# alice can use, and later DEBUG was removed. However if we invert the order +# of two ACL rules the result will be different: +# +# user alice on -DEBUG +@all ~* >somepassword +# +# Now DEBUG was removed when alice had yet no commands in the set of allowed +# commands, later all the commands are added, so the user will be able to +# execute everything. +# +# Basically ACL rules are processed left-to-right. +# +# For more information about ACL configuration please refer to +# the Redis web site at https://redis.io/topics/acl + +# ACL LOG +# +# The ACL Log tracks failed commands and authentication events associated +# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked +# by ACLs. The ACL Log is stored in memory. You can reclaim memory with +# ACL LOG RESET. Define the maximum entry length of the ACL Log below. +acllog-max-len 128 + +# Using an external ACL file +# +# Instead of configuring users here in this file, it is possible to use +# a stand-alone file just listing users. The two methods cannot be mixed: +# if you configure users here and at the same time you activate the external +# ACL file, the server will refuse to start. +# +# The format of the external ACL user file is exactly the same as the +# format that is used inside redis.conf to describe users. +# +# aclfile /etc/redis/users.acl + +# IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatibility +# layer on top of the new ACL system. The option effect will be just setting +# the password for the default user. Clients will still authenticate using +# AUTH as usually, or more explicitly with AUTH default +# if they follow the new protocol: both will work. +# +# The requirepass is not compatable with aclfile option and the ACL LOAD +# command, these will cause requirepass to be ignored. # # requirepass foobared -# Command renaming. +# New users are initialized with restrictive permissions by default, via the +# equivalent of this ACL rule 'off resetkeys -@all'. Starting with Redis 6.2, it +# is possible to manage access to Pub/Sub channels with ACL rules as well. The +# default Pub/Sub channels permission if new users is controlled by the +# acl-pubsub-default configuration directive, which accepts one of these values: +# +# allchannels: grants access to all Pub/Sub channels +# resetchannels: revokes access to all Pub/Sub channels +# +# To ensure backward compatibility while upgrading Redis 6.0, acl-pubsub-default +# defaults to the 'allchannels' permission. +# +# Future compatibility note: it is very likely that in a future version of Redis +# the directive's default of 'allchannels' will be changed to 'resetchannels' in +# order to provide better out-of-the-box Pub/Sub security. Therefore, it is +# recommended that you explicitly define Pub/Sub permissions for all users +# rather then rely on implicit default values. Once you've set explicit +# Pub/Sub for all existing users, you should uncomment the following line. +# +# acl-pubsub-default resetchannels + +# Command renaming (DEPRECATED). +# +# ------------------------------------------------------------------------ +# WARNING: avoid using this option if possible. Instead use ACLs to remove +# commands from the default user, and put them only in some admin user you +# create for administrative purposes. +# ------------------------------------------------------------------------ # # It is possible to change the name of dangerous commands in a shared # environment. For instance the CONFIG command may be renamed into something @@ -536,6 +957,11 @@ replica-priority 100 # Once the limit is reached Redis will close all the new connections sending # an error 'max number of clients reached'. # +# IMPORTANT: When Redis Cluster is used, the max number of connections is also +# shared with the cluster bus: every node in the cluster will use two +# connections, one incoming and another outgoing. It is important to size the +# limit accordingly in case of very large clusters. +# # maxclients 10000 ############################## MEMORY MANAGEMENT ################################ @@ -566,13 +992,13 @@ replica-priority 100 # maxmemory # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory -# is reached. You can select among five behaviors: +# is reached. You can select one from the following behaviors: # -# volatile-lru -> Evict using approximated LRU among the keys with an expire set. +# volatile-lru -> Evict using approximated LRU, only keys with an expire set. # allkeys-lru -> Evict any key using approximated LRU. -# volatile-lfu -> Evict using approximated LFU among the keys with an expire set. +# volatile-lfu -> Evict using approximated LFU, only keys with an expire set. # allkeys-lfu -> Evict any key using approximated LFU. -# volatile-random -> Remove a random key among the ones with an expire set. +# volatile-random -> Remove a random key having an expire set. # allkeys-random -> Remove a random key, any key. # volatile-ttl -> Remove the key with the nearest expire time (minor TTL) # noeviction -> Don't evict anything, just return an error on write operations. @@ -583,14 +1009,12 @@ replica-priority 100 # Both LRU, LFU and volatile-ttl are implemented using approximated # randomized algorithms. # -# Note: with any of the above policies, Redis will return an error on write -# operations, when there are no suitable keys for eviction. -# -# At the date of writing these commands are: set setnx setex append -# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd -# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby -# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby -# getset mset msetnx exec sort +# Note: with any of the above policies, when there are no suitable keys for +# eviction, Redis will return an error on write operations that require +# more memory. These are usually commands that create new keys, add data or +# modify existing keys. A few examples are: SET, INCR, HSET, LPUSH, SUNIONSTORE, +# SORT (due to the STORE argument), and EXEC (if the transaction includes any +# command that requires memory). # # The default is: # @@ -598,8 +1022,8 @@ replica-priority 100 # LRU, LFU and minimal TTL algorithms are not precise algorithms but approximated # algorithms (in order to save memory), so you can tune it for speed or -# accuracy. For default Redis will check five keys and pick the one that was -# used less recently, you can change the sample size using the following +# accuracy. By default Redis will check five keys and pick the one that was +# used least recently, you can change the sample size using the following # configuration directive. # # The default of 5 produces good enough results. 10 Approximates very closely @@ -607,26 +1031,51 @@ replica-priority 100 # # maxmemory-samples 5 +# Eviction processing is designed to function well with the default setting. +# If there is an unusually large amount of write traffic, this value may need to +# be increased. Decreasing this value may reduce latency at the risk of +# eviction processing effectiveness +# 0 = minimum latency, 10 = default, 100 = process without regard to latency +# +# maxmemory-eviction-tenacity 10 + # Starting from Redis 5, by default a replica will ignore its maxmemory setting # (unless it is promoted to master after a failover or manually). It means # that the eviction of keys will be just handled by the master, sending the # DEL commands to the replica as keys evict in the master side. # # This behavior ensures that masters and replicas stay consistent, and is usually -# what you want, however if your replica is writable, or you want the replica to have -# a different memory setting, and you are sure all the writes performed to the -# replica are idempotent, then you may change this default (but be sure to understand -# what you are doing). +# what you want, however if your replica is writable, or you want the replica +# to have a different memory setting, and you are sure all the writes performed +# to the replica are idempotent, then you may change this default (but be sure +# to understand what you are doing). # # Note that since the replica by default does not evict, it may end using more # memory than the one set via maxmemory (there are certain buffers that may -# be larger on the replica, or data structures may sometimes take more memory and so -# forth). So make sure you monitor your replicas and make sure they have enough -# memory to never hit a real out-of-memory condition before the master hits -# the configured maxmemory setting. +# be larger on the replica, or data structures may sometimes take more memory +# and so forth). So make sure you monitor your replicas and make sure they +# have enough memory to never hit a real out-of-memory condition before the +# master hits the configured maxmemory setting. # # replica-ignore-maxmemory yes +# Redis reclaims expired keys in two ways: upon access when those keys are +# found to be expired, and also in background, in what is called the +# "active expire key". The key space is slowly and interactively scanned +# looking for expired keys to reclaim, so that it is possible to free memory +# of keys that are expired and will never be accessed again in a short time. +# +# The default effort of the expire cycle will try to avoid having more than +# ten percent of expired keys still in memory, and will try to avoid consuming +# more than 25% of total memory and to add latency to the system. However +# it is possible to increase the expire "effort" that is normally set to +# "1", to a greater value, up to the value "10". At its maximum value the +# system will use more CPU, longer cycles (and technically may introduce +# more latency), and will tolerate less already expired keys still present +# in the system. It's a tradeoff between memory, CPU and latency. +# +# active-expire-effort 1 + ############################# LAZY FREEING #################################### # Redis has two primitives to delete keys. One is called DEL and is a blocking @@ -669,13 +1118,117 @@ replica-priority 100 # In all the above cases the default is to delete objects in a blocking way, # like if DEL was called. However you can configure each case specifically # in order to instead release memory in a non-blocking way like if UNLINK -# was called, using the following configuration directives: +# was called, using the following configuration directives. lazyfree-lazy-eviction no lazyfree-lazy-expire no lazyfree-lazy-server-del no replica-lazy-flush no +# It is also possible, for the case when to replace the user code DEL calls +# with UNLINK calls is not easy, to modify the default behavior of the DEL +# command to act exactly like UNLINK, using the following configuration +# directive: + +lazyfree-lazy-user-del no + +# FLUSHDB, FLUSHALL, and SCRIPT FLUSH support both asynchronous and synchronous +# deletion, which can be controlled by passing the [SYNC|ASYNC] flags into the +# commands. When neither flag is passed, this directive will be used to determine +# if the data should be deleted asynchronously. + +lazyfree-lazy-user-flush no + +################################ THREADED I/O ################################# + +# Redis is mostly single threaded, however there are certain threaded +# operations such as UNLINK, slow I/O accesses and other things that are +# performed on side threads. +# +# Now it is also possible to handle Redis clients socket reads and writes +# in different I/O threads. Since especially writing is so slow, normally +# Redis users use pipelining in order to speed up the Redis performances per +# core, and spawn multiple instances in order to scale more. Using I/O +# threads it is possible to easily speedup two times Redis without resorting +# to pipelining nor sharding of the instance. +# +# By default threading is disabled, we suggest enabling it only in machines +# that have at least 4 or more cores, leaving at least one spare core. +# Using more than 8 threads is unlikely to help much. We also recommend using +# threaded I/O only if you actually have performance problems, with Redis +# instances being able to use a quite big percentage of CPU time, otherwise +# there is no point in using this feature. +# +# So for instance if you have a four cores boxes, try to use 2 or 3 I/O +# threads, if you have a 8 cores, try to use 6 threads. In order to +# enable I/O threads use the following configuration directive: +# +# io-threads 4 +# +# Setting io-threads to 1 will just use the main thread as usual. +# When I/O threads are enabled, we only use threads for writes, that is +# to thread the write(2) syscall and transfer the client buffers to the +# socket. However it is also possible to enable threading of reads and +# protocol parsing using the following configuration directive, by setting +# it to yes: +# +# io-threads-do-reads no +# +# Usually threading reads doesn't help much. +# +# NOTE 1: This configuration directive cannot be changed at runtime via +# CONFIG SET. Aso this feature currently does not work when SSL is +# enabled. +# +# NOTE 2: If you want to test the Redis speedup using redis-benchmark, make +# sure you also run the benchmark itself in threaded mode, using the +# --threads option to match the number of Redis threads, otherwise you'll not +# be able to notice the improvements. + +############################ KERNEL OOM CONTROL ############################## + +# On Linux, it is possible to hint the kernel OOM killer on what processes +# should be killed first when out of memory. +# +# Enabling this feature makes Redis actively control the oom_score_adj value +# for all its processes, depending on their role. The default scores will +# attempt to have background child processes killed before all others, and +# replicas killed before masters. +# +# Redis supports three options: +# +# no: Don't make changes to oom-score-adj (default). +# yes: Alias to "relative" see below. +# absolute: Values in oom-score-adj-values are written as is to the kernel. +# relative: Values are used relative to the initial value of oom_score_adj when +# the server starts and are then clamped to a range of -1000 to 1000. +# Because typically the initial value is 0, they will often match the +# absolute values. +oom-score-adj no + +# When oom-score-adj is used, this directive controls the specific values used +# for master, replica and background child processes. Values range -2000 to +# 2000 (higher means more likely to be killed). +# +# Unprivileged processes (not root, and without CAP_SYS_RESOURCE capabilities) +# can freely increase their value, but not decrease it below its initial +# settings. This means that setting oom-score-adj to "relative" and setting the +# oom-score-adj-values to positive values will always succeed. +oom-score-adj-values 0 200 800 + + +#################### KERNEL transparent hugepage CONTROL ###################### + +# Usually the kernel Transparent Huge Pages control is set to "madvise" or +# or "never" by default (/sys/kernel/mm/transparent_hugepage/enabled), in which +# case this config has no effect. On systems in which it is set to "always", +# redis will attempt to disable it specifically for the redis process in order +# to avoid latency problems specifically with fork(2) and CoW. +# If for some reason you prefer to keep it enabled, you can set this config to +# "no" and the kernel global to "always". + +disable-thp yes + ############################## APPEND ONLY MODE ############################### # By default Redis asynchronously dumps the dataset on disk. This mode is @@ -694,7 +1247,7 @@ replica-lazy-flush no # If the AOF is enabled on startup Redis will load the AOF, that is the file # with the better durability guarantees. # -# Please check http://redis.io/topics/persistence for more information. +# Please check https://redis.io/topics/persistence for more information. appendonly no @@ -800,8 +1353,8 @@ aof-load-truncated yes # # [RDB file][AOF tail] # -# When loading Redis recognizes that the AOF file starts with the "REDIS" -# string and loads the prefixed RDB file, and continues loading the AOF +# When loading, Redis recognizes that the AOF file starts with the "REDIS" +# string and loads the prefixed RDB file, then continues loading the AOF # tail. aof-use-rdb-preamble yes @@ -815,7 +1368,7 @@ aof-use-rdb-preamble yes # # When a long running script exceeds the maximum execution time only the # SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be -# used to stop a script that did not yet called write commands. The second +# used to stop a script that did not yet call any write commands. The second # is the only way to shut down the server in the case a write command was # already issued by the script but the user doesn't want to wait for the natural # termination of the script. @@ -841,7 +1394,7 @@ lua-time-limit 5000 # Cluster node timeout is the amount of milliseconds a node must be unreachable # for it to be considered in failure state. -# Most other internal time limits are multiple of the node timeout. +# Most other internal time limits are a multiple of the node timeout. # # cluster-node-timeout 15000 @@ -868,18 +1421,18 @@ lua-time-limit 5000 # the failover if, since the last interaction with the master, the time # elapsed is greater than: # -# (node-timeout * replica-validity-factor) + repl-ping-replica-period +# (node-timeout * cluster-replica-validity-factor) + repl-ping-replica-period # -# So for example if node-timeout is 30 seconds, and the replica-validity-factor +# So for example if node-timeout is 30 seconds, and the cluster-replica-validity-factor # is 10, and assuming a default repl-ping-replica-period of 10 seconds, the # replica will not try to failover if it was not able to talk with the master # for longer than 310 seconds. # -# A large replica-validity-factor may allow replicas with too old data to failover +# A large cluster-replica-validity-factor may allow replicas with too old data to failover # a master, while a too small value may prevent the cluster from being able to # elect a replica at all. # -# For maximum availability, it is possible to set the replica-validity-factor +# For maximum availability, it is possible to set the cluster-replica-validity-factor # to a value of 0, which means, that replicas will always try to failover the # master regardless of the last time they interacted with the master. # (However they'll always try to apply a delay proportional to their @@ -903,14 +1456,23 @@ lua-time-limit 5000 # master in your cluster. # # Default is 1 (replicas migrate only if their masters remain with at least -# one replica). To disable migration just set it to a very large value. +# one replica). To disable migration just set it to a very large value or +# set cluster-allow-replica-migration to 'no'. # A value of 0 can be set but is useful only for debugging and dangerous # in production. # # cluster-migration-barrier 1 +# Turning off this option allows to use less automatic cluster configuration. +# It both disables migration to orphaned masters and migration from masters +# that became empty. +# +# Default is 'yes' (allow automatic migrations). +# +# cluster-allow-replica-migration yes + # By default Redis Cluster nodes stop accepting queries if they detect there -# is at least an hash slot uncovered (no available node is serving it). +# is at least a hash slot uncovered (no available node is serving it). # This way if the cluster is partially down (for example a range of hash slots # are no longer covered) all the cluster becomes, eventually, unavailable. # It automatically returns available as soon as all the slots are covered again. @@ -923,7 +1485,7 @@ lua-time-limit 5000 # cluster-require-full-coverage yes # This option, when set to yes, prevents replicas from trying to failover its -# master during master failures. However the master can still perform a +# master during master failures. However the replica can still perform a # manual failover, if forced to do so. # # This is useful in different scenarios, especially in the case of multiple @@ -932,8 +1494,24 @@ lua-time-limit 5000 # # cluster-replica-no-failover no +# This option, when set to yes, allows nodes to serve read traffic while the +# the cluster is in a down state, as long as it believes it owns the slots. +# +# This is useful for two cases. The first case is for when an application +# doesn't require consistency of data during node failures or network partitions. +# One example of this is a cache, where as long as the node has the data it +# should be able to serve it. +# +# The second use case is for configurations that don't meet the recommended +# three shards but want to enable cluster mode and scale later. A +# master outage in a 1 or 2 shard configuration causes a read/write outage to the +# entire cluster without this option set, with it set there is only a write outage. +# Without a quorum of masters, slot ownership will not change automatically. +# +# cluster-allow-reads-when-down no + # In order to setup your cluster make sure to read the documentation -# available at http://redis.io web site. +# available at https://redis.io web site. ########################## CLUSTER DOCKER/NAT support ######################## @@ -943,16 +1521,21 @@ lua-time-limit 5000 # # In order to make Redis Cluster working in such environments, a static # configuration where each node knows its public address is needed. The -# following two options are used for this scope, and are: +# following four options are used for this scope, and are: # # * cluster-announce-ip # * cluster-announce-port +# * cluster-announce-tls-port # * cluster-announce-bus-port # -# Each instruct the node about its address, client port, and cluster message -# bus port. The information is then published in the header of the bus packets -# so that other nodes will be able to correctly map the address of the node -# publishing the information. +# Each instructs the node about its address, client ports (for connections +# without and with TLS) and cluster message bus port. The information is then +# published in the header of the bus packets so that other nodes will be able to +# correctly map the address of the node publishing the information. +# +# If cluster-tls is set to yes and cluster-announce-tls-port is omitted or set +# to zero, then cluster-announce-port refers to the TLS port. Note also that +# cluster-announce-tls-port has no effect if cluster-tls is set to no. # # If the above options are not used, the normal Redis Cluster auto-detection # will be used instead. @@ -960,12 +1543,13 @@ lua-time-limit 5000 # Note that when remapped, the bus port may not be at the fixed offset of # clients port + 10000, so you can specify any port and bus-port depending # on how they get remapped. If the bus-port is not set, a fixed offset of -# 10000 will be used as usually. +# 10000 will be used as usual. # # Example: # # cluster-announce-ip 10.1.1.5 -# cluster-announce-port 6379 +# cluster-announce-tls-port 6379 +# cluster-announce-port 0 # cluster-announce-bus-port 6380 ################################## SLOW LOG ################################### @@ -1016,7 +1600,7 @@ latency-monitor-threshold 0 ############################# EVENT NOTIFICATION ############################## # Redis can notify Pub/Sub clients about events happening in the key space. -# This feature is documented at http://redis.io/topics/notifications +# This feature is documented at https://redis.io/topics/notifications # # For instance if keyspace events notification is enabled, and a client # performs a DEL operation on key "foo" stored in the Database 0, two @@ -1038,7 +1622,12 @@ latency-monitor-threshold 0 # z Sorted set commands # x Expired events (events generated every time a key expires) # e Evicted events (events generated when a key is evicted for maxmemory) -# A Alias for g$lshzxe, so that the "AKE" string means all the events. +# t Stream commands +# d Module key type events +# m Key-miss events (Note: It is not included in the 'A' class) +# A Alias for g$lshzxetd, so that the "AKE" string means all the events +# (Except key-miss events which are excluded from 'A' due to their +# unique nature). # # The "notify-keyspace-events" takes as argument a string that is composed # of zero or multiple characters. The empty string means that notifications @@ -1059,6 +1648,64 @@ latency-monitor-threshold 0 # specify at least one of K or E, no events will be delivered. notify-keyspace-events "" +############################### GOPHER SERVER ################################# + +# Redis contains an implementation of the Gopher protocol, as specified in +# the RFC 1436 (https://www.ietf.org/rfc/rfc1436.txt). +# +# The Gopher protocol was very popular in the late '90s. It is an alternative +# to the web, and the implementation both server and client side is so simple +# that the Redis server has just 100 lines of code in order to implement this +# support. +# +# What do you do with Gopher nowadays? Well Gopher never *really* died, and +# lately there is a movement in order for the Gopher more hierarchical content +# composed of just plain text documents to be resurrected. Some want a simpler +# internet, others believe that the mainstream internet became too much +# controlled, and it's cool to create an alternative space for people that +# want a bit of fresh air. +# +# Anyway for the 10nth birthday of the Redis, we gave it the Gopher protocol +# as a gift. +# +# --- HOW IT WORKS? --- +# +# The Redis Gopher support uses the inline protocol of Redis, and specifically +# two kind of inline requests that were anyway illegal: an empty request +# or any request that starts with "/" (there are no Redis commands starting +# with such a slash). Normal RESP2/RESP3 requests are completely out of the +# path of the Gopher protocol implementation and are served as usual as well. +# +# If you open a connection to Redis when Gopher is enabled and send it +# a string like "/foo", if there is a key named "/foo" it is served via the +# Gopher protocol. +# +# In order to create a real Gopher "hole" (the name of a Gopher site in Gopher +# talking), you likely need a script like the following: +# +# https://github.com/antirez/gopher2redis +# +# --- SECURITY WARNING --- +# +# If you plan to put Redis on the internet in a publicly accessible address +# to server Gopher pages MAKE SURE TO SET A PASSWORD to the instance. +# Once a password is set: +# +# 1. The Gopher server (when enabled, not by default) will still serve +# content via Gopher. +# 2. However other commands cannot be called before the client will +# authenticate. +# +# So use the 'requirepass' option to protect your instance. +# +# Note that Gopher is not currently supported when 'io-threads-do-reads' +# is enabled. +# +# To enable Gopher support, uncomment the following line and set the option +# from no (the default) to yes. +# +# gopher-enabled no + ############################### ADVANCED CONFIG ############################### # Hashes are encoded using a memory efficient data structure when they have a @@ -1131,7 +1778,7 @@ hll-sparse-max-bytes 3000 # maximum number of items it may contain before switching to a new node when # appending new stream entries. If any of the following settings are set to # zero, the limit is ignored, so for instance it is possible to set just a -# max entires limit by setting max-bytes to 0 and max-entries to the desired +# max entries limit by setting max-bytes to 0 and max-entries to the desired # value. stream-node-max-bytes 4096 stream-node-max-entries 100 @@ -1202,8 +1849,8 @@ client-output-buffer-limit pubsub 32mb 8mb 60 # client-query-buffer-limit 1gb # In the Redis protocol, bulk requests, that are, elements representing single -# strings, are normally limited ot 512 mb. However you can change this limit -# here. +# strings, are normally limited to 512 mb. However you can change this limit +# here, but must be 1mb or greater # # proto-max-bulk-len 512mb @@ -1231,9 +1878,9 @@ hz 10 # # Since the default HZ value by default is conservatively set to 10, Redis # offers, and enables by default, the ability to use an adaptive HZ value -# which will temporary raise when there are many connected clients. +# which will temporarily raise when there are many connected clients. # -# When dynamic HZ is enabled, the actual configured HZ will be used as +# When dynamic HZ is enabled, the actual configured HZ will be used # as a baseline, but multiples of the configured HZ value will be actually # used as needed once more clients are connected. In this way an idle # instance will use very little CPU time while a busy instance will be @@ -1298,7 +1945,7 @@ rdb-save-incremental-fsync yes # for the key counter to be divided by two (or decremented if it has a value # less <= 10). # -# The default value for the lfu-decay-time is 1. A Special value of 0 means to +# The default value for the lfu-decay-time is 1. A special value of 0 means to # decay the counter every time it happens to be scanned. # # lfu-log-factor 10 @@ -1306,10 +1953,6 @@ rdb-save-incremental-fsync yes ########################### ACTIVE DEFRAGMENTATION ####################### # -# WARNING THIS FEATURE IS EXPERIMENTAL. However it was stress tested -# even in production and manually tested by multiple engineers for some -# time. -# # What is active defragmentation? # ------------------------------- # @@ -1322,7 +1965,7 @@ rdb-save-incremental-fsync yes # restart is needed in order to lower the fragmentation, or at least to flush # away all the data and create it again. However thanks to this feature # implemented by Oran Agra for Redis 4.0 this process can happen at runtime -# in an "hot" way, while the server is running. +# in a "hot" way, while the server is running. # # Basically when the fragmentation is over a certain level (see the # configuration options below) Redis will start to create new copies of the @@ -1349,7 +1992,7 @@ rdb-save-incremental-fsync yes # a good idea to leave the defaults untouched. # Enabled active defragmentation -# activedefrag yes +# activedefrag no # Minimum amount of fragmentation waste to start active defrag # active-defrag-ignore-bytes 100mb @@ -1360,13 +2003,49 @@ rdb-save-incremental-fsync yes # Maximum percentage of fragmentation at which we use maximum effort # active-defrag-threshold-upper 100 -# Minimal effort for defrag in CPU percentage -# active-defrag-cycle-min 5 +# Minimal effort for defrag in CPU percentage, to be used when the lower +# threshold is reached +# active-defrag-cycle-min 1 -# Maximal effort for defrag in CPU percentage -# active-defrag-cycle-max 75 +# Maximal effort for defrag in CPU percentage, to be used when the upper +# threshold is reached +# active-defrag-cycle-max 25 # Maximum number of set/hash/zset/list fields that will be processed from # the main dictionary scan # active-defrag-max-scan-fields 1000 +# Jemalloc background thread for purging will be enabled by default +jemalloc-bg-thread yes + +# It is possible to pin different threads and processes of Redis to specific +# CPUs in your system, in order to maximize the performances of the server. +# This is useful both in order to pin different Redis threads in different +# CPUs, but also in order to make sure that multiple Redis instances running +# in the same host will be pinned to different CPUs. +# +# Normally you can do this using the "taskset" command, however it is also +# possible to this via Redis configuration directly, both in Linux and FreeBSD. +# +# You can pin the server/IO threads, bio threads, aof rewrite child process, and +# the bgsave child process. The syntax to specify the cpu list is the same as +# the taskset command: +# +# Set redis server/io threads to cpu affinity 0,2,4,6: +# server_cpulist 0-7:2 +# +# Set bio threads to cpu affinity 1,3: +# bio_cpulist 1,3 +# +# Set aof rewrite child process to cpu affinity 8,9,10,11: +# aof_rewrite_cpulist 8-11 +# +# Set bgsave child process to cpu affinity 1,10,11 +# bgsave_cpulist 1,10-11 + +# In some cases redis will emit warnings and even refuse to start if it detects +# that the system is in bad state, it is possible to suppress these warnings +# by setting the following config which takes a space delimited list of warnings +# to suppress +# +# ignore-warnings ARM64-COW-BUG diff --git a/redis.submodule/runtest-cluster b/redis.submodule/runtest-cluster index 27829a5..a86e931 100755 --- a/redis.submodule/runtest-cluster +++ b/redis.submodule/runtest-cluster @@ -8,7 +8,7 @@ done if [ -z $TCLSH ] then - echo "You need tcl 8.5 or newer in order to run the Redis Sentinel test" + echo "You need tcl 8.5 or newer in order to run the Redis Cluster test" exit 1 fi $TCLSH tests/cluster/run.tcl $* diff --git a/redis.submodule/runtest-moduleapi b/redis.submodule/runtest-moduleapi index 8e1c0cb..56d1496 100755 --- a/redis.submodule/runtest-moduleapi +++ b/redis.submodule/runtest-moduleapi @@ -1,6 +1,7 @@ #!/bin/sh TCL_VERSIONS="8.5 8.6" TCLSH="" +[ -z "$MAKE" ] && MAKE=make for VERSION in $TCL_VERSIONS; do TCL=`which tclsh$VERSION 2>/dev/null` && TCLSH=$TCL @@ -8,9 +9,31 @@ done if [ -z $TCLSH ] then - echo "You need tcl 8.5 or newer in order to run the Redis test" + echo "You need tcl 8.5 or newer in order to run the Redis ModuleApi test" exit 1 fi -make -C tests/modules && \ -$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/testrdb "${@}" +$MAKE -C tests/modules && \ +$TCLSH tests/test_helper.tcl \ +--single unit/moduleapi/commandfilter \ +--single unit/moduleapi/basics \ +--single unit/moduleapi/fork \ +--single unit/moduleapi/testrdb \ +--single unit/moduleapi/infotest \ +--single unit/moduleapi/propagate \ +--single unit/moduleapi/hooks \ +--single unit/moduleapi/misc \ +--single unit/moduleapi/blockonkeys \ +--single unit/moduleapi/blockonbackground \ +--single unit/moduleapi/scan \ +--single unit/moduleapi/datatype \ +--single unit/moduleapi/auth \ +--single unit/moduleapi/keyspace_events \ +--single unit/moduleapi/blockedclient \ +--single unit/moduleapi/getkeys \ +--single unit/moduleapi/test_lazyfree \ +--single unit/moduleapi/defrag \ +--single unit/moduleapi/hash \ +--single unit/moduleapi/zset \ +--single unit/moduleapi/stream \ +"${@}" diff --git a/redis.submodule/sentinel.conf b/redis.submodule/sentinel.conf index bc9a705..8647379 100644 --- a/redis.submodule/sentinel.conf +++ b/redis.submodule/sentinel.conf @@ -102,6 +102,18 @@ sentinel monitor mymaster 127.0.0.1 6379 2 # # sentinel auth-pass mymaster MySUPER--secret-0123passw0rd +# sentinel auth-user +# +# This is useful in order to authenticate to instances having ACL capabilities, +# that is, running Redis 6.0 or greater. When just auth-pass is provided the +# Sentinel instance will authenticate to Redis using the old "AUTH " +# method. When also an username is provided, it will use "AUTH ". +# In the Redis servers side, the ACL to provide just minimal access to +# Sentinel instances, should be configured along the following lines: +# +# user sentinel-user >somepassword +client +subscribe +publish \ +# +ping +info +multi +slaveof +config +client +exec on + # sentinel down-after-milliseconds # # Number of milliseconds the master (or any attached replica or sentinel) should @@ -112,6 +124,73 @@ sentinel monitor mymaster 127.0.0.1 6379 2 # Default is 30 seconds. sentinel down-after-milliseconds mymaster 30000 +# IMPORTANT NOTE: starting with Redis 6.2 ACL capability is supported for +# Sentinel mode, please refer to the Redis website https://redis.io/topics/acl +# for more details. + +# Sentinel's ACL users are defined in the following format: +# +# user ... acl rules ... +# +# For example: +# +# user worker +@admin +@connection ~* on >ffa9203c493aa99 +# +# For more information about ACL configuration please refer to the Redis +# website at https://redis.io/topics/acl and redis server configuration +# template redis.conf. + +# ACL LOG +# +# The ACL Log tracks failed commands and authentication events associated +# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked +# by ACLs. The ACL Log is stored in memory. You can reclaim memory with +# ACL LOG RESET. Define the maximum entry length of the ACL Log below. +acllog-max-len 128 + +# Using an external ACL file +# +# Instead of configuring users here in this file, it is possible to use +# a stand-alone file just listing users. The two methods cannot be mixed: +# if you configure users here and at the same time you activate the external +# ACL file, the server will refuse to start. +# +# The format of the external ACL user file is exactly the same as the +# format that is used inside redis.conf to describe users. +# +# aclfile /etc/redis/sentinel-users.acl + +# requirepass +# +# You can configure Sentinel itself to require a password, however when doing +# so Sentinel will try to authenticate with the same password to all the +# other Sentinels. So you need to configure all your Sentinels in a given +# group with the same "requirepass" password. Check the following documentation +# for more info: https://redis.io/topics/sentinel +# +# IMPORTANT NOTE: starting with Redis 6.2 "requirepass" is a compatibility +# layer on top of the ACL system. The option effect will be just setting +# the password for the default user. Clients will still authenticate using +# AUTH as usually, or more explicitly with AUTH default +# if they follow the new protocol: both will work. +# +# New config files are advised to use separate authentication control for +# incoming connections (via ACL), and for outgoing connections (via +# sentinel-user and sentinel-pass) +# +# The requirepass is not compatable with aclfile option and the ACL LOAD +# command, these will cause requirepass to be ignored. + +# sentinel sentinel-user +# +# You can configure Sentinel to authenticate with other Sentinels with specific +# user name. + +# sentinel sentinel-pass +# +# The password for Sentinel to authenticate with other Sentinels. If sentinel-user +# is not configured, Sentinel will use 'default' user with sentinel-pass to authenticate. + # sentinel parallel-syncs # # How many replicas we can reconfigure to point to the new replica simultaneously @@ -239,6 +318,24 @@ sentinel deny-scripts-reconfig yes # SENTINEL SET can also be used in order to perform this configuration at runtime. # # In order to set a command back to its original name (undo the renaming), it -# is possible to just rename a command to itsef: +# is possible to just rename a command to itself: # # SENTINEL rename-command mymaster CONFIG CONFIG + +# HOSTNAMES SUPPORT +# +# Normally Sentinel uses only IP addresses and requires SENTINEL MONITOR +# to specify an IP address. Also, it requires the Redis replica-announce-ip +# keyword to specify only IP addresses. +# +# You may enable hostnames support by enabling resolve-hostnames. Note +# that you must make sure your DNS is configured properly and that DNS +# resolution does not introduce very long delays. +# +SENTINEL resolve-hostnames no + +# When resolve-hostnames is enabled, Sentinel still uses IP addresses +# when exposing instances to users, configuration files, etc. If you want +# to retain the hostnames when announced, enable announce-hostnames below. +# +SENTINEL announce-hostnames no diff --git a/redis.submodule/src/Makefile b/redis.submodule/src/Makefile index 2a68649..1001f5b 100644 --- a/redis.submodule/src/Makefile +++ b/redis.submodule/src/Makefile @@ -16,12 +16,17 @@ release_hdr := $(shell sh -c './mkreleasehdr.sh') uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not') OPTIMIZATION?=-O2 -DEPENDENCY_TARGETS=hiredis linenoise lua +DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram NODEPS:=clean distclean # Default settings -STD=-std=c99 -pedantic -DREDIS_STATIC='' +STD=-pedantic -DREDIS_STATIC='' + +# Use -Wno-c11-extensions on clang, either where explicitly used or on +# platforms we can assume it's being used. ifneq (,$(findstring clang,$(CC))) + STD+=-Wno-c11-extensions +else ifneq (,$(findstring FreeBSD,$(uname_S))) STD+=-Wno-c11-extensions endif @@ -29,9 +34,20 @@ endif WARN=-Wall -W -Wno-missing-field-initializers OPT=$(OPTIMIZATION) +# Detect if the compiler supports C11 _Atomic +C11_ATOMIC := $(shell sh -c 'echo "\#include " > foo.c; \ + $(CC) -std=c11 -c foo.c -o foo.o > /dev/null 2>&1; \ + if [ -f foo.o ]; then echo "yes"; rm foo.o; fi; rm foo.c') +ifeq ($(C11_ATOMIC),yes) + STD+=-std=c11 +else + STD+=-std=c99 +endif + PREFIX?=/usr/local INSTALL_BIN=$(PREFIX)/bin INSTALL=install +PKG_CONFIG?=pkg-config # Default allocator defaults to Jemalloc if it's not an ARM MALLOC=libc @@ -77,11 +93,18 @@ FINAL_LDFLAGS=$(LDFLAGS) $(REDIS_LDFLAGS) $(DEBUG) FINAL_LIBS=-lm DEBUG=-g -ggdb +# Linux ARM32 needs -latomic at linking time +ifneq (,$(findstring armv,$(uname_M))) + FINAL_LIBS+=-latomic +endif + ifeq ($(uname_S),SunOS) # SunOS - ifneq ($(@@),32bit) - CFLAGS+= -m64 - LDFLAGS+= -m64 + ifeq ($(findstring -m32,$(FINAL_CFLAGS)),) + CFLAGS+=-m64 + endif + ifeq ($(findstring -m32,$(FINAL_LDFLAGS)),) + LDFLAGS+=-m64 endif DEBUG=-g DEBUG_FLAGS=-g @@ -93,6 +116,16 @@ else ifeq ($(uname_S),Darwin) # Darwin FINAL_LIBS+= -ldl + # Homebrew's OpenSSL is not linked to /usr/local to avoid + # conflicts with the system's LibreSSL installation so it + # must be referenced explicitly during build. +ifeq ($(uname_M),arm64) + # Homebrew arm64 uses /opt/homebrew as HOMEBREW_PREFIX + OPENSSL_PREFIX?=/opt/homebrew/opt/openssl +else + # Homebrew x86/ppc uses /usr/local as HOMEBREW_PREFIX + OPENSSL_PREFIX?=/usr/local/opt/openssl +endif else ifeq ($(uname_S),AIX) # AIX @@ -109,13 +142,36 @@ ifeq ($(uname_S),OpenBSD) endif else +ifeq ($(uname_S),NetBSD) + # NetBSD + FINAL_LIBS+= -lpthread + ifeq ($(USE_BACKTRACE),yes) + FINAL_CFLAGS+= -DUSE_BACKTRACE -I/usr/pkg/include + FINAL_LDFLAGS+= -L/usr/pkg/lib + FINAL_LIBS+= -lexecinfo + endif +else ifeq ($(uname_S),FreeBSD) # FreeBSD FINAL_LIBS+= -lpthread -lexecinfo else ifeq ($(uname_S),DragonFly) - # FreeBSD + # DragonFly FINAL_LIBS+= -lpthread -lexecinfo +else +ifeq ($(uname_S),OpenBSD) + # OpenBSD + FINAL_LIBS+= -lpthread -lexecinfo +else +ifeq ($(uname_S),NetBSD) + # NetBSD + FINAL_LIBS+= -lpthread -lexecinfo +else +ifeq ($(uname_S),Haiku) + # Haiku + FINAL_CFLAGS+= -DBSD_SOURCE + FINAL_LDFLAGS+= -lbsd -lnetwork + FINAL_LIBS+= -lpthread else # All the other OSes (notably Linux) FINAL_LDFLAGS+= -rdynamic @@ -126,8 +182,47 @@ endif endif endif endif +endif +endif +endif +endif + +ifdef OPENSSL_PREFIX + OPENSSL_CFLAGS=-I$(OPENSSL_PREFIX)/include + OPENSSL_LDFLAGS=-L$(OPENSSL_PREFIX)/lib + # Also export OPENSSL_PREFIX so it ends up in deps sub-Makefiles + export OPENSSL_PREFIX +endif + # Include paths to dependencies -FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src +FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram + +# Determine systemd support and/or build preference (defaulting to auto-detection) +BUILD_WITH_SYSTEMD=no +LIBSYSTEMD_LIBS=-lsystemd + +# If 'USE_SYSTEMD' in the environment is neither "no" nor "yes", try to +# auto-detect libsystemd's presence and link accordingly. +ifneq ($(USE_SYSTEMD),no) + LIBSYSTEMD_PKGCONFIG := $(shell $(PKG_CONFIG) --exists libsystemd && echo $$?) +# If libsystemd cannot be detected, continue building without support for it +# (unless a later check tells us otherwise) +ifeq ($(LIBSYSTEMD_PKGCONFIG),0) + BUILD_WITH_SYSTEMD=yes + LIBSYSTEMD_LIBS=$(shell $(PKG_CONFIG) --libs libsystemd) +endif +endif + +# If 'USE_SYSTEMD' is set to "yes" use pkg-config if available or fall back to +# default -lsystemd. +ifeq ($(USE_SYSTEMD),yes) + BUILD_WITH_SYSTEMD=yes +endif + +ifeq ($(BUILD_WITH_SYSTEMD),yes) + FINAL_LIBS+=$(LIBSYSTEMD_LIBS) + FINAL_CFLAGS+= -DHAVE_LIBSYSTEMD +endif ifeq ($(MALLOC),tcmalloc) FINAL_CFLAGS+= -DUSE_TCMALLOC @@ -145,6 +240,35 @@ ifeq ($(MALLOC),jemalloc) FINAL_LIBS := ../deps/jemalloc/lib/libjemalloc.a $(FINAL_LIBS) endif +ifeq ($(BUILD_TLS),yes) + FINAL_CFLAGS+=-DUSE_OPENSSL $(OPENSSL_CFLAGS) + FINAL_LDFLAGS+=$(OPENSSL_LDFLAGS) + LIBSSL_PKGCONFIG := $(shell $(PKG_CONFIG) --exists libssl && echo $$?) +ifeq ($(LIBSSL_PKGCONFIG),0) + LIBSSL_LIBS=$(shell $(PKG_CONFIG) --libs libssl) +else + LIBSSL_LIBS=-lssl +endif + LIBCRYPTO_PKGCONFIG := $(shell $(PKG_CONFIG) --exists libcrypto && echo $$?) +ifeq ($(LIBCRYPTO_PKGCONFIG),0) + LIBCRYPTO_LIBS=$(shell $(PKG_CONFIG) --libs libcrypto) +else + LIBCRYPTO_LIBS=-lcrypto +endif + FINAL_LIBS += ../deps/hiredis/libhiredis_ssl.a $(LIBSSL_LIBS) $(LIBCRYPTO_LIBS) +endif + +ifndef V + define MAKE_INSTALL + @printf ' %b %b\n' $(LINKCOLOR)INSTALL$(ENDCOLOR) $(BINCOLOR)$(1)$(ENDCOLOR) 1>&2 + @$(INSTALL) $(1) $(2) + endef +else + define MAKE_INSTALL + $(INSTALL) $(1) $(2) + endef +endif + REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS) REDIS_INSTALL=$(QUIET_INSTALL)$(INSTALL) @@ -162,15 +286,15 @@ QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(EN QUIET_INSTALL = @printf ' %b %b\n' $(LINKCOLOR)INSTALL$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) 1>&2; endif -REDIS_SERVER_NAME=redis-server -REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o -REDIS_CLI_NAME=redis-cli -REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o -REDIS_BENCHMARK_NAME=redis-benchmark -REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o zmalloc.o redis-benchmark.o -REDIS_CHECK_RDB_NAME=redis-check-rdb -REDIS_CHECK_AOF_NAME=redis-check-aof +REDIS_SERVER_NAME=redis-server$(PROG_SUFFIX) +REDIS_SENTINEL_NAME=redis-sentinel$(PROG_SUFFIX) +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o +REDIS_CLI_NAME=redis-cli$(PROG_SUFFIX) +REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o +REDIS_BENCHMARK_NAME=redis-benchmark$(PROG_SUFFIX) +REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o release.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o +REDIS_CHECK_RDB_NAME=redis-check-rdb$(PROG_SUFFIX) +REDIS_CHECK_AOF_NAME=redis-check-aof$(PROG_SUFFIX) all: $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) @echo "" @@ -191,6 +315,8 @@ persist-settings: distclean echo WARN=$(WARN) >> .make-settings echo OPT=$(OPT) >> .make-settings echo MALLOC=$(MALLOC) >> .make-settings + echo BUILD_TLS=$(BUILD_TLS) >> .make-settings + echo USE_SYSTEMD=$(USE_SYSTEMD) >> .make-settings echo CFLAGS=$(CFLAGS) >> .make-settings echo LDFLAGS=$(LDFLAGS) >> .make-settings echo REDIS_CFLAGS=$(REDIS_CFLAGS) >> .make-settings @@ -236,32 +362,38 @@ $(REDIS_CLI_NAME): $(REDIS_CLI_OBJ) # redis-benchmark $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ) - $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a $(FINAL_LIBS) + $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/hdr_histogram/hdr_histogram.o $(FINAL_LIBS) -dict-benchmark: dict.c zmalloc.c sds.c siphash.c - $(REDIS_CC) $(FINAL_CFLAGS) $^ -D DICT_BENCHMARK_MAIN -o $@ $(FINAL_LIBS) +DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d) +-include $(DEP) # Because the jemalloc.h header is generated as a part of the jemalloc build, # building it should complete before building any other object. Instead of # depending on a single artifact, build all dependencies first. %.o: %.c .make-prerequisites - $(REDIS_CC) -c $< + $(REDIS_CC) -MMD -o $@ -c $< clean: - rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html Makefile.dep dict-benchmark + rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html Makefile.dep + rm -f $(DEP) .PHONY: clean distclean: clean -(cd ../deps && $(MAKE) distclean) + -(cd modules && $(MAKE) clean) + -(cd ../tests/modules && $(MAKE) clean) -(rm -f .make-*) .PHONY: distclean -test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) +test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) @(cd ..; ./runtest) -test-sentinel: $(REDIS_SENTINEL_NAME) +test-modules: $(REDIS_SERVER_NAME) + @(cd ..; ./runtest-moduleapi) + +test-sentinel: $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) @(cd ..; ./runtest-sentinel) check: test @@ -272,10 +404,6 @@ lcov: @geninfo -o redis.info . @genhtml --legend -o lcov-html redis.info -test-sds: sds.c sds.h - $(REDIS_CC) sds.c zmalloc.c -DSDS_TEST_MAIN $(FINAL_LIBS) -o /tmp/sds_test - /tmp/sds_test - .PHONY: lcov bench: $(REDIS_BENCHMARK_NAME) @@ -297,18 +425,18 @@ valgrind: $(MAKE) OPTIMIZATION="-O0" MALLOC="libc" helgrind: - $(MAKE) OPTIMIZATION="-O0" MALLOC="libc" CFLAGS="-D__ATOMIC_VAR_FORCE_SYNC_MACROS" + $(MAKE) OPTIMIZATION="-O0" MALLOC="libc" CFLAGS="-D__ATOMIC_VAR_FORCE_SYNC_MACROS" REDIS_CFLAGS="-I/usr/local/include" REDIS_LDFLAGS="-L/usr/local/lib" src/help.h: @../utils/generate-command-help.rb > help.h install: all @mkdir -p $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_BENCHMARK_NAME) $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_CLI_NAME) $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_CHECK_RDB_NAME) $(INSTALL_BIN) - $(REDIS_INSTALL) $(REDIS_CHECK_AOF_NAME) $(INSTALL_BIN) + $(call MAKE_INSTALL,$(REDIS_SERVER_NAME),$(INSTALL_BIN)) + $(call MAKE_INSTALL,$(REDIS_BENCHMARK_NAME),$(INSTALL_BIN)) + $(call MAKE_INSTALL,$(REDIS_CLI_NAME),$(INSTALL_BIN)) + @ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_CHECK_RDB_NAME) + @ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_CHECK_AOF_NAME) @ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_SENTINEL_NAME) uninstall: diff --git a/redis.submodule/src/acl.c b/redis.submodule/src/acl.c new file mode 100644 index 0000000..86f73fe --- /dev/null +++ b/redis.submodule/src/acl.c @@ -0,0 +1,2292 @@ +/* + * Copyright (c) 2018, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "server.h" +#include "sha256.h" +#include +#include + +/* ============================================================================= + * Global state for ACLs + * ==========================================================================*/ + +rax *Users; /* Table mapping usernames to user structures. */ + +user *DefaultUser; /* Global reference to the default user. + Every new connection is associated to it, if no + AUTH or HELLO is used to authenticate with a + different user. */ + +list *UsersToLoad; /* This is a list of users found in the configuration file + that we'll need to load in the final stage of Redis + initialization, after all the modules are already + loaded. Every list element is a NULL terminated + array of SDS pointers: the first is the user name, + all the remaining pointers are ACL rules in the same + format as ACLSetUser(). */ +list *ACLLog; /* Our security log, the user is able to inspect that + using the ACL LOG command .*/ + +static rax *commandId = NULL; /* Command name to id mapping */ + +static unsigned long nextid = 0; /* Next command id that has not been assigned */ + +struct ACLCategoryItem { + const char *name; + uint64_t flag; +} ACLCommandCategories[] = { + {"keyspace", CMD_CATEGORY_KEYSPACE}, + {"read", CMD_CATEGORY_READ}, + {"write", CMD_CATEGORY_WRITE}, + {"set", CMD_CATEGORY_SET}, + {"sortedset", CMD_CATEGORY_SORTEDSET}, + {"list", CMD_CATEGORY_LIST}, + {"hash", CMD_CATEGORY_HASH}, + {"string", CMD_CATEGORY_STRING}, + {"bitmap", CMD_CATEGORY_BITMAP}, + {"hyperloglog", CMD_CATEGORY_HYPERLOGLOG}, + {"geo", CMD_CATEGORY_GEO}, + {"stream", CMD_CATEGORY_STREAM}, + {"pubsub", CMD_CATEGORY_PUBSUB}, + {"admin", CMD_CATEGORY_ADMIN}, + {"fast", CMD_CATEGORY_FAST}, + {"slow", CMD_CATEGORY_SLOW}, + {"blocking", CMD_CATEGORY_BLOCKING}, + {"dangerous", CMD_CATEGORY_DANGEROUS}, + {"connection", CMD_CATEGORY_CONNECTION}, + {"transaction", CMD_CATEGORY_TRANSACTION}, + {"scripting", CMD_CATEGORY_SCRIPTING}, + {NULL,0} /* Terminator. */ +}; + +struct ACLUserFlag { + const char *name; + uint64_t flag; +} ACLUserFlags[] = { + /* Note: the order here dictates the emitted order at ACLDescribeUser */ + {"on", USER_FLAG_ENABLED}, + {"off", USER_FLAG_DISABLED}, + {"allkeys", USER_FLAG_ALLKEYS}, + {"allchannels", USER_FLAG_ALLCHANNELS}, + {"allcommands", USER_FLAG_ALLCOMMANDS}, + {"nopass", USER_FLAG_NOPASS}, + {"skip-sanitize-payload", USER_FLAG_SANITIZE_PAYLOAD_SKIP}, + {"sanitize-payload", USER_FLAG_SANITIZE_PAYLOAD}, + {NULL,0} /* Terminator. */ +}; + +void ACLResetSubcommandsForCommand(user *u, unsigned long id); +void ACLResetSubcommands(user *u); +void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub); +void ACLFreeLogEntry(void *le); + +/* The length of the string representation of a hashed password. */ +#define HASH_PASSWORD_LEN SHA256_BLOCK_SIZE*2 + +/* ============================================================================= + * Helper functions for the rest of the ACL implementation + * ==========================================================================*/ + +/* Return zero if strings are the same, non-zero if they are not. + * The comparison is performed in a way that prevents an attacker to obtain + * information about the nature of the strings just monitoring the execution + * time of the function. + * + * Note that limiting the comparison length to strings up to 512 bytes we + * can avoid leaking any information about the password length and any + * possible branch misprediction related leak. + */ +int time_independent_strcmp(char *a, char *b) { + char bufa[CONFIG_AUTHPASS_MAX_LEN], bufb[CONFIG_AUTHPASS_MAX_LEN]; + /* The above two strlen perform len(a) + len(b) operations where either + * a or b are fixed (our password) length, and the difference is only + * relative to the length of the user provided string, so no information + * leak is possible in the following two lines of code. */ + unsigned int alen = strlen(a); + unsigned int blen = strlen(b); + unsigned int j; + int diff = 0; + + /* We can't compare strings longer than our static buffers. + * Note that this will never pass the first test in practical circumstances + * so there is no info leak. */ + if (alen > sizeof(bufa) || blen > sizeof(bufb)) return 1; + + memset(bufa,0,sizeof(bufa)); /* Constant time. */ + memset(bufb,0,sizeof(bufb)); /* Constant time. */ + /* Again the time of the following two copies is proportional to + * len(a) + len(b) so no info is leaked. */ + memcpy(bufa,a,alen); + memcpy(bufb,b,blen); + + /* Always compare all the chars in the two buffers without + * conditional expressions. */ + for (j = 0; j < sizeof(bufa); j++) { + diff |= (bufa[j] ^ bufb[j]); + } + /* Length must be equal as well. */ + diff |= alen ^ blen; + return diff; /* If zero strings are the same. */ +} + +/* Given an SDS string, returns the SHA256 hex representation as a + * new SDS string. */ +sds ACLHashPassword(unsigned char *cleartext, size_t len) { + SHA256_CTX ctx; + unsigned char hash[SHA256_BLOCK_SIZE]; + char hex[HASH_PASSWORD_LEN]; + char *cset = "0123456789abcdef"; + + sha256_init(&ctx); + sha256_update(&ctx,(unsigned char*)cleartext,len); + sha256_final(&ctx,hash); + + for (int j = 0; j < SHA256_BLOCK_SIZE; j++) { + hex[j*2] = cset[((hash[j]&0xF0)>>4)]; + hex[j*2+1] = cset[(hash[j]&0xF)]; + } + return sdsnewlen(hex,HASH_PASSWORD_LEN); +} + +/* Given a hash and the hash length, returns C_OK if it is a valid password + * hash, or C_ERR otherwise. */ +int ACLCheckPasswordHash(unsigned char *hash, int hashlen) { + if (hashlen != HASH_PASSWORD_LEN) { + return C_ERR; + } + + /* Password hashes can only be characters that represent + * hexadecimal values, which are numbers and lowercase + * characters 'a' through 'f'. */ + for(int i = 0; i < HASH_PASSWORD_LEN; i++) { + char c = hash[i]; + if ((c < 'a' || c > 'f') && (c < '0' || c > '9')) { + return C_ERR; + } + } + return C_OK; +} + +/* ============================================================================= + * Low level ACL API + * ==========================================================================*/ + +/* Return 1 if the specified string contains spaces or null characters. + * We do this for usernames and key patterns for simpler rewriting of + * ACL rules, presentation on ACL list, and to avoid subtle security bugs + * that may arise from parsing the rules in presence of escapes. + * The function returns 0 if the string has no spaces. */ +int ACLStringHasSpaces(const char *s, size_t len) { + for (size_t i = 0; i < len; i++) { + if (isspace(s[i]) || s[i] == 0) return 1; + } + return 0; +} + +/* Given the category name the command returns the corresponding flag, or + * zero if there is no match. */ +uint64_t ACLGetCommandCategoryFlagByName(const char *name) { + for (int j = 0; ACLCommandCategories[j].flag != 0; j++) { + if (!strcasecmp(name,ACLCommandCategories[j].name)) { + return ACLCommandCategories[j].flag; + } + } + return 0; /* No match. */ +} + +/* Method for passwords/pattern comparison used for the user->passwords list + * so that we can search for items with listSearchKey(). */ +int ACLListMatchSds(void *a, void *b) { + return sdscmp(a,b) == 0; +} + +/* Method to free list elements from ACL users password/patterns lists. */ +void ACLListFreeSds(void *item) { + sdsfree(item); +} + +/* Method to duplicate list elements from ACL users password/patterns lists. */ +void *ACLListDupSds(void *item) { + return sdsdup(item); +} + +/* Create a new user with the specified name, store it in the list + * of users (the Users global radix tree), and returns a reference to + * the structure representing the user. + * + * If the user with such name already exists NULL is returned. */ +user *ACLCreateUser(const char *name, size_t namelen) { + if (raxFind(Users,(unsigned char*)name,namelen) != raxNotFound) return NULL; + user *u = zmalloc(sizeof(*u)); + u->name = sdsnewlen(name,namelen); + u->flags = USER_FLAG_DISABLED | server.acl_pubsub_default; + u->allowed_subcommands = NULL; + u->passwords = listCreate(); + u->patterns = listCreate(); + u->channels = listCreate(); + listSetMatchMethod(u->passwords,ACLListMatchSds); + listSetFreeMethod(u->passwords,ACLListFreeSds); + listSetDupMethod(u->passwords,ACLListDupSds); + listSetMatchMethod(u->patterns,ACLListMatchSds); + listSetFreeMethod(u->patterns,ACLListFreeSds); + listSetDupMethod(u->patterns,ACLListDupSds); + listSetMatchMethod(u->channels,ACLListMatchSds); + listSetFreeMethod(u->channels,ACLListFreeSds); + listSetDupMethod(u->channels,ACLListDupSds); + memset(u->allowed_commands,0,sizeof(u->allowed_commands)); + raxInsert(Users,(unsigned char*)name,namelen,u,NULL); + return u; +} + +/* This function should be called when we need an unlinked "fake" user + * we can use in order to validate ACL rules or for other similar reasons. + * The user will not get linked to the Users radix tree. The returned + * user should be released with ACLFreeUser() as usually. */ +user *ACLCreateUnlinkedUser(void) { + char username[64]; + for (int j = 0; ; j++) { + snprintf(username,sizeof(username),"__fakeuser:%d__",j); + user *fakeuser = ACLCreateUser(username,strlen(username)); + if (fakeuser == NULL) continue; + int retval = raxRemove(Users,(unsigned char*) username, + strlen(username),NULL); + serverAssert(retval != 0); + return fakeuser; + } +} + +/* Release the memory used by the user structure. Note that this function + * will not remove the user from the Users global radix tree. */ +void ACLFreeUser(user *u) { + sdsfree(u->name); + listRelease(u->passwords); + listRelease(u->patterns); + listRelease(u->channels); + ACLResetSubcommands(u); + zfree(u); +} + +/* When a user is deleted we need to cycle the active + * connections in order to kill all the pending ones that + * are authenticated with such user. */ +void ACLFreeUserAndKillClients(user *u) { + listIter li; + listNode *ln; + listRewind(server.clients,&li); + while ((ln = listNext(&li)) != NULL) { + client *c = listNodeValue(ln); + if (c->user == u) { + /* We'll free the connection asynchronously, so + * in theory to set a different user is not needed. + * However if there are bugs in Redis, soon or later + * this may result in some security hole: it's much + * more defensive to set the default user and put + * it in non authenticated mode. */ + c->user = DefaultUser; + c->authenticated = 0; + /* We will write replies to this client later, so we can't + * close it directly even if async. */ + if (c == server.current_client) { + c->flags |= CLIENT_CLOSE_AFTER_COMMAND; + } else { + freeClientAsync(c); + } + } + } + ACLFreeUser(u); +} + +/* Copy the user ACL rules from the source user 'src' to the destination + * user 'dst' so that at the end of the process they'll have exactly the + * same rules (but the names will continue to be the original ones). */ +void ACLCopyUser(user *dst, user *src) { + listRelease(dst->passwords); + listRelease(dst->patterns); + listRelease(dst->channels); + dst->passwords = listDup(src->passwords); + dst->patterns = listDup(src->patterns); + dst->channels = listDup(src->channels); + memcpy(dst->allowed_commands,src->allowed_commands, + sizeof(dst->allowed_commands)); + dst->flags = src->flags; + ACLResetSubcommands(dst); + /* Copy the allowed subcommands array of array of SDS strings. */ + if (src->allowed_subcommands) { + for (int j = 0; j < USER_COMMAND_BITS_COUNT; j++) { + if (src->allowed_subcommands[j]) { + for (int i = 0; src->allowed_subcommands[j][i]; i++) + { + ACLAddAllowedSubcommand(dst, j, + src->allowed_subcommands[j][i]); + } + } + } + } +} + +/* Free all the users registered in the radix tree 'users' and free the + * radix tree itself. */ +void ACLFreeUsersSet(rax *users) { + raxFreeWithCallback(users,(void(*)(void*))ACLFreeUserAndKillClients); +} + +/* Given a command ID, this function set by reference 'word' and 'bit' + * so that user->allowed_commands[word] will address the right word + * where the corresponding bit for the provided ID is stored, and + * so that user->allowed_commands[word]&bit will identify that specific + * bit. The function returns C_ERR in case the specified ID overflows + * the bitmap in the user representation. */ +int ACLGetCommandBitCoordinates(uint64_t id, uint64_t *word, uint64_t *bit) { + if (id >= USER_COMMAND_BITS_COUNT) return C_ERR; + *word = id / sizeof(uint64_t) / 8; + *bit = 1ULL << (id % (sizeof(uint64_t) * 8)); + return C_OK; +} + +/* Check if the specified command bit is set for the specified user. + * The function returns 1 is the bit is set or 0 if it is not. + * Note that this function does not check the ALLCOMMANDS flag of the user + * but just the lowlevel bitmask. + * + * If the bit overflows the user internal representation, zero is returned + * in order to disallow the execution of the command in such edge case. */ +int ACLGetUserCommandBit(user *u, unsigned long id) { + uint64_t word, bit; + if (ACLGetCommandBitCoordinates(id,&word,&bit) == C_ERR) return 0; + return (u->allowed_commands[word] & bit) != 0; +} + +/* When +@all or allcommands is given, we set a reserved bit as well that we + * can later test, to see if the user has the right to execute "future commands", + * that is, commands loaded later via modules. */ +int ACLUserCanExecuteFutureCommands(user *u) { + return ACLGetUserCommandBit(u,USER_COMMAND_BITS_COUNT-1); +} + +/* Set the specified command bit for the specified user to 'value' (0 or 1). + * If the bit overflows the user internal representation, no operation + * is performed. As a side effect of calling this function with a value of + * zero, the user flag ALLCOMMANDS is cleared since it is no longer possible + * to skip the command bit explicit test. */ +void ACLSetUserCommandBit(user *u, unsigned long id, int value) { + uint64_t word, bit; + if (ACLGetCommandBitCoordinates(id,&word,&bit) == C_ERR) return; + if (value) { + u->allowed_commands[word] |= bit; + } else { + u->allowed_commands[word] &= ~bit; + u->flags &= ~USER_FLAG_ALLCOMMANDS; + } +} + +/* This is like ACLSetUserCommandBit(), but instead of setting the specified + * ID, it will check all the commands in the category specified as argument, + * and will set all the bits corresponding to such commands to the specified + * value. Since the category passed by the user may be non existing, the + * function returns C_ERR if the category was not found, or C_OK if it was + * found and the operation was performed. */ +int ACLSetUserCommandBitsForCategory(user *u, const char *category, int value) { + uint64_t cflag = ACLGetCommandCategoryFlagByName(category); + if (!cflag) return C_ERR; + dictIterator *di = dictGetIterator(server.orig_commands); + dictEntry *de; + while ((de = dictNext(di)) != NULL) { + struct redisCommand *cmd = dictGetVal(de); + if (cmd->flags & CMD_MODULE) continue; /* Ignore modules commands. */ + if (cmd->flags & cflag) { + ACLSetUserCommandBit(u,cmd->id,value); + ACLResetSubcommandsForCommand(u,cmd->id); + } + } + dictReleaseIterator(di); + return C_OK; +} + +/* Return the number of commands allowed (on) and denied (off) for the user 'u' + * in the subset of commands flagged with the specified category name. + * If the category name is not valid, C_ERR is returned, otherwise C_OK is + * returned and on and off are populated by reference. */ +int ACLCountCategoryBitsForUser(user *u, unsigned long *on, unsigned long *off, + const char *category) +{ + uint64_t cflag = ACLGetCommandCategoryFlagByName(category); + if (!cflag) return C_ERR; + + *on = *off = 0; + dictIterator *di = dictGetIterator(server.orig_commands); + dictEntry *de; + while ((de = dictNext(di)) != NULL) { + struct redisCommand *cmd = dictGetVal(de); + if (cmd->flags & cflag) { + if (ACLGetUserCommandBit(u,cmd->id)) + (*on)++; + else + (*off)++; + } + } + dictReleaseIterator(di); + return C_OK; +} + +/* This function returns an SDS string representing the specified user ACL + * rules related to command execution, in the same format you could set them + * back using ACL SETUSER. The function will return just the set of rules needed + * to recreate the user commands bitmap, without including other user flags such + * as on/off, passwords and so forth. The returned string always starts with + * the +@all or -@all rule, depending on the user bitmap, and is followed, if + * needed, by the other rules needed to narrow or extend what the user can do. */ +sds ACLDescribeUserCommandRules(user *u) { + sds rules = sdsempty(); + int additive; /* If true we start from -@all and add, otherwise if + false we start from +@all and remove. */ + + /* This code is based on a trick: as we generate the rules, we apply + * them to a fake user, so that as we go we still know what are the + * bit differences we should try to address by emitting more rules. */ + user fu = {0}; + user *fakeuser = &fu; + + /* Here we want to understand if we should start with +@all and remove + * the commands corresponding to the bits that are not set in the user + * commands bitmap, or the contrary. Note that semantically the two are + * different. For instance starting with +@all and subtracting, the user + * will be able to execute future commands, while -@all and adding will just + * allow the user the run the selected commands and/or categories. + * How do we test for that? We use the trick of a reserved command ID bit + * that is set only by +@all (and its alias "allcommands"). */ + if (ACLUserCanExecuteFutureCommands(u)) { + additive = 0; + rules = sdscat(rules,"+@all "); + ACLSetUser(fakeuser,"+@all",-1); + } else { + additive = 1; + rules = sdscat(rules,"-@all "); + ACLSetUser(fakeuser,"-@all",-1); + } + + /* Attempt to find a good approximation for categories and commands + * based on the current bits used, by looping over the category list + * and applying the best fit each time. Often a set of categories will not + * perfectly match the set of commands into it, so at the end we do a + * final pass adding/removing the single commands needed to make the bitmap + * exactly match. A temp user is maintained to keep track of categories + * already applied. */ + user tu = {0}; + user *tempuser = &tu; + + /* Keep track of the categories that have been applied, to prevent + * applying them twice. */ + char applied[sizeof(ACLCommandCategories)/sizeof(ACLCommandCategories[0])]; + memset(applied, 0, sizeof(applied)); + + memcpy(tempuser->allowed_commands, + u->allowed_commands, + sizeof(u->allowed_commands)); + while (1) { + int best = -1; + unsigned long mindiff = INT_MAX, maxsame = 0; + for (int j = 0; ACLCommandCategories[j].flag != 0; j++) { + if (applied[j]) continue; + + unsigned long on, off, diff, same; + ACLCountCategoryBitsForUser(tempuser,&on,&off,ACLCommandCategories[j].name); + /* Check if the current category is the best this loop: + * * It has more commands in common with the user than commands + * that are different. + * AND EITHER + * * It has the fewest number of differences + * than the best match we have found so far. + * * OR it matches the fewest number of differences + * that we've seen but it has more in common. */ + diff = additive ? off : on; + same = additive ? on : off; + if (same > diff && + ((diff < mindiff) || (diff == mindiff && same > maxsame))) + { + best = j; + mindiff = diff; + maxsame = same; + } + } + + /* We didn't find a match */ + if (best == -1) break; + + sds op = sdsnewlen(additive ? "+@" : "-@", 2); + op = sdscat(op,ACLCommandCategories[best].name); + ACLSetUser(fakeuser,op,-1); + + sds invop = sdsnewlen(additive ? "-@" : "+@", 2); + invop = sdscat(invop,ACLCommandCategories[best].name); + ACLSetUser(tempuser,invop,-1); + + rules = sdscatsds(rules,op); + rules = sdscatlen(rules," ",1); + sdsfree(op); + sdsfree(invop); + + applied[best] = 1; + } + + /* Fix the final ACLs with single commands differences. */ + dictIterator *di = dictGetIterator(server.orig_commands); + dictEntry *de; + while ((de = dictNext(di)) != NULL) { + struct redisCommand *cmd = dictGetVal(de); + int userbit = ACLGetUserCommandBit(u,cmd->id); + int fakebit = ACLGetUserCommandBit(fakeuser,cmd->id); + if (userbit != fakebit) { + rules = sdscatlen(rules, userbit ? "+" : "-", 1); + rules = sdscat(rules,cmd->name); + rules = sdscatlen(rules," ",1); + ACLSetUserCommandBit(fakeuser,cmd->id,userbit); + } + + /* Emit the subcommands if there are any. */ + if (userbit == 0 && u->allowed_subcommands && + u->allowed_subcommands[cmd->id]) + { + for (int j = 0; u->allowed_subcommands[cmd->id][j]; j++) { + rules = sdscatlen(rules,"+",1); + rules = sdscat(rules,cmd->name); + rules = sdscatlen(rules,"|",1); + rules = sdscatsds(rules,u->allowed_subcommands[cmd->id][j]); + rules = sdscatlen(rules," ",1); + } + } + } + dictReleaseIterator(di); + + /* Trim the final useless space. */ + sdsrange(rules,0,-2); + + /* This is technically not needed, but we want to verify that now the + * predicted bitmap is exactly the same as the user bitmap, and abort + * otherwise, because aborting is better than a security risk in this + * code path. */ + if (memcmp(fakeuser->allowed_commands, + u->allowed_commands, + sizeof(u->allowed_commands)) != 0) + { + serverLog(LL_WARNING, + "CRITICAL ERROR: User ACLs don't match final bitmap: '%s'", + rules); + serverPanic("No bitmap match in ACLDescribeUserCommandRules()"); + } + return rules; +} + +/* This is similar to ACLDescribeUserCommandRules(), however instead of + * describing just the user command rules, everything is described: user + * flags, keys, passwords and finally the command rules obtained via + * the ACLDescribeUserCommandRules() function. This is the function we call + * when we want to rewrite the configuration files describing ACLs and + * in order to show users with ACL LIST. */ +sds ACLDescribeUser(user *u) { + sds res = sdsempty(); + + /* Flags. */ + for (int j = 0; ACLUserFlags[j].flag; j++) { + /* Skip the allcommands, allkeys and allchannels flags because they'll + * be emitted later as +@all, ~* and &*. */ + if (ACLUserFlags[j].flag == USER_FLAG_ALLKEYS || + ACLUserFlags[j].flag == USER_FLAG_ALLCHANNELS || + ACLUserFlags[j].flag == USER_FLAG_ALLCOMMANDS) continue; + if (u->flags & ACLUserFlags[j].flag) { + res = sdscat(res,ACLUserFlags[j].name); + res = sdscatlen(res," ",1); + } + } + + /* Passwords. */ + listIter li; + listNode *ln; + listRewind(u->passwords,&li); + while((ln = listNext(&li))) { + sds thispass = listNodeValue(ln); + res = sdscatlen(res,"#",1); + res = sdscatsds(res,thispass); + res = sdscatlen(res," ",1); + } + + /* Key patterns. */ + if (u->flags & USER_FLAG_ALLKEYS) { + res = sdscatlen(res,"~* ",3); + } else { + listRewind(u->patterns,&li); + while((ln = listNext(&li))) { + sds thispat = listNodeValue(ln); + res = sdscatlen(res,"~",1); + res = sdscatsds(res,thispat); + res = sdscatlen(res," ",1); + } + } + + /* Pub/sub channel patterns. */ + if (u->flags & USER_FLAG_ALLCHANNELS) { + res = sdscatlen(res,"&* ",3); + } else { + res = sdscatlen(res,"resetchannels ",14); + listRewind(u->channels,&li); + while((ln = listNext(&li))) { + sds thispat = listNodeValue(ln); + res = sdscatlen(res,"&",1); + res = sdscatsds(res,thispat); + res = sdscatlen(res," ",1); + } + } + + /* Command rules. */ + sds rules = ACLDescribeUserCommandRules(u); + res = sdscatsds(res,rules); + sdsfree(rules); + return res; +} + +/* Get a command from the original command table, that is not affected + * by the command renaming operations: we base all the ACL work from that + * table, so that ACLs are valid regardless of command renaming. */ +struct redisCommand *ACLLookupCommand(const char *name) { + struct redisCommand *cmd; + sds sdsname = sdsnew(name); + cmd = dictFetchValue(server.orig_commands, sdsname); + sdsfree(sdsname); + return cmd; +} + +/* Flush the array of allowed subcommands for the specified user + * and command ID. */ +void ACLResetSubcommandsForCommand(user *u, unsigned long id) { + if (u->allowed_subcommands && u->allowed_subcommands[id]) { + for (int i = 0; u->allowed_subcommands[id][i]; i++) + sdsfree(u->allowed_subcommands[id][i]); + zfree(u->allowed_subcommands[id]); + u->allowed_subcommands[id] = NULL; + } +} + +/* Flush the entire table of subcommands. This is useful on +@all, -@all + * or similar to return back to the minimal memory usage (and checks to do) + * for the user. */ +void ACLResetSubcommands(user *u) { + if (u->allowed_subcommands == NULL) return; + for (int j = 0; j < USER_COMMAND_BITS_COUNT; j++) { + if (u->allowed_subcommands[j]) { + for (int i = 0; u->allowed_subcommands[j][i]; i++) + sdsfree(u->allowed_subcommands[j][i]); + zfree(u->allowed_subcommands[j]); + } + } + zfree(u->allowed_subcommands); + u->allowed_subcommands = NULL; +} + +/* Add a subcommand to the list of subcommands for the user 'u' and + * the command id specified. */ +void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { + /* If this is the first subcommand to be configured for + * this user, we have to allocate the subcommands array. */ + if (u->allowed_subcommands == NULL) { + u->allowed_subcommands = zcalloc(USER_COMMAND_BITS_COUNT * + sizeof(sds*)); + } + + /* We also need to enlarge the allocation pointing to the + * null terminated SDS array, to make space for this one. + * To start check the current size, and while we are here + * make sure the subcommand is not already specified inside. */ + long items = 0; + if (u->allowed_subcommands[id]) { + while(u->allowed_subcommands[id][items]) { + /* If it's already here do not add it again. */ + if (!strcasecmp(u->allowed_subcommands[id][items],sub)) return; + items++; + } + } + + /* Now we can make space for the new item (and the null term). */ + items += 2; + u->allowed_subcommands[id] = zrealloc(u->allowed_subcommands[id], + sizeof(sds)*items); + u->allowed_subcommands[id][items-2] = sdsnew(sub); + u->allowed_subcommands[id][items-1] = NULL; +} + +/* Set user properties according to the string "op". The following + * is a description of what different strings will do: + * + * on Enable the user: it is possible to authenticate as this user. + * off Disable the user: it's no longer possible to authenticate + * with this user, however the already authenticated connections + * will still work. + * + Allow the execution of that command + * - Disallow the execution of that command + * +@ Allow the execution of all the commands in such category + * with valid categories are like @admin, @set, @sortedset, ... + * and so forth, see the full list in the server.c file where + * the Redis command table is described and defined. + * The special category @all means all the commands, but currently + * present in the server, and that will be loaded in the future + * via modules. + * +|subcommand Allow a specific subcommand of an otherwise + * disabled command. Note that this form is not + * allowed as negative like -DEBUG|SEGFAULT, but + * only additive starting with "+". + * allcommands Alias for +@all. Note that it implies the ability to execute + * all the future commands loaded via the modules system. + * nocommands Alias for -@all. + * ~ Add a pattern of keys that can be mentioned as part of + * commands. For instance ~* allows all the keys. The pattern + * is a glob-style pattern like the one of KEYS. + * It is possible to specify multiple patterns. + * allkeys Alias for ~* + * resetkeys Flush the list of allowed keys patterns. + * & Add a pattern of channels that can be mentioned as part of + * Pub/Sub commands. For instance &* allows all the channels. The + * pattern is a glob-style pattern like the one of PSUBSCRIBE. + * It is possible to specify multiple patterns. + * allchannels Alias for &* + * resetchannels Flush the list of allowed keys patterns. + * > Add this password to the list of valid password for the user. + * For example >mypass will add "mypass" to the list. + * This directive clears the "nopass" flag (see later). + * # Add this password hash to the list of valid hashes for + * the user. This is useful if you have previously computed + * the hash, and don't want to store it in plaintext. + * This directive clears the "nopass" flag (see later). + * < Remove this password from the list of valid passwords. + * ! Remove this hashed password from the list of valid passwords. + * This is useful when you want to remove a password just by + * hash without knowing its plaintext version at all. + * nopass All the set passwords of the user are removed, and the user + * is flagged as requiring no password: it means that every + * password will work against this user. If this directive is + * used for the default user, every new connection will be + * immediately authenticated with the default user without + * any explicit AUTH command required. Note that the "resetpass" + * directive will clear this condition. + * resetpass Flush the list of allowed passwords. Moreover removes the + * "nopass" status. After "resetpass" the user has no associated + * passwords and there is no way to authenticate without adding + * some password (or setting it as "nopass" later). + * reset Performs the following actions: resetpass, resetkeys, off, + * -@all. The user returns to the same state it has immediately + * after its creation. + * + * The 'op' string must be null terminated. The 'oplen' argument should + * specify the length of the 'op' string in case the caller requires to pass + * binary data (for instance the >password form may use a binary password). + * Otherwise the field can be set to -1 and the function will use strlen() + * to determine the length. + * + * The function returns C_OK if the action to perform was understood because + * the 'op' string made sense. Otherwise C_ERR is returned if the operation + * is unknown or has some syntax error. + * + * When an error is returned, errno is set to the following values: + * + * EINVAL: The specified opcode is not understood or the key/channel pattern is + * invalid (contains non allowed characters). + * ENOENT: The command name or command category provided with + or - is not + * known. + * EEXIST: You are adding a key pattern after "*" was already added. This is + * almost surely an error on the user side. + * EISDIR: You are adding a channel pattern after "*" was already added. This is + * almost surely an error on the user side. + * ENODEV: The password you are trying to remove from the user does not exist. + * EBADMSG: The hash you are trying to add is not a valid hash. + */ +int ACLSetUser(user *u, const char *op, ssize_t oplen) { + if (oplen == -1) oplen = strlen(op); + if (oplen == 0) return C_OK; /* Empty string is a no-operation. */ + if (!strcasecmp(op,"on")) { + u->flags |= USER_FLAG_ENABLED; + u->flags &= ~USER_FLAG_DISABLED; + } else if (!strcasecmp(op,"off")) { + u->flags |= USER_FLAG_DISABLED; + u->flags &= ~USER_FLAG_ENABLED; + } else if (!strcasecmp(op,"skip-sanitize-payload")) { + u->flags |= USER_FLAG_SANITIZE_PAYLOAD_SKIP; + u->flags &= ~USER_FLAG_SANITIZE_PAYLOAD; + } else if (!strcasecmp(op,"sanitize-payload")) { + u->flags &= ~USER_FLAG_SANITIZE_PAYLOAD_SKIP; + u->flags |= USER_FLAG_SANITIZE_PAYLOAD; + } else if (!strcasecmp(op,"allkeys") || + !strcasecmp(op,"~*")) + { + u->flags |= USER_FLAG_ALLKEYS; + listEmpty(u->patterns); + } else if (!strcasecmp(op,"resetkeys")) { + u->flags &= ~USER_FLAG_ALLKEYS; + listEmpty(u->patterns); + } else if (!strcasecmp(op,"allchannels") || + !strcasecmp(op,"&*")) + { + u->flags |= USER_FLAG_ALLCHANNELS; + listEmpty(u->channels); + } else if (!strcasecmp(op,"resetchannels")) { + u->flags &= ~USER_FLAG_ALLCHANNELS; + listEmpty(u->channels); + } else if (!strcasecmp(op,"allcommands") || + !strcasecmp(op,"+@all")) + { + memset(u->allowed_commands,255,sizeof(u->allowed_commands)); + u->flags |= USER_FLAG_ALLCOMMANDS; + ACLResetSubcommands(u); + } else if (!strcasecmp(op,"nocommands") || + !strcasecmp(op,"-@all")) + { + memset(u->allowed_commands,0,sizeof(u->allowed_commands)); + u->flags &= ~USER_FLAG_ALLCOMMANDS; + ACLResetSubcommands(u); + } else if (!strcasecmp(op,"nopass")) { + u->flags |= USER_FLAG_NOPASS; + listEmpty(u->passwords); + } else if (!strcasecmp(op,"resetpass")) { + u->flags &= ~USER_FLAG_NOPASS; + listEmpty(u->passwords); + } else if (op[0] == '>' || op[0] == '#') { + sds newpass; + if (op[0] == '>') { + newpass = ACLHashPassword((unsigned char*)op+1,oplen-1); + } else { + if (ACLCheckPasswordHash((unsigned char*)op+1,oplen-1) == C_ERR) { + errno = EBADMSG; + return C_ERR; + } + newpass = sdsnewlen(op+1,oplen-1); + } + + listNode *ln = listSearchKey(u->passwords,newpass); + /* Avoid re-adding the same password multiple times. */ + if (ln == NULL) + listAddNodeTail(u->passwords,newpass); + else + sdsfree(newpass); + u->flags &= ~USER_FLAG_NOPASS; + } else if (op[0] == '<' || op[0] == '!') { + sds delpass; + if (op[0] == '<') { + delpass = ACLHashPassword((unsigned char*)op+1,oplen-1); + } else { + if (ACLCheckPasswordHash((unsigned char*)op+1,oplen-1) == C_ERR) { + errno = EBADMSG; + return C_ERR; + } + delpass = sdsnewlen(op+1,oplen-1); + } + listNode *ln = listSearchKey(u->passwords,delpass); + sdsfree(delpass); + if (ln) { + listDelNode(u->passwords,ln); + } else { + errno = ENODEV; + return C_ERR; + } + } else if (op[0] == '~') { + if (u->flags & USER_FLAG_ALLKEYS) { + errno = EEXIST; + return C_ERR; + } + if (ACLStringHasSpaces(op+1,oplen-1)) { + errno = EINVAL; + return C_ERR; + } + sds newpat = sdsnewlen(op+1,oplen-1); + listNode *ln = listSearchKey(u->patterns,newpat); + /* Avoid re-adding the same key pattern multiple times. */ + if (ln == NULL) + listAddNodeTail(u->patterns,newpat); + else + sdsfree(newpat); + u->flags &= ~USER_FLAG_ALLKEYS; + } else if (op[0] == '&') { + if (u->flags & USER_FLAG_ALLCHANNELS) { + errno = EISDIR; + return C_ERR; + } + if (ACLStringHasSpaces(op+1,oplen-1)) { + errno = EINVAL; + return C_ERR; + } + sds newpat = sdsnewlen(op+1,oplen-1); + listNode *ln = listSearchKey(u->channels,newpat); + /* Avoid re-adding the same channel pattern multiple times. */ + if (ln == NULL) + listAddNodeTail(u->channels,newpat); + else + sdsfree(newpat); + u->flags &= ~USER_FLAG_ALLCHANNELS; + } else if (op[0] == '+' && op[1] != '@') { + if (strchr(op,'|') == NULL) { + if (ACLLookupCommand(op+1) == NULL) { + errno = ENOENT; + return C_ERR; + } + unsigned long id = ACLGetCommandID(op+1); + ACLSetUserCommandBit(u,id,1); + ACLResetSubcommandsForCommand(u,id); + } else { + /* Split the command and subcommand parts. */ + char *copy = zstrdup(op+1); + char *sub = strchr(copy,'|'); + sub[0] = '\0'; + sub++; + + /* Check if the command exists. We can't check the + * subcommand to see if it is valid. */ + if (ACLLookupCommand(copy) == NULL) { + zfree(copy); + errno = ENOENT; + return C_ERR; + } + + /* The subcommand cannot be empty, so things like DEBUG| + * are syntax errors of course. */ + if (strlen(sub) == 0) { + zfree(copy); + errno = EINVAL; + return C_ERR; + } + + unsigned long id = ACLGetCommandID(copy); + /* Add the subcommand to the list of valid ones, if the command is not set. */ + if (ACLGetUserCommandBit(u,id) == 0) { + ACLAddAllowedSubcommand(u,id,sub); + } + + zfree(copy); + } + } else if (op[0] == '-' && op[1] != '@') { + if (ACLLookupCommand(op+1) == NULL) { + errno = ENOENT; + return C_ERR; + } + unsigned long id = ACLGetCommandID(op+1); + ACLSetUserCommandBit(u,id,0); + ACLResetSubcommandsForCommand(u,id); + } else if ((op[0] == '+' || op[0] == '-') && op[1] == '@') { + int bitval = op[0] == '+' ? 1 : 0; + if (ACLSetUserCommandBitsForCategory(u,op+2,bitval) == C_ERR) { + errno = ENOENT; + return C_ERR; + } + } else if (!strcasecmp(op,"reset")) { + serverAssert(ACLSetUser(u,"resetpass",-1) == C_OK); + serverAssert(ACLSetUser(u,"resetkeys",-1) == C_OK); + serverAssert(ACLSetUser(u,"resetchannels",-1) == C_OK); + if (server.acl_pubsub_default & USER_FLAG_ALLCHANNELS) + serverAssert(ACLSetUser(u,"allchannels",-1) == C_OK); + serverAssert(ACLSetUser(u,"off",-1) == C_OK); + serverAssert(ACLSetUser(u,"sanitize-payload",-1) == C_OK); + serverAssert(ACLSetUser(u,"-@all",-1) == C_OK); + } else { + errno = EINVAL; + return C_ERR; + } + return C_OK; +} + +/* Return a description of the error that occurred in ACLSetUser() according to + * the errno value set by the function on error. */ +const char *ACLSetUserStringError(void) { + const char *errmsg = "Wrong format"; + if (errno == ENOENT) + errmsg = "Unknown command or category name in ACL"; + else if (errno == EINVAL) + errmsg = "Syntax error"; + else if (errno == EEXIST) + errmsg = "Adding a pattern after the * pattern (or the " + "'allkeys' flag) is not valid and does not have any " + "effect. Try 'resetkeys' to start with an empty " + "list of patterns"; + else if (errno == EISDIR) + errmsg = "Adding a pattern after the * pattern (or the " + "'allchannels' flag) is not valid and does not have any " + "effect. Try 'resetchannels' to start with an empty " + "list of channels"; + else if (errno == ENODEV) + errmsg = "The password you are trying to remove from the user does " + "not exist"; + else if (errno == EBADMSG) + errmsg = "The password hash must be exactly 64 characters and contain " + "only lowercase hexadecimal characters"; + return errmsg; +} + +/* Initialize the default user, that will always exist for all the process + * lifetime. */ +void ACLInitDefaultUser(void) { + DefaultUser = ACLCreateUser("default",7); + ACLSetUser(DefaultUser,"+@all",-1); + ACLSetUser(DefaultUser,"~*",-1); + ACLSetUser(DefaultUser,"&*",-1); + ACLSetUser(DefaultUser,"on",-1); + ACLSetUser(DefaultUser,"nopass",-1); +} + +/* Initialization of the ACL subsystem. */ +void ACLInit(void) { + Users = raxNew(); + UsersToLoad = listCreate(); + ACLLog = listCreate(); + ACLInitDefaultUser(); +} + +/* Check the username and password pair and return C_OK if they are valid, + * otherwise C_ERR is returned and errno is set to: + * + * EINVAL: if the username-password do not match. + * ENONENT: if the specified user does not exist at all. + */ +int ACLCheckUserCredentials(robj *username, robj *password) { + user *u = ACLGetUserByName(username->ptr,sdslen(username->ptr)); + if (u == NULL) { + errno = ENOENT; + return C_ERR; + } + + /* Disabled users can't login. */ + if (u->flags & USER_FLAG_DISABLED) { + errno = EINVAL; + return C_ERR; + } + + /* If the user is configured to don't require any password, we + * are already fine here. */ + if (u->flags & USER_FLAG_NOPASS) return C_OK; + + /* Check all the user passwords for at least one to match. */ + listIter li; + listNode *ln; + listRewind(u->passwords,&li); + sds hashed = ACLHashPassword(password->ptr,sdslen(password->ptr)); + while((ln = listNext(&li))) { + sds thispass = listNodeValue(ln); + if (!time_independent_strcmp(hashed, thispass)) { + sdsfree(hashed); + return C_OK; + } + } + sdsfree(hashed); + + /* If we reached this point, no password matched. */ + errno = EINVAL; + return C_ERR; +} + +/* This is like ACLCheckUserCredentials(), however if the user/pass + * are correct, the connection is put in authenticated state and the + * connection user reference is populated. + * + * The return value is C_OK or C_ERR with the same meaning as + * ACLCheckUserCredentials(). */ +int ACLAuthenticateUser(client *c, robj *username, robj *password) { + if (ACLCheckUserCredentials(username,password) == C_OK) { + c->authenticated = 1; + c->user = ACLGetUserByName(username->ptr,sdslen(username->ptr)); + moduleNotifyUserChanged(c); + return C_OK; + } else { + addACLLogEntry(c,ACL_DENIED_AUTH,0,username->ptr); + return C_ERR; + } +} + +/* For ACL purposes, every user has a bitmap with the commands that such + * user is allowed to execute. In order to populate the bitmap, every command + * should have an assigned ID (that is used to index the bitmap). This function + * creates such an ID: it uses sequential IDs, reusing the same ID for the same + * command name, so that a command retains the same ID in case of modules that + * are unloaded and later reloaded. */ +unsigned long ACLGetCommandID(const char *cmdname) { + + sds lowername = sdsnew(cmdname); + sdstolower(lowername); + if (commandId == NULL) commandId = raxNew(); + void *id = raxFind(commandId,(unsigned char*)lowername,sdslen(lowername)); + if (id != raxNotFound) { + sdsfree(lowername); + return (unsigned long)id; + } + raxInsert(commandId,(unsigned char*)lowername,strlen(lowername), + (void*)nextid,NULL); + sdsfree(lowername); + unsigned long thisid = nextid; + nextid++; + + /* We never assign the last bit in the user commands bitmap structure, + * this way we can later check if this bit is set, understanding if the + * current ACL for the user was created starting with a +@all to add all + * the possible commands and just subtracting other single commands or + * categories, or if, instead, the ACL was created just adding commands + * and command categories from scratch, not allowing future commands by + * default (loaded via modules). This is useful when rewriting the ACLs + * with ACL SAVE. */ + if (nextid == USER_COMMAND_BITS_COUNT-1) nextid++; + return thisid; +} + +/* Clear command id table and reset nextid to 0. */ +void ACLClearCommandID(void) { + if (commandId) raxFree(commandId); + commandId = NULL; + nextid = 0; +} + +/* Return an username by its name, or NULL if the user does not exist. */ +user *ACLGetUserByName(const char *name, size_t namelen) { + void *myuser = raxFind(Users,(unsigned char*)name,namelen); + if (myuser == raxNotFound) return NULL; + return myuser; +} + +/* Check if the command is ready to be executed in the client 'c', already + * referenced by c->cmd, and can be executed by this client according to the + * ACLs associated to the client user c->user. + * + * If the user can execute the command ACL_OK is returned, otherwise + * ACL_DENIED_CMD or ACL_DENIED_KEY is returned: the first in case the + * command cannot be executed because the user is not allowed to run such + * command, the second if the command is denied because the user is trying + * to access keys that are not among the specified patterns. */ +int ACLCheckCommandPerm(client *c, int *keyidxptr) { + user *u = c->user; + uint64_t id = c->cmd->id; + + /* If there is no associated user, the connection can run anything. */ + if (u == NULL) return ACL_OK; + + /* Check if the user can execute this command or if the command + * doesn't need to be authenticated (hello, auth). */ + if (!(u->flags & USER_FLAG_ALLCOMMANDS) && !(c->cmd->flags & CMD_NO_AUTH)) + { + /* If the bit is not set we have to check further, in case the + * command is allowed just with that specific subcommand. */ + if (ACLGetUserCommandBit(u,id) == 0) { + /* Check if the subcommand matches. */ + if (c->argc < 2 || + u->allowed_subcommands == NULL || + u->allowed_subcommands[id] == NULL) + { + return ACL_DENIED_CMD; + } + + long subid = 0; + while (1) { + if (u->allowed_subcommands[id][subid] == NULL) + return ACL_DENIED_CMD; + if (!strcasecmp(c->argv[1]->ptr, + u->allowed_subcommands[id][subid])) + break; /* Subcommand match found. Stop here. */ + subid++; + } + } + } + + /* Check if the user can execute commands explicitly touching the keys + * mentioned in the command arguments. */ + if (!(c->user->flags & USER_FLAG_ALLKEYS) && + (c->cmd->getkeys_proc || c->cmd->firstkey)) + { + getKeysResult result = GETKEYS_RESULT_INIT; + int numkeys = getKeysFromCommand(c->cmd,c->argv,c->argc,&result); + int *keyidx = result.keys; + for (int j = 0; j < numkeys; j++) { + listIter li; + listNode *ln; + listRewind(u->patterns,&li); + + /* Test this key against every pattern. */ + int match = 0; + while((ln = listNext(&li))) { + sds pattern = listNodeValue(ln); + size_t plen = sdslen(pattern); + int idx = keyidx[j]; + if (stringmatchlen(pattern,plen,c->argv[idx]->ptr, + sdslen(c->argv[idx]->ptr),0)) + { + match = 1; + break; + } + } + if (!match) { + if (keyidxptr) *keyidxptr = keyidx[j]; + getKeysFreeResult(&result); + return ACL_DENIED_KEY; + } + } + getKeysFreeResult(&result); + } + + /* If we survived all the above checks, the user can execute the + * command. */ + return ACL_OK; +} + +/* Check if the provided channel is whitelisted by the given allowed channels + * list. Glob-style pattern matching is employed, unless the literal flag is + * set. Returns ACL_OK if access is granted or ACL_DENIED_CHANNEL otherwise. */ +int ACLCheckPubsubChannelPerm(sds channel, list *allowed, int literal) { + listIter li; + listNode *ln; + size_t clen = sdslen(channel); + int match = 0; + + listRewind(allowed,&li); + while((ln = listNext(&li))) { + sds pattern = listNodeValue(ln); + size_t plen = sdslen(pattern); + + if ((literal && !sdscmp(pattern,channel)) || + (!literal && stringmatchlen(pattern,plen,channel,clen,0))) + { + match = 1; + break; + } + } + if (!match) { + return ACL_DENIED_CHANNEL; + } + return ACL_OK; +} + +/* Check if the user's existing pub/sub clients violate the ACL pub/sub + * permissions specified via the upcoming argument, and kill them if so. */ +void ACLKillPubsubClientsIfNeeded(user *u, list *upcoming) { + listIter li, lpi; + listNode *ln, *lpn; + robj *o; + int kill = 0; + + /* Nothing to kill when the upcoming are a literal super set of the original + * permissions. */ + listRewind(u->channels,&li); + while (!kill && ((ln = listNext(&li)) != NULL)) { + sds pattern = listNodeValue(ln); + kill = (ACLCheckPubsubChannelPerm(pattern,upcoming,1) == + ACL_DENIED_CHANNEL); + } + if (!kill) return; + + /* Scan all connected clients to find the user's pub/subs. */ + listRewind(server.clients,&li); + while ((ln = listNext(&li)) != NULL) { + client *c = listNodeValue(ln); + kill = 0; + + if (c->user == u && getClientType(c) == CLIENT_TYPE_PUBSUB) { + /* Check for pattern violations. */ + listRewind(c->pubsub_patterns,&lpi); + while (!kill && ((lpn = listNext(&lpi)) != NULL)) { + o = lpn->value; + kill = (ACLCheckPubsubChannelPerm(o->ptr,upcoming,1) == + ACL_DENIED_CHANNEL); + } + /* Check for channel violations. */ + if (!kill) { + dictIterator *di = dictGetIterator(c->pubsub_channels); + dictEntry *de; + while (!kill && ((de = dictNext(di)) != NULL)) { + o = dictGetKey(de); + kill = (ACLCheckPubsubChannelPerm(o->ptr,upcoming,0) == + ACL_DENIED_CHANNEL); + } + dictReleaseIterator(di); + } + + /* Kill it. */ + if (kill) { + freeClient(c); + } + } + } +} + +/* Check if the pub/sub channels of the command, that's ready to be executed in + * the client 'c', can be executed by this client according to the ACLs channels + * associated to the client user c->user. + * + * idx and count are the index and count of channel arguments from the + * command. The literal argument controls whether the user's ACL channels are + * evaluated as literal values or matched as glob-like patterns. + * + * If the user can execute the command ACL_OK is returned, otherwise + * ACL_DENIED_CHANNEL. */ +int ACLCheckPubsubPerm(client *c, int idx, int count, int literal, int *idxptr) { + user *u = c->user; + + /* If there is no associated user, the connection can run anything. */ + if (u == NULL) return ACL_OK; + + /* Check if the user can access the channels mentioned in the command's + * arguments. */ + if (!(c->user->flags & USER_FLAG_ALLCHANNELS)) { + for (int j = idx; j < idx+count; j++) { + if (ACLCheckPubsubChannelPerm(c->argv[j]->ptr,u->channels,literal) + != ACL_OK) { + if (idxptr) *idxptr = j; + return ACL_DENIED_CHANNEL; + } + } + } + + /* If we survived all the above checks, the user can execute the + * command. */ + return ACL_OK; + +} + +/* Check whether the command is ready to be exceuted by ACLCheckCommandPerm. + * If check passes, then check whether pub/sub channels of the command is + * ready to be executed by ACLCheckPubsubPerm */ +int ACLCheckAllPerm(client *c, int *idxptr) { + int acl_retval = ACLCheckCommandPerm(c,idxptr); + if (acl_retval != ACL_OK) + return acl_retval; + if (c->cmd->proc == publishCommand) + acl_retval = ACLCheckPubsubPerm(c,1,1,0,idxptr); + else if (c->cmd->proc == subscribeCommand) + acl_retval = ACLCheckPubsubPerm(c,1,c->argc-1,0,idxptr); + else if (c->cmd->proc == psubscribeCommand) + acl_retval = ACLCheckPubsubPerm(c,1,c->argc-1,1,idxptr); + return acl_retval; +} + +/* ============================================================================= + * ACL loading / saving functions + * ==========================================================================*/ + +/* Given an argument vector describing a user in the form: + * + * user ... ACL rules and flags ... + * + * this function validates, and if the syntax is valid, appends + * the user definition to a list for later loading. + * + * The rules are tested for validity and if there obvious syntax errors + * the function returns C_ERR and does nothing, otherwise C_OK is returned + * and the user is appended to the list. + * + * Note that this function cannot stop in case of commands that are not found + * and, in that case, the error will be emitted later, because certain + * commands may be defined later once modules are loaded. + * + * When an error is detected and C_ERR is returned, the function populates + * by reference (if not set to NULL) the argc_err argument with the index + * of the argv vector that caused the error. */ +int ACLAppendUserForLoading(sds *argv, int argc, int *argc_err) { + if (argc < 2 || strcasecmp(argv[0],"user")) { + if (argc_err) *argc_err = 0; + return C_ERR; + } + + /* Try to apply the user rules in a fake user to see if they + * are actually valid. */ + user *fakeuser = ACLCreateUnlinkedUser(); + + for (int j = 2; j < argc; j++) { + if (ACLSetUser(fakeuser,argv[j],sdslen(argv[j])) == C_ERR) { + if (errno != ENOENT) { + ACLFreeUser(fakeuser); + if (argc_err) *argc_err = j; + return C_ERR; + } + } + } + + /* Rules look valid, let's append the user to the list. */ + sds *copy = zmalloc(sizeof(sds)*argc); + for (int j = 1; j < argc; j++) copy[j-1] = sdsdup(argv[j]); + copy[argc-1] = NULL; + listAddNodeTail(UsersToLoad,copy); + ACLFreeUser(fakeuser); + return C_OK; +} + +/* This function will load the configured users appended to the server + * configuration via ACLAppendUserForLoading(). On loading errors it will + * log an error and return C_ERR, otherwise C_OK will be returned. */ +int ACLLoadConfiguredUsers(void) { + listIter li; + listNode *ln; + listRewind(UsersToLoad,&li); + while ((ln = listNext(&li)) != NULL) { + sds *aclrules = listNodeValue(ln); + sds username = aclrules[0]; + + if (ACLStringHasSpaces(aclrules[0],sdslen(aclrules[0]))) { + serverLog(LL_WARNING,"Spaces not allowed in ACL usernames"); + return C_ERR; + } + + user *u = ACLCreateUser(username,sdslen(username)); + if (!u) { + u = ACLGetUserByName(username,sdslen(username)); + serverAssert(u != NULL); + ACLSetUser(u,"reset",-1); + } + + /* Load every rule defined for this user. */ + for (int j = 1; aclrules[j]; j++) { + if (ACLSetUser(u,aclrules[j],sdslen(aclrules[j])) != C_OK) { + const char *errmsg = ACLSetUserStringError(); + serverLog(LL_WARNING,"Error loading ACL rule '%s' for " + "the user named '%s': %s", + aclrules[j],aclrules[0],errmsg); + return C_ERR; + } + } + + /* Having a disabled user in the configuration may be an error, + * warn about it without returning any error to the caller. */ + if (u->flags & USER_FLAG_DISABLED) { + serverLog(LL_NOTICE, "The user '%s' is disabled (there is no " + "'on' modifier in the user description). Make " + "sure this is not a configuration error.", + aclrules[0]); + } + } + return C_OK; +} + +/* This function loads the ACL from the specified filename: every line + * is validated and should be either empty or in the format used to specify + * users in the redis.conf configuration or in the ACL file, that is: + * + * user ... rules ... + * + * Note that this function considers comments starting with '#' as errors + * because the ACL file is meant to be rewritten, and comments would be + * lost after the rewrite. Yet empty lines are allowed to avoid being too + * strict. + * + * One important part of implementing ACL LOAD, that uses this function, is + * to avoid ending with broken rules if the ACL file is invalid for some + * reason, so the function will attempt to validate the rules before loading + * each user. For every line that will be found broken the function will + * collect an error message. + * + * IMPORTANT: If there is at least a single error, nothing will be loaded + * and the rules will remain exactly as they were. + * + * At the end of the process, if no errors were found in the whole file then + * NULL is returned. Otherwise an SDS string describing in a single line + * a description of all the issues found is returned. */ +sds ACLLoadFromFile(const char *filename) { + FILE *fp; + char buf[1024]; + + /* Open the ACL file. */ + if ((fp = fopen(filename,"r")) == NULL) { + sds errors = sdscatprintf(sdsempty(), + "Error loading ACLs, opening file '%s': %s", + filename, strerror(errno)); + return errors; + } + + /* Load the whole file as a single string in memory. */ + sds acls = sdsempty(); + while(fgets(buf,sizeof(buf),fp) != NULL) + acls = sdscat(acls,buf); + fclose(fp); + + /* Split the file into lines and attempt to load each line. */ + int totlines; + sds *lines, errors = sdsempty(); + lines = sdssplitlen(acls,strlen(acls),"\n",1,&totlines); + sdsfree(acls); + + /* We need a fake user to validate the rules before making changes + * to the real user mentioned in the ACL line. */ + user *fakeuser = ACLCreateUnlinkedUser(); + + /* We do all the loading in a fresh instance of the Users radix tree, + * so if there are errors loading the ACL file we can rollback to the + * old version. */ + rax *old_users = Users; + user *old_default_user = DefaultUser; + Users = raxNew(); + ACLInitDefaultUser(); + + /* Load each line of the file. */ + for (int i = 0; i < totlines; i++) { + sds *argv; + int argc; + int linenum = i+1; + + lines[i] = sdstrim(lines[i]," \t\r\n"); + + /* Skip blank lines */ + if (lines[i][0] == '\0') continue; + + /* Split into arguments */ + argv = sdssplitlen(lines[i],sdslen(lines[i])," ",1,&argc); + if (argv == NULL) { + errors = sdscatprintf(errors, + "%s:%d: unbalanced quotes in acl line. ", + server.acl_filename, linenum); + continue; + } + + /* Skip this line if the resulting command vector is empty. */ + if (argc == 0) { + sdsfreesplitres(argv,argc); + continue; + } + + /* The line should start with the "user" keyword. */ + if (strcmp(argv[0],"user") || argc < 2) { + errors = sdscatprintf(errors, + "%s:%d should start with user keyword followed " + "by the username. ", server.acl_filename, + linenum); + sdsfreesplitres(argv,argc); + continue; + } + + /* Spaces are not allowed in usernames. */ + if (ACLStringHasSpaces(argv[1],sdslen(argv[1]))) { + errors = sdscatprintf(errors, + "'%s:%d: username '%s' contains invalid characters. ", + server.acl_filename, linenum, argv[1]); + sdsfreesplitres(argv,argc); + continue; + } + + /* Try to process the line using the fake user to validate if + * the rules are able to apply cleanly. At this stage we also + * trim trailing spaces, so that we don't have to handle that + * in ACLSetUser(). */ + ACLSetUser(fakeuser,"reset",-1); + int j; + for (j = 2; j < argc; j++) { + argv[j] = sdstrim(argv[j],"\t\r\n"); + if (ACLSetUser(fakeuser,argv[j],sdslen(argv[j])) != C_OK) { + const char *errmsg = ACLSetUserStringError(); + errors = sdscatprintf(errors, + "%s:%d: %s. ", + server.acl_filename, linenum, errmsg); + continue; + } + } + + /* Apply the rule to the new users set only if so far there + * are no errors, otherwise it's useless since we are going + * to discard the new users set anyway. */ + if (sdslen(errors) != 0) { + sdsfreesplitres(argv,argc); + continue; + } + + /* We can finally lookup the user and apply the rule. If the + * user already exists we always reset it to start. */ + user *u = ACLCreateUser(argv[1],sdslen(argv[1])); + if (!u) { + u = ACLGetUserByName(argv[1],sdslen(argv[1])); + serverAssert(u != NULL); + ACLSetUser(u,"reset",-1); + } + + /* Note that the same rules already applied to the fake user, so + * we just assert that everything goes well: it should. */ + for (j = 2; j < argc; j++) + serverAssert(ACLSetUser(u,argv[j],sdslen(argv[j])) == C_OK); + + sdsfreesplitres(argv,argc); + } + + ACLFreeUser(fakeuser); + sdsfreesplitres(lines,totlines); + DefaultUser = old_default_user; /* This pointer must never change. */ + + /* Check if we found errors and react accordingly. */ + if (sdslen(errors) == 0) { + /* The default user pointer is referenced in different places: instead + * of replacing such occurrences it is much simpler to copy the new + * default user configuration in the old one. */ + user *new = ACLGetUserByName("default",7); + serverAssert(new != NULL); + ACLCopyUser(DefaultUser,new); + ACLFreeUser(new); + raxInsert(Users,(unsigned char*)"default",7,DefaultUser,NULL); + raxRemove(old_users,(unsigned char*)"default",7,NULL); + ACLFreeUsersSet(old_users); + sdsfree(errors); + return NULL; + } else { + ACLFreeUsersSet(Users); + Users = old_users; + errors = sdscat(errors,"WARNING: ACL errors detected, no change to the previously active ACL rules was performed"); + return errors; + } +} + +/* Generate a copy of the ACLs currently in memory in the specified filename. + * Returns C_OK on success or C_ERR if there was an error during the I/O. + * When C_ERR is returned a log is produced with hints about the issue. */ +int ACLSaveToFile(const char *filename) { + sds acl = sdsempty(); + int fd = -1; + sds tmpfilename = NULL; + int retval = C_ERR; + + /* Let's generate an SDS string containing the new version of the + * ACL file. */ + raxIterator ri; + raxStart(&ri,Users); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + user *u = ri.data; + /* Return information in the configuration file format. */ + sds user = sdsnew("user "); + user = sdscatsds(user,u->name); + user = sdscatlen(user," ",1); + sds descr = ACLDescribeUser(u); + user = sdscatsds(user,descr); + sdsfree(descr); + acl = sdscatsds(acl,user); + acl = sdscatlen(acl,"\n",1); + sdsfree(user); + } + raxStop(&ri); + + /* Create a temp file with the new content. */ + tmpfilename = sdsnew(filename); + tmpfilename = sdscatfmt(tmpfilename,".tmp-%i-%I", + (int)getpid(),(int)mstime()); + if ((fd = open(tmpfilename,O_WRONLY|O_CREAT,0644)) == -1) { + serverLog(LL_WARNING,"Opening temp ACL file for ACL SAVE: %s", + strerror(errno)); + goto cleanup; + } + + /* Write it. */ + if (write(fd,acl,sdslen(acl)) != (ssize_t)sdslen(acl)) { + serverLog(LL_WARNING,"Writing ACL file for ACL SAVE: %s", + strerror(errno)); + goto cleanup; + } + close(fd); fd = -1; + + /* Let's replace the new file with the old one. */ + if (rename(tmpfilename,filename) == -1) { + serverLog(LL_WARNING,"Renaming ACL file for ACL SAVE: %s", + strerror(errno)); + goto cleanup; + } + sdsfree(tmpfilename); tmpfilename = NULL; + retval = C_OK; /* If we reached this point, everything is fine. */ + +cleanup: + if (fd != -1) close(fd); + if (tmpfilename) unlink(tmpfilename); + sdsfree(tmpfilename); + sdsfree(acl); + return retval; +} + +/* This function is called once the server is already running, modules are + * loaded, and we are ready to start, in order to load the ACLs either from + * the pending list of users defined in redis.conf, or from the ACL file. + * The function will just exit with an error if the user is trying to mix + * both the loading methods. */ +void ACLLoadUsersAtStartup(void) { + if (server.acl_filename[0] != '\0' && listLength(UsersToLoad) != 0) { + serverLog(LL_WARNING, + "Configuring Redis with users defined in redis.conf and at " + "the same setting an ACL file path is invalid. This setup " + "is very likely to lead to configuration errors and security " + "holes, please define either an ACL file or declare users " + "directly in your redis.conf, but not both."); + exit(1); + } + + if (ACLLoadConfiguredUsers() == C_ERR) { + serverLog(LL_WARNING, + "Critical error while loading ACLs. Exiting."); + exit(1); + } + + if (server.acl_filename[0] != '\0') { + sds errors = ACLLoadFromFile(server.acl_filename); + if (errors) { + serverLog(LL_WARNING, + "Aborting Redis startup because of ACL errors: %s", errors); + sdsfree(errors); + exit(1); + } + } +} + +/* ============================================================================= + * ACL log + * ==========================================================================*/ + +#define ACL_LOG_CTX_TOPLEVEL 0 +#define ACL_LOG_CTX_LUA 1 +#define ACL_LOG_CTX_MULTI 2 +#define ACL_LOG_GROUPING_MAX_TIME_DELTA 60000 + +/* This structure defines an entry inside the ACL log. */ +typedef struct ACLLogEntry { + uint64_t count; /* Number of times this happened recently. */ + int reason; /* Reason for denying the command. ACL_DENIED_*. */ + int context; /* Toplevel, Lua or MULTI/EXEC? ACL_LOG_CTX_*. */ + sds object; /* The key name or command name. */ + sds username; /* User the client is authenticated with. */ + mstime_t ctime; /* Milliseconds time of last update to this entry. */ + sds cinfo; /* Client info (last client if updated). */ +} ACLLogEntry; + +/* This function will check if ACL entries 'a' and 'b' are similar enough + * that we should actually update the existing entry in our ACL log instead + * of creating a new one. */ +int ACLLogMatchEntry(ACLLogEntry *a, ACLLogEntry *b) { + if (a->reason != b->reason) return 0; + if (a->context != b->context) return 0; + mstime_t delta = a->ctime - b->ctime; + if (delta < 0) delta = -delta; + if (delta > ACL_LOG_GROUPING_MAX_TIME_DELTA) return 0; + if (sdscmp(a->object,b->object) != 0) return 0; + if (sdscmp(a->username,b->username) != 0) return 0; + return 1; +} + +/* Release an ACL log entry. */ +void ACLFreeLogEntry(void *leptr) { + ACLLogEntry *le = leptr; + sdsfree(le->object); + sdsfree(le->username); + sdsfree(le->cinfo); + zfree(le); +} + +/* Adds a new entry in the ACL log, making sure to delete the old entry + * if we reach the maximum length allowed for the log. This function attempts + * to find similar entries in the current log in order to bump the counter of + * the log entry instead of creating many entries for very similar ACL + * rules issues. + * + * The argpos argument is used when the reason is ACL_DENIED_KEY or + * ACL_DENIED_CHANNEL, since it allows the function to log the key or channel + * name that caused the problem. Similarly the username is only passed when we + * failed to authenticate the user with AUTH or HELLO, for the ACL_DENIED_AUTH + * reason. Otherwise it will just be NULL. + */ +void addACLLogEntry(client *c, int reason, int argpos, sds username) { + /* Create a new entry. */ + struct ACLLogEntry *le = zmalloc(sizeof(*le)); + le->count = 1; + le->reason = reason; + le->username = sdsdup(reason == ACL_DENIED_AUTH ? username : c->user->name); + le->ctime = mstime(); + + switch(reason) { + case ACL_DENIED_CMD: le->object = sdsnew(c->cmd->name); break; + case ACL_DENIED_KEY: le->object = sdsdup(c->argv[argpos]->ptr); break; + case ACL_DENIED_CHANNEL: le->object = sdsdup(c->argv[argpos]->ptr); break; + case ACL_DENIED_AUTH: le->object = sdsdup(c->argv[0]->ptr); break; + default: le->object = sdsempty(); + } + + client *realclient = c; + if (realclient->flags & CLIENT_LUA) realclient = server.lua_caller; + + le->cinfo = catClientInfoString(sdsempty(),realclient); + if (c->flags & CLIENT_MULTI) { + le->context = ACL_LOG_CTX_MULTI; + } else if (c->flags & CLIENT_LUA) { + le->context = ACL_LOG_CTX_LUA; + } else { + le->context = ACL_LOG_CTX_TOPLEVEL; + } + + /* Try to match this entry with past ones, to see if we can just + * update an existing entry instead of creating a new one. */ + long toscan = 10; /* Do a limited work trying to find duplicated. */ + listIter li; + listNode *ln; + listRewind(ACLLog,&li); + ACLLogEntry *match = NULL; + while (toscan-- && (ln = listNext(&li)) != NULL) { + ACLLogEntry *current = listNodeValue(ln); + if (ACLLogMatchEntry(current,le)) { + match = current; + listDelNode(ACLLog,ln); + listAddNodeHead(ACLLog,current); + break; + } + } + + /* If there is a match update the entry, otherwise add it as a + * new one. */ + if (match) { + /* We update a few fields of the existing entry and bump the + * counter of events for this entry. */ + sdsfree(match->cinfo); + match->cinfo = le->cinfo; + match->ctime = le->ctime; + match->count++; + + /* Release the old entry. */ + le->cinfo = NULL; + ACLFreeLogEntry(le); + } else { + /* Add it to our list of entries. We'll have to trim the list + * to its maximum size. */ + listAddNodeHead(ACLLog, le); + while(listLength(ACLLog) > server.acllog_max_len) { + listNode *ln = listLast(ACLLog); + ACLLogEntry *le = listNodeValue(ln); + ACLFreeLogEntry(le); + listDelNode(ACLLog,ln); + } + } +} + +/* ============================================================================= + * ACL related commands + * ==========================================================================*/ + +/* ACL -- show and modify the configuration of ACL users. + * ACL HELP + * ACL LOAD + * ACL SAVE + * ACL LIST + * ACL USERS + * ACL CAT [] + * ACL SETUSER ... acl rules ... + * ACL DELUSER [...] + * ACL GETUSER + * ACL GENPASS [] + * ACL WHOAMI + * ACL LOG [ | RESET] + */ +void aclCommand(client *c) { + char *sub = c->argv[1]->ptr; + if (!strcasecmp(sub,"setuser") && c->argc >= 3) { + sds username = c->argv[2]->ptr; + /* Check username validity. */ + if (ACLStringHasSpaces(username,sdslen(username))) { + addReplyErrorFormat(c, + "Usernames can't contain spaces or null characters"); + return; + } + + /* Create a temporary user to validate and stage all changes against + * before applying to an existing user or creating a new user. If all + * arguments are valid the user parameters will all be applied together. + * If there are any errors then none of the changes will be applied. */ + user *tempu = ACLCreateUnlinkedUser(); + user *u = ACLGetUserByName(username,sdslen(username)); + if (u) ACLCopyUser(tempu, u); + + /* Initially redact all of the arguments to not leak any information + * about the user. */ + for (int j = 2; j < c->argc; j++) { + redactClientCommandArgument(c, j); + } + + for (int j = 3; j < c->argc; j++) { + if (ACLSetUser(tempu,c->argv[j]->ptr,sdslen(c->argv[j]->ptr)) != C_OK) { + const char *errmsg = ACLSetUserStringError(); + addReplyErrorFormat(c, + "Error in ACL SETUSER modifier '%s': %s", + (char*)c->argv[j]->ptr, errmsg); + + ACLFreeUser(tempu); + return; + } + } + + /* Existing pub/sub clients authenticated with the user may need to be + * disconnected if (some of) their channel permissions were revoked. */ + if (u && !(tempu->flags & USER_FLAG_ALLCHANNELS)) + ACLKillPubsubClientsIfNeeded(u,tempu->channels); + + /* Overwrite the user with the temporary user we modified above. */ + if (!u) u = ACLCreateUser(username,sdslen(username)); + serverAssert(u != NULL); + ACLCopyUser(u, tempu); + ACLFreeUser(tempu); + addReply(c,shared.ok); + } else if (!strcasecmp(sub,"deluser") && c->argc >= 3) { + int deleted = 0; + for (int j = 2; j < c->argc; j++) { + sds username = c->argv[j]->ptr; + if (!strcmp(username,"default")) { + addReplyError(c,"The 'default' user cannot be removed"); + return; + } + } + + for (int j = 2; j < c->argc; j++) { + sds username = c->argv[j]->ptr; + user *u; + if (raxRemove(Users,(unsigned char*)username, + sdslen(username), + (void**)&u)) + { + ACLFreeUserAndKillClients(u); + deleted++; + } + } + addReplyLongLong(c,deleted); + } else if (!strcasecmp(sub,"getuser") && c->argc == 3) { + user *u = ACLGetUserByName(c->argv[2]->ptr,sdslen(c->argv[2]->ptr)); + if (u == NULL) { + addReplyNull(c); + return; + } + + addReplyMapLen(c,5); + + /* Flags */ + addReplyBulkCString(c,"flags"); + void *deflen = addReplyDeferredLen(c); + int numflags = 0; + for (int j = 0; ACLUserFlags[j].flag; j++) { + if (u->flags & ACLUserFlags[j].flag) { + addReplyBulkCString(c,ACLUserFlags[j].name); + numflags++; + } + } + setDeferredSetLen(c,deflen,numflags); + + /* Passwords */ + addReplyBulkCString(c,"passwords"); + addReplyArrayLen(c,listLength(u->passwords)); + listIter li; + listNode *ln; + listRewind(u->passwords,&li); + while((ln = listNext(&li))) { + sds thispass = listNodeValue(ln); + addReplyBulkCBuffer(c,thispass,sdslen(thispass)); + } + + /* Commands */ + addReplyBulkCString(c,"commands"); + sds cmddescr = ACLDescribeUserCommandRules(u); + addReplyBulkSds(c,cmddescr); + + /* Key patterns */ + addReplyBulkCString(c,"keys"); + if (u->flags & USER_FLAG_ALLKEYS) { + addReplyArrayLen(c,1); + addReplyBulkCBuffer(c,"*",1); + } else { + addReplyArrayLen(c,listLength(u->patterns)); + listIter li; + listNode *ln; + listRewind(u->patterns,&li); + while((ln = listNext(&li))) { + sds thispat = listNodeValue(ln); + addReplyBulkCBuffer(c,thispat,sdslen(thispat)); + } + } + + /* Pub/sub patterns */ + addReplyBulkCString(c,"channels"); + if (u->flags & USER_FLAG_ALLCHANNELS) { + addReplyArrayLen(c,1); + addReplyBulkCBuffer(c,"*",1); + } else { + addReplyArrayLen(c,listLength(u->channels)); + listIter li; + listNode *ln; + listRewind(u->channels,&li); + while((ln = listNext(&li))) { + sds thispat = listNodeValue(ln); + addReplyBulkCBuffer(c,thispat,sdslen(thispat)); + } + } + } else if ((!strcasecmp(sub,"list") || !strcasecmp(sub,"users")) && + c->argc == 2) + { + int justnames = !strcasecmp(sub,"users"); + addReplyArrayLen(c,raxSize(Users)); + raxIterator ri; + raxStart(&ri,Users); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + user *u = ri.data; + if (justnames) { + addReplyBulkCBuffer(c,u->name,sdslen(u->name)); + } else { + /* Return information in the configuration file format. */ + sds config = sdsnew("user "); + config = sdscatsds(config,u->name); + config = sdscatlen(config," ",1); + sds descr = ACLDescribeUser(u); + config = sdscatsds(config,descr); + sdsfree(descr); + addReplyBulkSds(c,config); + } + } + raxStop(&ri); + } else if (!strcasecmp(sub,"whoami") && c->argc == 2) { + if (c->user != NULL) { + addReplyBulkCBuffer(c,c->user->name,sdslen(c->user->name)); + } else { + addReplyNull(c); + } + } else if (server.acl_filename[0] == '\0' && + (!strcasecmp(sub,"load") || !strcasecmp(sub,"save"))) + { + addReplyError(c,"This Redis instance is not configured to use an ACL file. You may want to specify users via the ACL SETUSER command and then issue a CONFIG REWRITE (assuming you have a Redis configuration file set) in order to store users in the Redis configuration."); + return; + } else if (!strcasecmp(sub,"load") && c->argc == 2) { + sds errors = ACLLoadFromFile(server.acl_filename); + if (errors == NULL) { + addReply(c,shared.ok); + } else { + addReplyError(c,errors); + sdsfree(errors); + } + } else if (!strcasecmp(sub,"save") && c->argc == 2) { + if (ACLSaveToFile(server.acl_filename) == C_OK) { + addReply(c,shared.ok); + } else { + addReplyError(c,"There was an error trying to save the ACLs. " + "Please check the server logs for more " + "information"); + } + } else if (!strcasecmp(sub,"cat") && c->argc == 2) { + void *dl = addReplyDeferredLen(c); + int j; + for (j = 0; ACLCommandCategories[j].flag != 0; j++) + addReplyBulkCString(c,ACLCommandCategories[j].name); + setDeferredArrayLen(c,dl,j); + } else if (!strcasecmp(sub,"cat") && c->argc == 3) { + uint64_t cflag = ACLGetCommandCategoryFlagByName(c->argv[2]->ptr); + if (cflag == 0) { + addReplyErrorFormat(c, "Unknown category '%s'", (char*)c->argv[2]->ptr); + return; + } + int arraylen = 0; + void *dl = addReplyDeferredLen(c); + dictIterator *di = dictGetIterator(server.orig_commands); + dictEntry *de; + while ((de = dictNext(di)) != NULL) { + struct redisCommand *cmd = dictGetVal(de); + if (cmd->flags & CMD_MODULE) continue; + if (cmd->flags & cflag) { + addReplyBulkCString(c,cmd->name); + arraylen++; + } + } + dictReleaseIterator(di); + setDeferredArrayLen(c,dl,arraylen); + } else if (!strcasecmp(sub,"genpass") && (c->argc == 2 || c->argc == 3)) { + #define GENPASS_MAX_BITS 4096 + char pass[GENPASS_MAX_BITS/8*2]; /* Hex representation. */ + long bits = 256; /* By default generate 256 bits passwords. */ + + if (c->argc == 3 && getLongFromObjectOrReply(c,c->argv[2],&bits,NULL) + != C_OK) return; + + if (bits <= 0 || bits > GENPASS_MAX_BITS) { + addReplyErrorFormat(c, + "ACL GENPASS argument must be the number of " + "bits for the output password, a positive number " + "up to %d",GENPASS_MAX_BITS); + return; + } + + long chars = (bits+3)/4; /* Round to number of characters to emit. */ + getRandomHexChars(pass,chars); + addReplyBulkCBuffer(c,pass,chars); + } else if (!strcasecmp(sub,"log") && (c->argc == 2 || c->argc ==3)) { + long count = 10; /* Number of entries to emit by default. */ + + /* Parse the only argument that LOG may have: it could be either + * the number of entries the user wants to display, or alternatively + * the "RESET" command in order to flush the old entries. */ + if (c->argc == 3) { + if (!strcasecmp(c->argv[2]->ptr,"reset")) { + listSetFreeMethod(ACLLog,ACLFreeLogEntry); + listEmpty(ACLLog); + listSetFreeMethod(ACLLog,NULL); + addReply(c,shared.ok); + return; + } else if (getLongFromObjectOrReply(c,c->argv[2],&count,NULL) + != C_OK) + { + return; + } + if (count < 0) count = 0; + } + + /* Fix the count according to the number of entries we got. */ + if ((size_t)count > listLength(ACLLog)) + count = listLength(ACLLog); + + addReplyArrayLen(c,count); + listIter li; + listNode *ln; + listRewind(ACLLog,&li); + mstime_t now = mstime(); + while (count-- && (ln = listNext(&li)) != NULL) { + ACLLogEntry *le = listNodeValue(ln); + addReplyMapLen(c,7); + addReplyBulkCString(c,"count"); + addReplyLongLong(c,le->count); + + addReplyBulkCString(c,"reason"); + char *reasonstr; + switch(le->reason) { + case ACL_DENIED_CMD: reasonstr="command"; break; + case ACL_DENIED_KEY: reasonstr="key"; break; + case ACL_DENIED_CHANNEL: reasonstr="channel"; break; + case ACL_DENIED_AUTH: reasonstr="auth"; break; + default: reasonstr="unknown"; + } + addReplyBulkCString(c,reasonstr); + + addReplyBulkCString(c,"context"); + char *ctxstr; + switch(le->context) { + case ACL_LOG_CTX_TOPLEVEL: ctxstr="toplevel"; break; + case ACL_LOG_CTX_MULTI: ctxstr="multi"; break; + case ACL_LOG_CTX_LUA: ctxstr="lua"; break; + default: ctxstr="unknown"; + } + addReplyBulkCString(c,ctxstr); + + addReplyBulkCString(c,"object"); + addReplyBulkCBuffer(c,le->object,sdslen(le->object)); + addReplyBulkCString(c,"username"); + addReplyBulkCBuffer(c,le->username,sdslen(le->username)); + addReplyBulkCString(c,"age-seconds"); + double age = (double)(now - le->ctime)/1000; + addReplyDouble(c,age); + addReplyBulkCString(c,"client-info"); + addReplyBulkCBuffer(c,le->cinfo,sdslen(le->cinfo)); + } + } else if (c->argc == 2 && !strcasecmp(sub,"help")) { + const char *help[] = { +"CAT []", +" List all commands that belong to , or all command categories", +" when no category is specified.", +"DELUSER [ ...]", +" Delete a list of users.", +"GETUSER ", +" Get the user's details.", +"GENPASS []", +" Generate a secure 256-bit user password. The optional `bits` argument can", +" be used to specify a different size.", +"LIST", +" Show users details in config file format.", +"LOAD", +" Reload users from the ACL file.", +"LOG [ | RESET]", +" Show the ACL log entries.", +"SAVE", +" Save the current config to the ACL file.", +"SETUSER [ ...]", +" Create or modify a user with the specified attributes.", +"USERS", +" List all the registered usernames.", +"WHOAMI", +" Return the current connection username.", +NULL + }; + addReplyHelp(c,help); + } else { + addReplySubcommandSyntaxError(c); + } +} + +void addReplyCommandCategories(client *c, struct redisCommand *cmd) { + int flagcount = 0; + void *flaglen = addReplyDeferredLen(c); + for (int j = 0; ACLCommandCategories[j].flag != 0; j++) { + if (cmd->flags & ACLCommandCategories[j].flag) { + addReplyStatusFormat(c, "@%s", ACLCommandCategories[j].name); + flagcount++; + } + } + setDeferredSetLen(c, flaglen, flagcount); +} + +/* AUTH + * AUTH (Redis >= 6.0 form) + * + * When the user is omitted it means that we are trying to authenticate + * against the default user. */ +void authCommand(client *c) { + /* Only two or three argument forms are allowed. */ + if (c->argc > 3) { + addReplyErrorObject(c,shared.syntaxerr); + return; + } + /* Always redact the second argument */ + redactClientCommandArgument(c, 1); + + /* Handle the two different forms here. The form with two arguments + * will just use "default" as username. */ + robj *username, *password; + if (c->argc == 2) { + /* Mimic the old behavior of giving an error for the two commands + * from if no password is configured. */ + if (DefaultUser->flags & USER_FLAG_NOPASS) { + addReplyError(c,"AUTH called without any password " + "configured for the default user. Are you sure " + "your configuration is correct?"); + return; + } + + username = shared.default_username; + password = c->argv[1]; + } else { + username = c->argv[1]; + password = c->argv[2]; + redactClientCommandArgument(c, 2); + } + + if (ACLAuthenticateUser(c,username,password) == C_OK) { + addReply(c,shared.ok); + } else { + addReplyError(c,"-WRONGPASS invalid username-password pair or user is disabled."); + } +} + +/* Set the password for the "default" ACL user. This implements supports for + * requirepass config, so passing in NULL will set the user to be nopass. */ +void ACLUpdateDefaultUserPassword(sds password) { + ACLSetUser(DefaultUser,"resetpass",-1); + if (password) { + sds aclop = sdscatlen(sdsnew(">"), password, sdslen(password)); + ACLSetUser(DefaultUser,aclop,sdslen(aclop)); + sdsfree(aclop); + } else { + ACLSetUser(DefaultUser,"nopass",-1); + } +} diff --git a/redis.submodule/src/adlist.c b/redis.submodule/src/adlist.c index ec5f8bb..7670b2c 100644 --- a/redis.submodule/src/adlist.c +++ b/redis.submodule/src/adlist.c @@ -34,8 +34,9 @@ #include "zmalloc.h" /* Create a new list. The created list can be freed with - * AlFreeList(), but private value of every node need to be freed - * by the user before to call AlFreeList(). + * listRelease(), but private value of every node need to be freed + * by the user before to call listRelease(), or by setting a free method using + * listSetFreeMethod. * * On error, NULL is returned. Otherwise the pointer to the new list. */ list *listCreate(void) @@ -217,8 +218,8 @@ void listRewindTail(list *list, listIter *li) { * listDelNode(), but not to remove other elements. * * The function returns a pointer to the next element of the list, - * or NULL if there are no more elements, so the classical usage patter - * is: + * or NULL if there are no more elements, so the classical usage + * pattern is: * * iter = listGetIterator(list,); * while ((node = listNext(iter)) != NULL) { @@ -327,12 +328,11 @@ listNode *listIndex(list *list, long index) { } /* Rotate the list removing the tail node and inserting it to the head. */ -void listRotate(list *list) { - listNode *tail = list->tail; - +void listRotateTailToHead(list *list) { if (listLength(list) <= 1) return; /* Detach current tail */ + listNode *tail = list->tail; list->tail = tail->prev; list->tail->next = NULL; /* Move it as head */ @@ -342,18 +342,34 @@ void listRotate(list *list) { list->head = tail; } +/* Rotate the list removing the head node and inserting it to the tail. */ +void listRotateHeadToTail(list *list) { + if (listLength(list) <= 1) return; + + listNode *head = list->head; + /* Detach current head */ + list->head = head->next; + list->head->prev = NULL; + /* Move it as tail */ + list->tail->next = head; + head->next = NULL; + head->prev = list->tail; + list->tail = head; +} + /* Add all the elements of the list 'o' at the end of the * list 'l'. The list 'other' remains empty but otherwise valid. */ void listJoin(list *l, list *o) { - if (o->head) - o->head->prev = l->tail; + if (o->len == 0) return; + + o->head->prev = l->tail; if (l->tail) l->tail->next = o->head; else l->head = o->head; - if (o->tail) l->tail = o->tail; + l->tail = o->tail; l->len += o->len; /* Setup other as an empty list. */ diff --git a/redis.submodule/src/adlist.h b/redis.submodule/src/adlist.h index c954fac..dd8a8d6 100644 --- a/redis.submodule/src/adlist.h +++ b/redis.submodule/src/adlist.h @@ -66,7 +66,7 @@ typedef struct list { #define listSetMatchMethod(l,m) ((l)->match = (m)) #define listGetDupMethod(l) ((l)->dup) -#define listGetFree(l) ((l)->free) +#define listGetFreeMethod(l) ((l)->free) #define listGetMatchMethod(l) ((l)->match) /* Prototypes */ @@ -85,7 +85,8 @@ listNode *listSearchKey(list *list, void *key); listNode *listIndex(list *list, long index); void listRewind(list *list, listIter *li); void listRewindTail(list *list, listIter *li); -void listRotate(list *list); +void listRotateTailToHead(list *list); +void listRotateHeadToTail(list *list); void listJoin(list *l, list *o); /* Directions for iterators */ diff --git a/redis.submodule/src/ae.c b/redis.submodule/src/ae.c index 1ea6715..48fb63e 100644 --- a/redis.submodule/src/ae.c +++ b/redis.submodule/src/ae.c @@ -30,6 +30,9 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#include "ae.h" +#include "anet.h" + #include #include #include @@ -40,7 +43,6 @@ #include #include -#include "ae.h" #include "zmalloc.h" #include "config.h" @@ -60,22 +62,25 @@ #endif #endif + aeEventLoop *aeCreateEventLoop(int setsize) { aeEventLoop *eventLoop; int i; + monotonicInit(); /* just in case the calling app didn't initialize */ + if ((eventLoop = zmalloc(sizeof(*eventLoop))) == NULL) goto err; eventLoop->events = zmalloc(sizeof(aeFileEvent)*setsize); eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize); if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err; eventLoop->setsize = setsize; - eventLoop->lastTime = time(NULL); eventLoop->timeEventHead = NULL; eventLoop->timeEventNextId = 0; eventLoop->stop = 0; eventLoop->maxfd = -1; eventLoop->beforesleep = NULL; eventLoop->aftersleep = NULL; + eventLoop->flags = 0; if (aeApiCreate(eventLoop) == -1) goto err; /* Events with mask == AE_NONE are not set. So let's initialize the * vector with it. */ @@ -97,6 +102,14 @@ int aeGetSetSize(aeEventLoop *eventLoop) { return eventLoop->setsize; } +/* Tells the next iteration/s of the event processing to set timeout of 0. */ +void aeSetDontWait(aeEventLoop *eventLoop, int noWait) { + if (noWait) + eventLoop->flags |= AE_DONT_WAIT; + else + eventLoop->flags &= ~AE_DONT_WAIT; +} + /* Resize the maximum set size of the event loop. * If the requested set size is smaller than the current set size, but * there is already a file descriptor in use that is >= the requested @@ -126,6 +139,14 @@ void aeDeleteEventLoop(aeEventLoop *eventLoop) { aeApiFree(eventLoop); zfree(eventLoop->events); zfree(eventLoop->fired); + + /* Free the time events list. */ + aeTimeEvent *next_te, *te = eventLoop->timeEventHead; + while (te) { + next_te = te->next; + zfree(te); + te = next_te; + } zfree(eventLoop); } @@ -182,29 +203,6 @@ int aeGetFileEvents(aeEventLoop *eventLoop, int fd) { return fe->mask; } -static void aeGetTime(long *seconds, long *milliseconds) -{ - struct timeval tv; - - gettimeofday(&tv, NULL); - *seconds = tv.tv_sec; - *milliseconds = tv.tv_usec/1000; -} - -static void aeAddMillisecondsToNow(long long milliseconds, long *sec, long *ms) { - long cur_sec, cur_ms, when_sec, when_ms; - - aeGetTime(&cur_sec, &cur_ms); - when_sec = cur_sec + milliseconds/1000; - when_ms = cur_ms + milliseconds%1000; - if (when_ms >= 1000) { - when_sec ++; - when_ms -= 1000; - } - *sec = when_sec; - *ms = when_ms; -} - long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, aeTimeProc *proc, void *clientData, aeEventFinalizerProc *finalizerProc) @@ -215,12 +213,13 @@ long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, te = zmalloc(sizeof(*te)); if (te == NULL) return AE_ERR; te->id = id; - aeAddMillisecondsToNow(milliseconds,&te->when_sec,&te->when_ms); + te->when = getMonotonicUs() + milliseconds * 1000; te->timeProc = proc; te->finalizerProc = finalizerProc; te->clientData = clientData; te->prev = NULL; te->next = eventLoop->timeEventHead; + te->refcount = 0; if (te->next) te->next->prev = te; eventLoop->timeEventHead = te; @@ -240,10 +239,8 @@ int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id) return AE_ERR; /* NO event with the specified ID found */ } -/* Search the first timer to fire. - * This operation is useful to know how many time the select can be - * put in sleep without to delay any event. - * If there are no timers NULL is returned. +/* How many microseconds until the first timer should fire. + * If there are no timers, -1 is returned. * * Note that's O(N) since time events are unsorted. * Possible optimizations (not needed by Redis so far, but...): @@ -251,19 +248,19 @@ int aeDeleteTimeEvent(aeEventLoop *eventLoop, long long id) * Much better but still insertion or deletion of timers is O(N). * 2) Use a skiplist to have this operation as O(1) and insertion as O(log(N)). */ -static aeTimeEvent *aeSearchNearestTimer(aeEventLoop *eventLoop) -{ +static int64_t usUntilEarliestTimer(aeEventLoop *eventLoop) { aeTimeEvent *te = eventLoop->timeEventHead; - aeTimeEvent *nearest = NULL; + if (te == NULL) return -1; - while(te) { - if (!nearest || te->when_sec < nearest->when_sec || - (te->when_sec == nearest->when_sec && - te->when_ms < nearest->when_ms)) - nearest = te; + aeTimeEvent *earliest = NULL; + while (te) { + if (!earliest || te->when < earliest->when) + earliest = te; te = te->next; } - return nearest; + + monotime now = getMonotonicUs(); + return (now >= earliest->when) ? 0 : earliest->when - now; } /* Process time events */ @@ -271,42 +268,33 @@ static int processTimeEvents(aeEventLoop *eventLoop) { int processed = 0; aeTimeEvent *te; long long maxId; - time_t now = time(NULL); - - /* If the system clock is moved to the future, and then set back to the - * right value, time events may be delayed in a random way. Often this - * means that scheduled operations will not be performed soon enough. - * - * Here we try to detect system clock skews, and force all the time - * events to be processed ASAP when this happens: the idea is that - * processing events earlier is less dangerous than delaying them - * indefinitely, and practice suggests it is. */ - if (now < eventLoop->lastTime) { - te = eventLoop->timeEventHead; - while(te) { - te->when_sec = 0; - te = te->next; - } - } - eventLoop->lastTime = now; te = eventLoop->timeEventHead; maxId = eventLoop->timeEventNextId-1; + monotime now = getMonotonicUs(); while(te) { - long now_sec, now_ms; long long id; /* Remove events scheduled for deletion. */ if (te->id == AE_DELETED_EVENT_ID) { aeTimeEvent *next = te->next; + /* If a reference exists for this timer event, + * don't free it. This is currently incremented + * for recursive timerProc calls */ + if (te->refcount) { + te = next; + continue; + } if (te->prev) te->prev->next = te->next; else eventLoop->timeEventHead = te->next; if (te->next) te->next->prev = te->prev; - if (te->finalizerProc) + if (te->finalizerProc) { te->finalizerProc(eventLoop, te->clientData); + now = getMonotonicUs(); + } zfree(te); te = next; continue; @@ -321,17 +309,18 @@ static int processTimeEvents(aeEventLoop *eventLoop) { te = te->next; continue; } - aeGetTime(&now_sec, &now_ms); - if (now_sec > te->when_sec || - (now_sec == te->when_sec && now_ms >= te->when_ms)) - { + + if (te->when <= now) { int retval; id = te->id; + te->refcount++; retval = te->timeProc(eventLoop, id, te->clientData); + te->refcount--; processed++; + now = getMonotonicUs(); if (retval != AE_NOMORE) { - aeAddMillisecondsToNow(retval,&te->when_sec,&te->when_ms); + te->when = now + retval * 1000; } else { te->id = AE_DELETED_EVENT_ID; } @@ -351,8 +340,9 @@ static int processTimeEvents(aeEventLoop *eventLoop) { * if flags has AE_FILE_EVENTS set, file events are processed. * if flags has AE_TIME_EVENTS set, time events are processed. * if flags has AE_DONT_WAIT set the function returns ASAP until all - * if flags has AE_CALL_AFTER_SLEEP set, the aftersleep callback is called. * the events that's possible to process without to wait are processed. + * if flags has AE_CALL_AFTER_SLEEP set, the aftersleep callback is called. + * if flags has AE_CALL_BEFORE_SLEEP set, the beforesleep callback is called. * * The function returns the number of events processed. */ int aeProcessEvents(aeEventLoop *eventLoop, int flags) @@ -362,37 +352,23 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) /* Nothing to do? return ASAP */ if (!(flags & AE_TIME_EVENTS) && !(flags & AE_FILE_EVENTS)) return 0; - /* Note that we want call select() even if there are no + /* Note that we want to call select() even if there are no * file events to process as long as we want to process time * events, in order to sleep until the next time event is ready * to fire. */ if (eventLoop->maxfd != -1 || ((flags & AE_TIME_EVENTS) && !(flags & AE_DONT_WAIT))) { int j; - aeTimeEvent *shortest = NULL; struct timeval tv, *tvp; + int64_t usUntilTimer = -1; if (flags & AE_TIME_EVENTS && !(flags & AE_DONT_WAIT)) - shortest = aeSearchNearestTimer(eventLoop); - if (shortest) { - long now_sec, now_ms; + usUntilTimer = usUntilEarliestTimer(eventLoop); - aeGetTime(&now_sec, &now_ms); + if (usUntilTimer >= 0) { + tv.tv_sec = usUntilTimer / 1000000; + tv.tv_usec = usUntilTimer % 1000000; tvp = &tv; - - /* How many milliseconds we need to wait for the next - * time event to fire? */ - long long ms = - (shortest->when_sec - now_sec)*1000 + - shortest->when_ms - now_ms; - - if (ms > 0) { - tvp->tv_sec = ms/1000; - tvp->tv_usec = (ms % 1000)*1000; - } else { - tvp->tv_sec = 0; - tvp->tv_usec = 0; - } } else { /* If we have to check for events but need to return * ASAP because of AE_DONT_WAIT we need to set the timeout @@ -406,6 +382,14 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) } } + if (eventLoop->flags & AE_DONT_WAIT) { + tv.tv_sec = tv.tv_usec = 0; + tvp = &tv; + } + + if (eventLoop->beforesleep != NULL && flags & AE_CALL_BEFORE_SLEEP) + eventLoop->beforesleep(eventLoop); + /* Call the multiplexing API, will return only on timeout or when * some event fires. */ numevents = aeApiPoll(eventLoop, tvp); @@ -421,7 +405,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) int fired = 0; /* Number of events fired for current fd. */ /* Normally we execute the readable event first, and the writable - * event laster. This is useful as sometimes we may be able + * event later. This is useful as sometimes we may be able * to serve the reply of a query immediately after processing the * query. * @@ -429,7 +413,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) * asking us to do the reverse: never fire the writable event * after the readable. In such a case, we invert the calls. * This is useful when, for instance, we want to do things - * in the beforeSleep() hook, like fsynching a file to disk, + * in the beforeSleep() hook, like fsyncing a file to disk, * before replying to a client. */ int invert = fe->mask & AE_BARRIER; @@ -442,6 +426,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) if (!invert && fe->mask & mask & AE_READABLE) { fe->rfileProc(eventLoop,fd,fe->clientData,mask); fired++; + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ } /* Fire the writable event. */ @@ -454,8 +439,11 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) /* If we have to invert the call, fire the readable event now * after the writable one. */ - if (invert && fe->mask & mask & AE_READABLE) { - if (!fired || fe->wfileProc != fe->rfileProc) { + if (invert) { + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ + if ((fe->mask & mask & AE_READABLE) && + (!fired || fe->wfileProc != fe->rfileProc)) + { fe->rfileProc(eventLoop,fd,fe->clientData,mask); fired++; } @@ -496,9 +484,9 @@ int aeWait(int fd, int mask, long long milliseconds) { void aeMain(aeEventLoop *eventLoop) { eventLoop->stop = 0; while (!eventLoop->stop) { - if (eventLoop->beforesleep != NULL) - eventLoop->beforesleep(eventLoop); - aeProcessEvents(eventLoop, AE_ALL_EVENTS|AE_CALL_AFTER_SLEEP); + aeProcessEvents(eventLoop, AE_ALL_EVENTS| + AE_CALL_BEFORE_SLEEP| + AE_CALL_AFTER_SLEEP); } } diff --git a/redis.submodule/src/ae.h b/redis.submodule/src/ae.h index 184fe3d..4a1edb6 100644 --- a/redis.submodule/src/ae.h +++ b/redis.submodule/src/ae.h @@ -33,7 +33,7 @@ #ifndef __AE_H__ #define __AE_H__ -#include +#include "monotonic.h" #define AE_OK 0 #define AE_ERR -1 @@ -47,11 +47,12 @@ things to disk before sending replies, and want to do that in a group fashion. */ -#define AE_FILE_EVENTS 1 -#define AE_TIME_EVENTS 2 +#define AE_FILE_EVENTS (1<<0) +#define AE_TIME_EVENTS (1<<1) #define AE_ALL_EVENTS (AE_FILE_EVENTS|AE_TIME_EVENTS) -#define AE_DONT_WAIT 4 -#define AE_CALL_AFTER_SLEEP 8 +#define AE_DONT_WAIT (1<<2) +#define AE_CALL_BEFORE_SLEEP (1<<3) +#define AE_CALL_AFTER_SLEEP (1<<4) #define AE_NOMORE -1 #define AE_DELETED_EVENT_ID -1 @@ -78,13 +79,14 @@ typedef struct aeFileEvent { /* Time event structure */ typedef struct aeTimeEvent { long long id; /* time event identifier. */ - long when_sec; /* seconds */ - long when_ms; /* milliseconds */ + monotime when; aeTimeProc *timeProc; aeEventFinalizerProc *finalizerProc; void *clientData; struct aeTimeEvent *prev; struct aeTimeEvent *next; + int refcount; /* refcount to prevent timer events from being + * freed in recursive time event calls. */ } aeTimeEvent; /* A fired event */ @@ -98,7 +100,6 @@ typedef struct aeEventLoop { int maxfd; /* highest file descriptor currently registered */ int setsize; /* max number of file descriptors tracked */ long long timeEventNextId; - time_t lastTime; /* Used to detect system clock skew */ aeFileEvent *events; /* Registered events */ aeFiredEvent *fired; /* Fired events */ aeTimeEvent *timeEventHead; @@ -106,6 +107,7 @@ typedef struct aeEventLoop { void *apidata; /* This is used for polling API specific data */ aeBeforeSleepProc *beforesleep; aeBeforeSleepProc *aftersleep; + int flags; } aeEventLoop; /* Prototypes */ @@ -128,5 +130,6 @@ void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep void aeSetAfterSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *aftersleep); int aeGetSetSize(aeEventLoop *eventLoop); int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); +void aeSetDontWait(aeEventLoop *eventLoop, int noWait); #endif diff --git a/redis.submodule/src/ae_epoll.c b/redis.submodule/src/ae_epoll.c index 410aac7..023c93a 100644 --- a/redis.submodule/src/ae_epoll.c +++ b/redis.submodule/src/ae_epoll.c @@ -51,6 +51,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { zfree(state); return -1; } + anetCloexec(state->epfd); eventLoop->apidata = state; return 0; } @@ -110,7 +111,7 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { int retval, numevents = 0; retval = epoll_wait(state->epfd,state->events,eventLoop->setsize, - tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1); + tvp ? (tvp->tv_sec*1000 + (tvp->tv_usec + 999)/1000) : -1); if (retval > 0) { int j; @@ -121,8 +122,8 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { if (e->events & EPOLLIN) mask |= AE_READABLE; if (e->events & EPOLLOUT) mask |= AE_WRITABLE; - if (e->events & EPOLLERR) mask |= AE_WRITABLE; - if (e->events & EPOLLHUP) mask |= AE_WRITABLE; + if (e->events & EPOLLERR) mask |= AE_WRITABLE|AE_READABLE; + if (e->events & EPOLLHUP) mask |= AE_WRITABLE|AE_READABLE; eventLoop->fired[j].fd = e->data.fd; eventLoop->fired[j].mask = mask; } diff --git a/redis.submodule/src/ae_evport.c b/redis.submodule/src/ae_evport.c index 5c317be..7a0b03a 100644 --- a/redis.submodule/src/ae_evport.c +++ b/redis.submodule/src/ae_evport.c @@ -67,7 +67,7 @@ static int evport_debug = 0; typedef struct aeApiState { int portfd; /* event port */ - int npending; /* # of pending fds */ + uint_t npending; /* # of pending fds */ int pending_fds[MAX_EVENT_BATCHSZ]; /* pending fds */ int pending_masks[MAX_EVENT_BATCHSZ]; /* pending fds' masks */ } aeApiState; @@ -82,6 +82,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { zfree(state); return -1; } + anetCloexec(state->portfd); state->npending = 0; @@ -95,6 +96,8 @@ static int aeApiCreate(aeEventLoop *eventLoop) { } static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + (void) eventLoop; + (void) setsize; /* Nothing to resize here. */ return 0; } @@ -107,7 +110,7 @@ static void aeApiFree(aeEventLoop *eventLoop) { } static int aeApiLookupPending(aeApiState *state, int fd) { - int i; + uint_t i; for (i = 0; i < state->npending; i++) { if (state->pending_fds[i] == fd) @@ -232,7 +235,7 @@ static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { /* * ENOMEM is a potentially transient condition, but the kernel won't * generally return it unless things are really bad. EAGAIN indicates - * we've reached an resource limit, for which it doesn't make sense to + * we've reached a resource limit, for which it doesn't make sense to * retry (counter-intuitively). All other errors indicate a bug. In any * of these cases, the best we can do is to abort. */ @@ -243,7 +246,7 @@ static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { aeApiState *state = eventLoop->apidata; struct timespec timeout, *tsp; - int mask, i; + uint_t mask, i; uint_t nevents; port_event_t event[MAX_EVENT_BATCHSZ]; diff --git a/redis.submodule/src/ae_kqueue.c b/redis.submodule/src/ae_kqueue.c index 6796f4c..6d62720 100644 --- a/redis.submodule/src/ae_kqueue.c +++ b/redis.submodule/src/ae_kqueue.c @@ -36,8 +36,29 @@ typedef struct aeApiState { int kqfd; struct kevent *events; + + /* Events mask for merge read and write event. + * To reduce memory consumption, we use 2 bits to store the mask + * of an event, so that 1 byte will store the mask of 4 events. */ + char *eventsMask; } aeApiState; +#define EVENT_MASK_MALLOC_SIZE(sz) (((sz) + 3) / 4) +#define EVENT_MASK_OFFSET(fd) ((fd) % 4 * 2) +#define EVENT_MASK_ENCODE(fd, mask) (((mask) & 0x3) << EVENT_MASK_OFFSET(fd)) + +static inline int getEventMask(const char *eventsMask, int fd) { + return (eventsMask[fd/4] >> EVENT_MASK_OFFSET(fd)) & 0x3; +} + +static inline void addEventMask(char *eventsMask, int fd, int mask) { + eventsMask[fd/4] |= EVENT_MASK_ENCODE(fd, mask); +} + +static inline void resetEventMask(char *eventsMask, int fd) { + eventsMask[fd/4] &= ~EVENT_MASK_ENCODE(fd, 0x3); +} + static int aeApiCreate(aeEventLoop *eventLoop) { aeApiState *state = zmalloc(sizeof(aeApiState)); @@ -53,6 +74,9 @@ static int aeApiCreate(aeEventLoop *eventLoop) { zfree(state); return -1; } + anetCloexec(state->kqfd); + state->eventsMask = zmalloc(EVENT_MASK_MALLOC_SIZE(eventLoop->setsize)); + memset(state->eventsMask, 0, EVENT_MASK_MALLOC_SIZE(eventLoop->setsize)); eventLoop->apidata = state; return 0; } @@ -61,6 +85,8 @@ static int aeApiResize(aeEventLoop *eventLoop, int setsize) { aeApiState *state = eventLoop->apidata; state->events = zrealloc(state->events, sizeof(struct kevent)*setsize); + state->eventsMask = zrealloc(state->eventsMask, EVENT_MASK_MALLOC_SIZE(setsize)); + memset(state->eventsMask, 0, EVENT_MASK_MALLOC_SIZE(setsize)); return 0; } @@ -69,6 +95,7 @@ static void aeApiFree(aeEventLoop *eventLoop) { close(state->kqfd); zfree(state->events); + zfree(state->eventsMask); zfree(state); } @@ -119,15 +146,37 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { if (retval > 0) { int j; - numevents = retval; - for(j = 0; j < numevents; j++) { - int mask = 0; + /* Normally we execute the read event first and then the write event. + * When the barrier is set, we will do it reverse. + * + * However, under kqueue, read and write events would be separate + * events, which would make it impossible to control the order of + * reads and writes. So we store the event's mask we've got and merge + * the same fd events later. */ + for (j = 0; j < retval; j++) { struct kevent *e = state->events+j; + int fd = e->ident; + int mask = 0; - if (e->filter == EVFILT_READ) mask |= AE_READABLE; - if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE; - eventLoop->fired[j].fd = e->ident; - eventLoop->fired[j].mask = mask; + if (e->filter == EVFILT_READ) mask = AE_READABLE; + else if (e->filter == EVFILT_WRITE) mask = AE_WRITABLE; + addEventMask(state->eventsMask, fd, mask); + } + + /* Re-traversal to merge read and write events, and set the fd's mask to + * 0 so that events are not added again when the fd is encountered again. */ + numevents = 0; + for (j = 0; j < retval; j++) { + struct kevent *e = state->events+j; + int fd = e->ident; + int mask = getEventMask(state->eventsMask, fd); + + if (mask) { + eventLoop->fired[numevents].fd = fd; + eventLoop->fired[numevents].mask = mask; + resetEventMask(state->eventsMask, fd); + numevents++; + } } } return numevents; diff --git a/redis.submodule/src/anet.c b/redis.submodule/src/anet.c index 2981fca..a121c27 100644 --- a/redis.submodule/src/anet.c +++ b/redis.submodule/src/anet.c @@ -69,6 +69,11 @@ int anetSetBlock(char *err, int fd, int non_block) { return ANET_ERR; } + /* Check if this flag has been set or unset, if so, + * then there is no need to call fcntl to set/unset it again. */ + if (!!(flags & O_NONBLOCK) == !!non_block) + return ANET_OK; + if (non_block) flags |= O_NONBLOCK; else @@ -89,6 +94,29 @@ int anetBlock(char *err, int fd) { return anetSetBlock(err,fd,0); } +/* Enable the FD_CLOEXEC on the given fd to avoid fd leaks. + * This function should be invoked for fd's on specific places + * where fork + execve system calls are called. */ +int anetCloexec(int fd) { + int r; + int flags; + + do { + r = fcntl(fd, F_GETFD); + } while (r == -1 && errno == EINTR); + + if (r == -1 || (r & FD_CLOEXEC)) + return r; + + flags = r | FD_CLOEXEC; + + do { + r = fcntl(fd, F_SETFD, flags); + } while (r == -1 && errno == EINTR); + + return r; +} + /* Set TCP keep alive option to detect dead peers. The interval option * is only used for Linux as we are using Linux-specific APIs to set * the probe send time, interval, and count. */ @@ -158,49 +186,41 @@ int anetDisableTcpNoDelay(char *err, int fd) return anetSetTcpNoDelay(err, fd, 0); } +/* Set the socket send timeout (SO_SNDTIMEO socket option) to the specified + * number of milliseconds, or disable it if the 'ms' argument is zero. */ +int anetSendTimeout(char *err, int fd, long long ms) { + struct timeval tv; -int anetSetSendBuffer(char *err, int fd, int buffsize) -{ - if (setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &buffsize, sizeof(buffsize)) == -1) - { - anetSetError(err, "setsockopt SO_SNDBUF: %s", strerror(errno)); - return ANET_ERR; - } - return ANET_OK; -} - -int anetTcpKeepAlive(char *err, int fd) -{ - int yes = 1; - if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes)) == -1) { - anetSetError(err, "setsockopt SO_KEEPALIVE: %s", strerror(errno)); + tv.tv_sec = ms/1000; + tv.tv_usec = (ms%1000)*1000; + if (setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)) == -1) { + anetSetError(err, "setsockopt SO_SNDTIMEO: %s", strerror(errno)); return ANET_ERR; } return ANET_OK; } -/* Set the socket send timeout (SO_SNDTIMEO socket option) to the specified +/* Set the socket receive timeout (SO_RCVTIMEO socket option) to the specified * number of milliseconds, or disable it if the 'ms' argument is zero. */ -int anetSendTimeout(char *err, int fd, long long ms) { +int anetRecvTimeout(char *err, int fd, long long ms) { struct timeval tv; tv.tv_sec = ms/1000; tv.tv_usec = (ms%1000)*1000; - if (setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)) == -1) { - anetSetError(err, "setsockopt SO_SNDTIMEO: %s", strerror(errno)); + if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) == -1) { + anetSetError(err, "setsockopt SO_RCVTIMEO: %s", strerror(errno)); return ANET_ERR; } return ANET_OK; } -/* anetGenericResolve() is called by anetResolve() and anetResolveIP() to - * do the actual work. It resolves the hostname "host" and set the string - * representation of the IP address into the buffer pointed by "ipbuf". +/* Resolve the hostname "host" and set the string representation of the + * IP address into the buffer pointed by "ipbuf". * * If flags is set to ANET_IP_ONLY the function only resolves hostnames * that are actually already IPv4 or IPv6 addresses. This turns the function * into a validating / normalizing function. */ -int anetGenericResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, +int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, int flags) { struct addrinfo hints, *info; @@ -227,14 +247,6 @@ int anetGenericResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, return ANET_OK; } -int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len) { - return anetGenericResolve(err,host,ipbuf,ipbuf_len,ANET_NONE); -} - -int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len) { - return anetGenericResolve(err,host,ipbuf,ipbuf_len,ANET_IP_ONLY); -} - static int anetSetReuseAddr(char *err, int fd) { int yes = 1; /* Make sure connection-intensive things like the redis benchmark @@ -265,8 +277,8 @@ static int anetCreateSocket(char *err, int domain) { #define ANET_CONNECT_NONE 0 #define ANET_CONNECT_NONBLOCK 1 #define ANET_CONNECT_BE_BINDING 2 /* Best effort binding. */ -static int anetTcpGenericConnect(char *err, char *addr, int port, - char *source_addr, int flags) +static int anetTcpGenericConnect(char *err, const char *addr, int port, + const char *source_addr, int flags) { int s = ANET_ERR, rv; char portstr[6]; /* strlen("65535") + 1; */ @@ -345,31 +357,19 @@ static int anetTcpGenericConnect(char *err, char *addr, int port, } } -int anetTcpConnect(char *err, char *addr, int port) -{ - return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONE); -} - -int anetTcpNonBlockConnect(char *err, char *addr, int port) +int anetTcpNonBlockConnect(char *err, const char *addr, int port) { return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONBLOCK); } -int anetTcpNonBlockBindConnect(char *err, char *addr, int port, - char *source_addr) -{ - return anetTcpGenericConnect(err,addr,port,source_addr, - ANET_CONNECT_NONBLOCK); -} - -int anetTcpNonBlockBestEffortBindConnect(char *err, char *addr, int port, - char *source_addr) +int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, + const char *source_addr) { return anetTcpGenericConnect(err,addr,port,source_addr, ANET_CONNECT_NONBLOCK|ANET_CONNECT_BE_BINDING); } -int anetUnixGenericConnect(char *err, char *path, int flags) +int anetUnixGenericConnect(char *err, const char *path, int flags) { int s; struct sockaddr_un sa; @@ -397,46 +397,6 @@ int anetUnixGenericConnect(char *err, char *path, int flags) return s; } -int anetUnixConnect(char *err, char *path) -{ - return anetUnixGenericConnect(err,path,ANET_CONNECT_NONE); -} - -int anetUnixNonBlockConnect(char *err, char *path) -{ - return anetUnixGenericConnect(err,path,ANET_CONNECT_NONBLOCK); -} - -/* Like read(2) but make sure 'count' is read before to return - * (unless error or EOF condition is encountered) */ -int anetRead(int fd, char *buf, int count) -{ - ssize_t nread, totlen = 0; - while(totlen != count) { - nread = read(fd,buf,count-totlen); - if (nread == 0) return totlen; - if (nread == -1) return -1; - totlen += nread; - buf += nread; - } - return totlen; -} - -/* Like write(2) but make sure 'count' is written before to return - * (unless error is encountered) */ -int anetWrite(int fd, char *buf, int count) -{ - ssize_t nwritten, totlen = 0; - while(totlen != count) { - nwritten = write(fd,buf,count-totlen); - if (nwritten == 0) return totlen; - if (nwritten == -1) return -1; - totlen += nwritten; - buf += nwritten; - } - return totlen; -} - static int anetListen(char *err, int s, struct sockaddr *sa, socklen_t len, int backlog) { if (bind(s,sa,len) == -1) { anetSetError(err, "bind: %s", strerror(errno)); @@ -456,7 +416,6 @@ static int anetV6Only(char *err, int s) { int yes = 1; if (setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,&yes,sizeof(yes)) == -1) { anetSetError(err, "setsockopt: %s", strerror(errno)); - close(s); return ANET_ERR; } return ANET_OK; @@ -473,6 +432,10 @@ static int _anetTcpServer(char *err, int port, char *bindaddr, int af, int backl hints.ai_family = af; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE; /* No effect if bindaddr != NULL */ + if (bindaddr && !strcmp("*", bindaddr)) + bindaddr = NULL; + if (af == AF_INET6 && bindaddr && !strcmp("::*", bindaddr)) + bindaddr = NULL; if ((rv = getaddrinfo(bindaddr,_port,&hints,&servinfo)) != 0) { anetSetError(err, "%s", gai_strerror(rv)); @@ -574,11 +537,15 @@ int anetUnixAccept(char *err, int s) { return fd; } -int anetPeerToString(int fd, char *ip, size_t ip_len, int *port) { +int anetFdToString(int fd, char *ip, size_t ip_len, int *port, int fd_to_str_type) { struct sockaddr_storage sa; socklen_t salen = sizeof(sa); - if (getpeername(fd,(struct sockaddr*)&sa,&salen) == -1) goto error; + if (fd_to_str_type == FD_TO_PEER_NAME) { + if (getpeername(fd, (struct sockaddr *)&sa, &salen) == -1) goto error; + } else { + if (getsockname(fd, (struct sockaddr *)&sa, &salen) == -1) goto error; + } if (ip_len == 0) goto error; if (sa.ss_family == AF_INET) { @@ -590,7 +557,7 @@ int anetPeerToString(int fd, char *ip, size_t ip_len, int *port) { if (ip) inet_ntop(AF_INET6,(void*)&(s->sin6_addr),ip,ip_len); if (port) *port = ntohs(s->sin6_port); } else if (sa.ss_family == AF_UNIX) { - if (ip) strncpy(ip,"/unixsocket",ip_len); + if (ip) snprintf(ip, ip_len, "/unixsocket"); if (port) *port = 0; } else { goto error; @@ -618,41 +585,11 @@ int anetFormatAddr(char *buf, size_t buf_len, char *ip, int port) { "[%s]:%d" : "%s:%d", ip, port); } -/* Like anetFormatAddr() but extract ip and port from the socket's peer. */ -int anetFormatPeer(int fd, char *buf, size_t buf_len) { +/* Like anetFormatAddr() but extract ip and port from the socket's peer/sockname. */ +int anetFormatFdAddr(int fd, char *buf, size_t buf_len, int fd_to_str_type) { char ip[INET6_ADDRSTRLEN]; int port; - anetPeerToString(fd,ip,sizeof(ip),&port); + anetFdToString(fd,ip,sizeof(ip),&port,fd_to_str_type); return anetFormatAddr(buf, buf_len, ip, port); } - -int anetSockName(int fd, char *ip, size_t ip_len, int *port) { - struct sockaddr_storage sa; - socklen_t salen = sizeof(sa); - - if (getsockname(fd,(struct sockaddr*)&sa,&salen) == -1) { - if (port) *port = 0; - ip[0] = '?'; - ip[1] = '\0'; - return -1; - } - if (sa.ss_family == AF_INET) { - struct sockaddr_in *s = (struct sockaddr_in *)&sa; - if (ip) inet_ntop(AF_INET,(void*)&(s->sin_addr),ip,ip_len); - if (port) *port = ntohs(s->sin_port); - } else { - struct sockaddr_in6 *s = (struct sockaddr_in6 *)&sa; - if (ip) inet_ntop(AF_INET6,(void*)&(s->sin6_addr),ip,ip_len); - if (port) *port = ntohs(s->sin6_port); - } - return 0; -} - -int anetFormatSock(int fd, char *fmt, size_t fmt_len) { - char ip[INET6_ADDRSTRLEN]; - int port; - - anetSockName(fd,ip,sizeof(ip),&port); - return anetFormatAddr(fmt, fmt_len, ip, port); -} diff --git a/redis.submodule/src/anet.h b/redis.submodule/src/anet.h index 7142f78..2a685cc 100644 --- a/redis.submodule/src/anet.h +++ b/redis.submodule/src/anet.h @@ -49,32 +49,28 @@ #undef ip_len #endif -int anetTcpConnect(char *err, char *addr, int port); -int anetTcpNonBlockConnect(char *err, char *addr, int port); -int anetTcpNonBlockBindConnect(char *err, char *addr, int port, char *source_addr); -int anetTcpNonBlockBestEffortBindConnect(char *err, char *addr, int port, char *source_addr); -int anetUnixConnect(char *err, char *path); -int anetUnixNonBlockConnect(char *err, char *path); -int anetRead(int fd, char *buf, int count); -int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len); -int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len); +/* FD to address string conversion types */ +#define FD_TO_PEER_NAME 0 +#define FD_TO_SOCK_NAME 1 + +int anetTcpNonBlockConnect(char *err, const char *addr, int port); +int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, const char *source_addr); +int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, int flags); int anetTcpServer(char *err, int port, char *bindaddr, int backlog); int anetTcp6Server(char *err, int port, char *bindaddr, int backlog); int anetUnixServer(char *err, char *path, mode_t perm, int backlog); int anetTcpAccept(char *err, int serversock, char *ip, size_t ip_len, int *port); int anetUnixAccept(char *err, int serversock); -int anetWrite(int fd, char *buf, int count); int anetNonBlock(char *err, int fd); int anetBlock(char *err, int fd); +int anetCloexec(int fd); int anetEnableTcpNoDelay(char *err, int fd); int anetDisableTcpNoDelay(char *err, int fd); -int anetTcpKeepAlive(char *err, int fd); int anetSendTimeout(char *err, int fd, long long ms); -int anetPeerToString(int fd, char *ip, size_t ip_len, int *port); +int anetRecvTimeout(char *err, int fd, long long ms); +int anetFdToString(int fd, char *ip, size_t ip_len, int *port, int fd_to_str_type); int anetKeepAlive(char *err, int fd, int interval); -int anetSockName(int fd, char *ip, size_t ip_len, int *port); int anetFormatAddr(char *fmt, size_t fmt_len, char *ip, int port); -int anetFormatPeer(int fd, char *fmt, size_t fmt_len); -int anetFormatSock(int fd, char *fmt, size_t fmt_len); +int anetFormatFdAddr(int fd, char *buf, size_t buf_len, int fd_to_str_type); #endif diff --git a/redis.submodule/src/aof.c b/redis.submodule/src/aof.c index 1014164..566a986 100644 --- a/redis.submodule/src/aof.c +++ b/redis.submodule/src/aof.c @@ -162,7 +162,9 @@ void aofRewriteBufferAppend(unsigned char *s, unsigned long len) { /* Install a file event to send data to the rewrite child if there is * not one already. */ - if (aeGetFileEvents(server.el,server.aof_pipe_write_data_to_child) == 0) { + if (!server.aof_stop_sending_diff && + aeGetFileEvents(server.el,server.aof_pipe_write_data_to_child) == 0) + { aeCreateFileEvent(server.el, server.aof_pipe_write_data_to_child, AE_WRITABLE, aofChildWriteDiffData, NULL); } @@ -206,24 +208,24 @@ int aofFsyncInProgress(void) { /* Starts a background task that performs fsync() against the specified * file descriptor (the one of the AOF file) in another thread. */ void aof_background_fsync(int fd) { - bioCreateBackgroundJob(BIO_AOF_FSYNC,(void*)(long)fd,NULL,NULL); + bioCreateFsyncJob(fd); } /* Kills an AOFRW child process if exists */ -static void killAppendOnlyChild(void) { +void killAppendOnlyChild(void) { int statloc; /* No AOFRW child? return. */ - if (server.aof_child_pid == -1) return; + if (server.child_type != CHILD_TYPE_AOF) return; /* Kill AOFRW child, wait for child exit. */ serverLog(LL_NOTICE,"Killing running AOF rewrite child: %ld", - (long) server.aof_child_pid); - if (kill(server.aof_child_pid,SIGUSR1) != -1) { - while(wait3(&statloc,0,NULL) != server.aof_child_pid); + (long) server.child_pid); + if (kill(server.child_pid,SIGUSR1) != -1) { + while(waitpid(-1, &statloc, 0) != server.child_pid); } /* Reset the buffer accumulating changes while the child saves. */ aofRewriteBufferReset(); - aofRemoveTempFile(server.aof_child_pid); - server.aof_child_pid = -1; + aofRemoveTempFile(server.child_pid); + resetChildState(); server.aof_rewrite_time_start = -1; /* Close pipes used for IPC between the two processes. */ aofClosePipes(); @@ -234,13 +236,21 @@ static void killAppendOnlyChild(void) { void stopAppendOnly(void) { serverAssert(server.aof_state != AOF_OFF); flushAppendOnlyFile(1); - redis_fsync(server.aof_fd); + if (redis_fsync(server.aof_fd) == -1) { + serverLog(LL_WARNING,"Fail to fsync the AOF file: %s",strerror(errno)); + } else { + server.aof_fsync_offset = server.aof_current_size; + server.aof_last_fsync = server.unixtime; + } close(server.aof_fd); server.aof_fd = -1; server.aof_selected_db = -1; server.aof_state = AOF_OFF; + server.aof_rewrite_scheduled = 0; killAppendOnlyChild(); + sdsfree(server.aof_buf); + server.aof_buf = sdsempty(); } /* Called when the user switches from "appendonly no" to "appendonly yes" @@ -262,14 +272,14 @@ int startAppendOnly(void) { strerror(errno)); return C_ERR; } - if (server.rdb_child_pid != -1) { + if (hasActiveChildProcess() && server.child_type != CHILD_TYPE_AOF) { server.aof_rewrite_scheduled = 1; - serverLog(LL_WARNING,"AOF was enabled but there is already a child process saving an RDB file on disk. An AOF background was scheduled to start when possible."); + serverLog(LL_WARNING,"AOF was enabled but there is already another background operation. An AOF background was scheduled to start when possible."); } else { /* If there is a pending AOF rewrite, we need to switch it off and * start a new one: the old one cannot be reused because it is not * accumulating the AOF buffer. */ - if (server.aof_child_pid != -1) { + if (server.child_type == CHILD_TYPE_AOF) { serverLog(LL_WARNING,"AOF was enabled but there is already an AOF rewriting in background. Stopping background AOF and starting a rewrite now."); killAppendOnlyChild(); } @@ -284,6 +294,21 @@ int startAppendOnly(void) { server.aof_state = AOF_WAIT_REWRITE; server.aof_last_fsync = server.unixtime; server.aof_fd = newfd; + + /* If AOF fsync error in bio job, we just ignore it and log the event. */ + int aof_bio_fsync_status; + atomicGet(server.aof_bio_fsync_status, aof_bio_fsync_status); + if (aof_bio_fsync_status == C_ERR) { + serverLog(LL_WARNING, + "AOF reopen, just ignore the AOF fsync error in bio job"); + atomicSet(server.aof_bio_fsync_status,C_OK); + } + + /* If AOF was in error state, we just ignore it and log the event. */ + if (server.aof_last_write_status == C_ERR) { + serverLog(LL_WARNING,"AOF reopen, just ignore the last error."); + server.aof_last_write_status = C_OK; + } return C_OK; } @@ -301,9 +326,7 @@ ssize_t aofWrite(int fd, const char *buf, size_t len) { nwritten = write(fd, buf, len); if (nwritten < 0) { - if (errno == EINTR) { - continue; - } + if (errno == EINTR) continue; return totwritten ? totwritten : -1; } @@ -385,6 +408,10 @@ void flushAppendOnlyFile(int force) { * there is much to do about the whole server stopping for power problems * or alike */ + if (server.aof_flush_sleep && sdslen(server.aof_buf)) { + usleep(server.aof_flush_sleep); + } + latencyStartMonitor(latency); nwritten = aofWrite(server.aof_fd,server.aof_buf,sdslen(server.aof_buf)); latencyEndMonitor(latency); @@ -395,7 +422,7 @@ void flushAppendOnlyFile(int force) { * useful for graphing / monitoring purposes. */ if (sync_in_progress) { latencyAddSampleIfNeeded("aof-write-pending-fsync",latency); - } else if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) { + } else if (hasActiveChildProcess()) { latencyAddSampleIfNeeded("aof-write-active-child",latency); } else { latencyAddSampleIfNeeded("aof-write-alone",latency); @@ -448,10 +475,11 @@ void flushAppendOnlyFile(int force) { /* Handle the AOF write error. */ if (server.aof_fsync == AOF_FSYNC_ALWAYS) { - /* We can't recover when the fsync policy is ALWAYS since the - * reply for the client is already in the output buffers, and we - * have the contract with the user that on acknowledged write data - * is synced on disk. */ + /* We can't recover when the fsync policy is ALWAYS since the reply + * for the client is already in the output buffers (both writes and + * reads), and the changes to the db can't be rolled back. Since we + * have a contract with the user that on acknowledged or observed + * writes are is synced on disk, we must exit. */ serverLog(LL_WARNING,"Can't recover from AOF write error when the AOF fsync policy is 'always'. Exiting..."); exit(1); } else { @@ -491,16 +519,22 @@ void flushAppendOnlyFile(int force) { try_fsync: /* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are * children doing I/O in the background. */ - if (server.aof_no_fsync_on_rewrite && - (server.aof_child_pid != -1 || server.rdb_child_pid != -1)) - return; + if (server.aof_no_fsync_on_rewrite && hasActiveChildProcess()) + return; /* Perform the fsync if needed. */ if (server.aof_fsync == AOF_FSYNC_ALWAYS) { /* redis_fsync is defined as fdatasync() for Linux in order to avoid * flushing metadata. */ latencyStartMonitor(latency); - redis_fsync(server.aof_fd); /* Let's try to get this data on the disk */ + /* Let's try to get this data on the disk. To guarantee data safe when + * the AOF fsync policy is 'always', we should exit if failed to fsync + * AOF (see comment next to the exit(1) after write error above). */ + if (redis_fsync(server.aof_fd) == -1) { + serverLog(LL_WARNING,"Can't persist AOF for fsync error when the " + "AOF fsync policy is 'always': %s. Exiting...", strerror(errno)); + exit(1); + } latencyEndMonitor(latency); latencyAddSampleIfNeeded("aof-fsync-always",latency); server.aof_fsync_offset = server.aof_current_size; @@ -540,7 +574,7 @@ sds catAppendOnlyGenericCommand(sds dst, int argc, robj **argv) { return dst; } -/* Create the sds representation of an PEXPIREAT command, using +/* Create the sds representation of a PEXPIREAT command, using * 'seconds' as time to live and 'cmd' to understand what command * we are translating into a PEXPIREAT. * @@ -568,19 +602,16 @@ sds catAppendOnlyExpireAtCommand(sds buf, struct redisCommand *cmd, robj *key, r } decrRefCount(seconds); - argv[0] = createStringObject("PEXPIREAT",9); + argv[0] = shared.pexpireat; argv[1] = key; argv[2] = createStringObjectFromLongLong(when); buf = catAppendOnlyGenericCommand(buf, 3, argv); - decrRefCount(argv[0]); decrRefCount(argv[2]); return buf; } void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int argc) { sds buf = sdsempty(); - robj *tmpargv[3]; - /* The DB this command was targeting is not the same as the last command * we appended. To issue a SELECT command is needed. */ if (dictid != server.aof_selected_db) { @@ -596,30 +627,34 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a cmd->proc == expireatCommand) { /* Translate EXPIRE/PEXPIRE/EXPIREAT into PEXPIREAT */ buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]); - } else if (cmd->proc == setexCommand || cmd->proc == psetexCommand) { - /* Translate SETEX/PSETEX to SET and PEXPIREAT */ - tmpargv[0] = createStringObject("SET",3); - tmpargv[1] = argv[1]; - tmpargv[2] = argv[3]; - buf = catAppendOnlyGenericCommand(buf,3,tmpargv); - decrRefCount(tmpargv[0]); - buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]); } else if (cmd->proc == setCommand && argc > 3) { - int i; - robj *exarg = NULL, *pxarg = NULL; - /* Translate SET [EX seconds][PX milliseconds] to SET and PEXPIREAT */ - buf = catAppendOnlyGenericCommand(buf,3,argv); - for (i = 3; i < argc; i ++) { - if (!strcasecmp(argv[i]->ptr, "ex")) exarg = argv[i+1]; - if (!strcasecmp(argv[i]->ptr, "px")) pxarg = argv[i+1]; + robj *pxarg = NULL; + /* When SET is used with EX/PX argument setGenericCommand propagates them with PX millisecond argument. + * So since the command arguments are re-written there, we can rely here on the index of PX being 3. */ + if (!strcasecmp(argv[3]->ptr, "px")) { + pxarg = argv[4]; + } + /* For AOF we convert SET key value relative time in milliseconds to SET key value absolute time in + * millisecond. Whenever the condition is true it implies that original SET has been transformed + * to SET PX with millisecond time argument so we do not need to worry about unit here.*/ + if (pxarg) { + robj *millisecond = getDecodedObject(pxarg); + long long when = strtoll(millisecond->ptr,NULL,10); + when += mstime(); + + decrRefCount(millisecond); + + robj *newargs[5]; + newargs[0] = argv[0]; + newargs[1] = argv[1]; + newargs[2] = argv[2]; + newargs[3] = shared.pxat; + newargs[4] = createStringObjectFromLongLong(when); + buf = catAppendOnlyGenericCommand(buf,5,newargs); + decrRefCount(newargs[4]); + } else { + buf = catAppendOnlyGenericCommand(buf,argc,argv); } - serverAssert(!(exarg && pxarg)); - if (exarg) - buf = catAppendOnlyExpireAtCommand(buf,server.expireCommand,argv[1], - exarg); - if (pxarg) - buf = catAppendOnlyExpireAtCommand(buf,server.pexpireCommand,argv[1], - pxarg); } else { /* All the other commands don't need translation or need the * same translation already operated in the command vector @@ -637,7 +672,7 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a * accumulate the differences between the child DB and the current one * in a buffer, so that when the child process will do its work we * can append the differences to the new append only file. */ - if (server.aof_child_pid != -1) + if (server.child_type == CHILD_TYPE_AOF) aofRewriteBufferAppend((unsigned char*)buf,sdslen(buf)); sdsfree(buf); @@ -649,18 +684,33 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a /* In Redis commands are always executed in the context of a client, so in * order to load the append only file we need to create a fake client. */ -struct client *createFakeClient(void) { +struct client *createAOFClient(void) { struct client *c = zmalloc(sizeof(*c)); selectDb(c,0); - c->fd = -1; + c->id = CLIENT_ID_AOF; /* So modules can identify it's the AOF client. */ + c->conn = NULL; c->name = NULL; c->querybuf = sdsempty(); c->querybuf_peak = 0; c->argc = 0; c->argv = NULL; + c->original_argc = 0; + c->original_argv = NULL; + c->argv_len_sum = 0; c->bufpos = 0; - c->flags = 0; + + /* + * The AOF client should never be blocked (unlike master + * replication connection). + * This is because blocking the AOF client might cause + * deadlock (because potentially no one will unblock it). + * Also, if the AOF client will be blocked just for + * background processing there is a chance that the + * command execution order will be violated. + */ + c->flags = CLIENT_DENY_BLOCKING; + c->btype = BLOCKED_NONE; /* We set the fake client as a slave waiting for the synchronization * so that Redis will not try to send replies to this client. */ @@ -670,6 +720,9 @@ struct client *createFakeClient(void) { c->obuf_soft_limit_reached_time = 0; c->watched_keys = listCreate(); c->peerid = NULL; + c->sockname = NULL; + c->resp = 2; + c->user = NULL; listSetFreeMethod(c->reply,freeClientReplyValue); listSetDupMethod(c->reply,dupClientReplyValue); initClientMultiState(c); @@ -682,6 +735,7 @@ void freeFakeClientArgv(struct client *c) { for (j = 0; j < c->argc; j++) decrRefCount(c->argv[j]); zfree(c->argv); + c->argv_len_sum = 0; } void freeFakeClient(struct client *c) { @@ -689,6 +743,7 @@ void freeFakeClient(struct client *c) { listRelease(c->reply); listRelease(c->watched_keys); freeClientMultiState(c); + freeClientOriginalArgv(c); zfree(c); } @@ -724,8 +779,8 @@ int loadAppendOnlyFile(char *filename) { * to the same file we're about to read. */ server.aof_state = AOF_OFF; - fakeClient = createFakeClient(); - startLoading(fp); + fakeClient = createAOFClient(); + startLoadingFile(fp, filename, RDBFLAGS_AOF_PREAMBLE); /* Check if this AOF file has an RDB preamble. In that case we need to * load the RDB file and later continue loading the AOF tail. */ @@ -740,7 +795,7 @@ int loadAppendOnlyFile(char *filename) { serverLog(LL_NOTICE,"Reading RDB preamble from AOF file..."); if (fseek(fp,0,SEEK_SET) == -1) goto readerr; rioInitWithFile(&rdb,fp); - if (rdbLoadRio(&rdb,NULL,1) != C_OK) { + if (rdbLoadRio(&rdb,RDBFLAGS_AOF_PREAMBLE,NULL) != C_OK) { serverLog(LL_WARNING,"Error reading the RDB preamble of the AOF file, AOF loading aborted"); goto readerr; } else { @@ -761,6 +816,7 @@ int loadAppendOnlyFile(char *filename) { if (!(loops++ % 1000)) { loadingProgress(ftello(fp)); processEventsWhileBlocked(); + processModuleLoadingProgressEvent(1); } if (fgets(buf,sizeof(buf),fp) == NULL) { @@ -774,18 +830,26 @@ int loadAppendOnlyFile(char *filename) { argc = atoi(buf+1); if (argc < 1) goto fmterr; + /* Load the next command in the AOF as our fake client + * argv. */ argv = zmalloc(sizeof(robj*)*argc); fakeClient->argc = argc; fakeClient->argv = argv; for (j = 0; j < argc; j++) { - if (fgets(buf,sizeof(buf),fp) == NULL) { + /* Parse the argument len. */ + char *readres = fgets(buf,sizeof(buf),fp); + if (readres == NULL || buf[0] != '$') { fakeClient->argc = j; /* Free up to j-1. */ freeFakeClientArgv(fakeClient); - goto readerr; + if (readres == NULL) + goto readerr; + else + goto fmterr; } - if (buf[0] != '$') goto fmterr; len = strtol(buf+1,NULL,10); + + /* Read it into a string object. */ argsds = sdsnewlen(SDS_NOINIT,len); if (len && fread(argsds,len,1,fp) == 0) { sdsfree(argsds); @@ -794,10 +858,12 @@ int loadAppendOnlyFile(char *filename) { goto readerr; } argv[j] = createObject(OBJ_STRING,argsds); + + /* Discard CRLF. */ if (fread(buf,2,1,fp) == 0) { fakeClient->argc = j+1; /* Free up to j. */ freeFakeClientArgv(fakeClient); - goto readerr; /* discard CRLF */ + goto readerr; } } @@ -813,7 +879,7 @@ int loadAppendOnlyFile(char *filename) { if (cmd == server.multiCommand) valid_before_multi = valid_up_to; /* Run the command in the context of a fake client */ - fakeClient->cmd = cmd; + fakeClient->cmd = fakeClient->lastcmd = cmd; if (fakeClient->flags & CLIENT_MULTI && fakeClient->cmd->proc != execCommand) { @@ -834,6 +900,8 @@ int loadAppendOnlyFile(char *filename) { freeFakeClientArgv(fakeClient); fakeClient->cmd = NULL; if (server.aof_load_truncated) valid_up_to = ftello(fp); + if (server.key_load_delay) + debugDelay(server.key_load_delay); } /* This point can only be reached when EOF is reached without errors. @@ -851,7 +919,7 @@ int loadAppendOnlyFile(char *filename) { fclose(fp); freeFakeClient(fakeClient); server.aof_state = old_aof_state; - stopLoading(); + stopLoading(1); aofUpdateCurrentSize(); server.aof_rewrite_base_size = server.aof_current_size; server.aof_fsync_offset = server.aof_current_size; @@ -860,6 +928,7 @@ int loadAppendOnlyFile(char *filename) { readerr: /* Read error. If feof(fp) is true, fall through to unexpected EOF. */ if (!feof(fp)) { if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ + fclose(fp); serverLog(LL_WARNING,"Unrecoverable error reading the append only file: %s", strerror(errno)); exit(1); } @@ -890,11 +959,13 @@ int loadAppendOnlyFile(char *filename) { } } if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ + fclose(fp); serverLog(LL_WARNING,"Unexpected end of file reading the append only file. You can: 1) Make a backup of your AOF file, then use ./redis-check-aof --fix . 2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server."); exit(1); fmterr: /* Format error. */ if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ + fclose(fp); serverLog(LL_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix "); exit(1); } @@ -931,15 +1002,25 @@ int rewriteListObject(rio *r, robj *key, robj *o) { if (count == 0) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0; - if (rioWriteBulkString(r,"RPUSH",5) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items) || + !rioWriteBulkString(r,"RPUSH",5) || + !rioWriteBulkObject(r,key)) + { + quicklistReleaseIterator(li); + return 0; + } } if (entry.value) { - if (rioWriteBulkString(r,(char*)entry.value,entry.sz) == 0) return 0; + if (!rioWriteBulkString(r,(char*)entry.value,entry.sz)) { + quicklistReleaseIterator(li); + return 0; + } } else { - if (rioWriteBulkLongLong(r,entry.longval) == 0) return 0; + if (!rioWriteBulkLongLong(r,entry.longval)) { + quicklistReleaseIterator(li); + return 0; + } } if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; @@ -965,11 +1046,14 @@ int rewriteSetObject(rio *r, robj *key, robj *o) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0; - if (rioWriteBulkString(r,"SADD",4) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items) || + !rioWriteBulkString(r,"SADD",4) || + !rioWriteBulkObject(r,key)) + { + return 0; + } } - if (rioWriteBulkLongLong(r,llval) == 0) return 0; + if (!rioWriteBulkLongLong(r,llval)) return 0; if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } @@ -983,11 +1067,18 @@ int rewriteSetObject(rio *r, robj *key, robj *o) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0; - if (rioWriteBulkString(r,"SADD",4) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items) || + !rioWriteBulkString(r,"SADD",4) || + !rioWriteBulkObject(r,key)) + { + dictReleaseIterator(di); + return 0; + } + } + if (!rioWriteBulkString(r,ele,sdslen(ele))) { + dictReleaseIterator(di); + return 0; } - if (rioWriteBulkString(r,ele,sdslen(ele)) == 0) return 0; if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } @@ -1024,15 +1115,18 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0; - if (rioWriteBulkString(r,"ZADD",4) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items*2) || + !rioWriteBulkString(r,"ZADD",4) || + !rioWriteBulkObject(r,key)) + { + return 0; + } } - if (rioWriteBulkDouble(r,score) == 0) return 0; + if (!rioWriteBulkDouble(r,score)) return 0; if (vstr != NULL) { - if (rioWriteBulkString(r,(char*)vstr,vlen) == 0) return 0; + if (!rioWriteBulkString(r,(char*)vstr,vlen)) return 0; } else { - if (rioWriteBulkLongLong(r,vll) == 0) return 0; + if (!rioWriteBulkLongLong(r,vll)) return 0; } zzlNext(zl,&eptr,&sptr); if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; @@ -1051,12 +1145,20 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0; - if (rioWriteBulkString(r,"ZADD",4) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items*2) || + !rioWriteBulkString(r,"ZADD",4) || + !rioWriteBulkObject(r,key)) + { + dictReleaseIterator(di); + return 0; + } + } + if (!rioWriteBulkDouble(r,*score) || + !rioWriteBulkString(r,ele,sdslen(ele))) + { + dictReleaseIterator(di); + return 0; } - if (rioWriteBulkDouble(r,*score) == 0) return 0; - if (rioWriteBulkString(r,ele,sdslen(ele)) == 0) return 0; if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } @@ -1105,13 +1207,21 @@ int rewriteHashObject(rio *r, robj *key, robj *o) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0; - if (rioWriteBulkString(r,"HMSET",5) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; + if (!rioWriteBulkCount(r,'*',2+cmd_items*2) || + !rioWriteBulkString(r,"HMSET",5) || + !rioWriteBulkObject(r,key)) + { + hashTypeReleaseIterator(hi); + return 0; + } } - if (rioWriteHashIteratorCursor(r, hi, OBJ_HASH_KEY) == 0) return 0; - if (rioWriteHashIteratorCursor(r, hi, OBJ_HASH_VALUE) == 0) return 0; + if (!rioWriteHashIteratorCursor(r, hi, OBJ_HASH_KEY) || + !rioWriteHashIteratorCursor(r, hi, OBJ_HASH_VALUE)) + { + hashTypeReleaseIterator(hi); + return 0; + } if (++count == AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } @@ -1127,7 +1237,7 @@ int rioWriteBulkStreamID(rio *r,streamID *id) { int retval; sds replyid = sdscatfmt(sdsempty(),"%U-%U",id->ms,id->seq); - if ((retval = rioWriteBulkString(r,replyid,sdslen(replyid))) == 0) return 0; + retval = rioWriteBulkString(r,replyid,sdslen(replyid)); sdsfree(replyid); return retval; } @@ -1157,6 +1267,20 @@ int rioWriteStreamPendingEntry(rio *r, robj *key, const char *groupname, size_t return 1; } +/* Helper for rewriteStreamObject(): emit the XGROUP CREATECONSUMER is + * needed in order to create consumers that do not have any pending entries. + * All this in the context of the specified key and group. */ +int rioWriteStreamEmptyConsumer(rio *r, robj *key, const char *groupname, size_t groupname_len, streamConsumer *consumer) { + /* XGROUP CREATECONSUMER */ + if (rioWriteBulkCount(r,'*',5) == 0) return 0; + if (rioWriteBulkString(r,"XGROUP",6) == 0) return 0; + if (rioWriteBulkString(r,"CREATECONSUMER",14) == 0) return 0; + if (rioWriteBulkObject(r,key) == 0) return 0; + if (rioWriteBulkString(r,groupname,groupname_len) == 0) return 0; + if (rioWriteBulkString(r,consumer->name,sdslen(consumer->name)) == 0) return 0; + return 1; +} + /* Emit the commands needed to rebuild a stream object. * The function returns 0 on error, 1 on success. */ int rewriteStreamObject(rio *r, robj *key, robj *o) { @@ -1173,38 +1297,55 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) { * the ID, the second is an array of field-value pairs. */ /* Emit the XADD ...fields... command. */ - if (rioWriteBulkCount(r,'*',3+numfields*2) == 0) return 0; - if (rioWriteBulkString(r,"XADD",4) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; - if (rioWriteBulkStreamID(r,&id) == 0) return 0; + if (!rioWriteBulkCount(r,'*',3+numfields*2) || + !rioWriteBulkString(r,"XADD",4) || + !rioWriteBulkObject(r,key) || + !rioWriteBulkStreamID(r,&id)) + { + streamIteratorStop(&si); + return 0; + } while(numfields--) { unsigned char *field, *value; int64_t field_len, value_len; streamIteratorGetField(&si,&field,&value,&field_len,&value_len); - if (rioWriteBulkString(r,(char*)field,field_len) == 0) return 0; - if (rioWriteBulkString(r,(char*)value,value_len) == 0) return 0; + if (!rioWriteBulkString(r,(char*)field,field_len) || + !rioWriteBulkString(r,(char*)value,value_len)) + { + streamIteratorStop(&si); + return 0; + } } } } else { /* Use the XADD MAXLEN 0 trick to generate an empty stream if * the key we are serializing is an empty string, which is possible * for the Stream type. */ - if (rioWriteBulkCount(r,'*',7) == 0) return 0; - if (rioWriteBulkString(r,"XADD",4) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; - if (rioWriteBulkString(r,"MAXLEN",6) == 0) return 0; - if (rioWriteBulkString(r,"0",1) == 0) return 0; - if (rioWriteBulkStreamID(r,&s->last_id) == 0) return 0; - if (rioWriteBulkString(r,"x",1) == 0) return 0; - if (rioWriteBulkString(r,"y",1) == 0) return 0; + id.ms = 0; id.seq = 1; + if (!rioWriteBulkCount(r,'*',7) || + !rioWriteBulkString(r,"XADD",4) || + !rioWriteBulkObject(r,key) || + !rioWriteBulkString(r,"MAXLEN",6) || + !rioWriteBulkString(r,"0",1) || + !rioWriteBulkStreamID(r,&id) || + !rioWriteBulkString(r,"x",1) || + !rioWriteBulkString(r,"y",1)) + { + streamIteratorStop(&si); + return 0; + } } /* Append XSETID after XADD, make sure lastid is correct, * in case of XDEL lastid. */ - if (rioWriteBulkCount(r,'*',3) == 0) return 0; - if (rioWriteBulkString(r,"XSETID",6) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; - if (rioWriteBulkStreamID(r,&s->last_id) == 0) return 0; + if (!rioWriteBulkCount(r,'*',3) || + !rioWriteBulkString(r,"XSETID",6) || + !rioWriteBulkObject(r,key) || + !rioWriteBulkStreamID(r,&s->last_id)) + { + streamIteratorStop(&si); + return 0; + } /* Create all the stream consumer groups. */ @@ -1215,21 +1356,38 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) { while(raxNext(&ri)) { streamCG *group = ri.data; /* Emit the XGROUP CREATE in order to create the group. */ - if (rioWriteBulkCount(r,'*',5) == 0) return 0; - if (rioWriteBulkString(r,"XGROUP",6) == 0) return 0; - if (rioWriteBulkString(r,"CREATE",6) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; - if (rioWriteBulkString(r,(char*)ri.key,ri.key_len) == 0) return 0; - if (rioWriteBulkStreamID(r,&group->last_id) == 0) return 0; + if (!rioWriteBulkCount(r,'*',5) || + !rioWriteBulkString(r,"XGROUP",6) || + !rioWriteBulkString(r,"CREATE",6) || + !rioWriteBulkObject(r,key) || + !rioWriteBulkString(r,(char*)ri.key,ri.key_len) || + !rioWriteBulkStreamID(r,&group->last_id)) + { + raxStop(&ri); + streamIteratorStop(&si); + return 0; + } /* Generate XCLAIMs for each consumer that happens to - * have pending entries. Empty consumers have no semantical - * value so they are discarded. */ + * have pending entries. Empty consumers would be generated with + * XGROUP CREATECONSUMER. */ raxIterator ri_cons; raxStart(&ri_cons,group->consumers); raxSeek(&ri_cons,"^",NULL,0); while(raxNext(&ri_cons)) { streamConsumer *consumer = ri_cons.data; + /* If there are no pending entries, just emit XGROUP CREATECONSUMER */ + if (raxSize(consumer->pel) == 0) { + if (rioWriteStreamEmptyConsumer(r,key,(char*)ri.key, + ri.key_len,consumer) == 0) + { + raxStop(&ri_cons); + raxStop(&ri); + streamIteratorStop(&si); + return 0; + } + continue; + } /* For the current consumer, iterate all the PEL entries * to emit the XCLAIM protocol. */ raxIterator ri_pel; @@ -1241,6 +1399,10 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) { ri.key_len,consumer, ri_pel.key,nack) == 0) { + raxStop(&ri_pel); + raxStop(&ri_cons); + raxStop(&ri); + streamIteratorStop(&si); return 0; } } @@ -1291,6 +1453,8 @@ int rewriteAppendOnlyFileRio(rio *aof) { dictEntry *de; size_t processed = 0; int j; + long key_count = 0; + long long updated_time = 0; for (j = 0; j < server.dbnum; j++) { char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n"; @@ -1350,6 +1514,17 @@ int rewriteAppendOnlyFileRio(rio *aof) { processed = aof->processed_bytes; aofReadDiffFromParent(); } + + /* Update info every 1 second (approximately). + * in order to avoid calling mstime() on each iteration, we will + * check the diff every 1024 keys */ + if ((key_count++ & 1023) == 0) { + long long now = mstime(); + if (now - updated_time >= 1000) { + sendChildInfo(CHILD_INFO_TYPE_CURRENT_INFO, key_count, "AOF rewrite"); + updated_time = now; + } + } } dictReleaseIterator(di); di = NULL; @@ -1370,7 +1545,7 @@ int rewriteAppendOnlyFileRio(rio *aof) { * are inserted using a single command. */ int rewriteAppendOnlyFile(char *filename) { rio aof; - FILE *fp; + FILE *fp = NULL; char tmpfile[256]; char byte; @@ -1389,9 +1564,11 @@ int rewriteAppendOnlyFile(char *filename) { if (server.aof_rewrite_incremental_fsync) rioSetAutoSync(&aof,REDIS_AUTOSYNC_BYTES); + startSaving(RDBFLAGS_AOF_PREAMBLE); + if (server.aof_use_rdb_preamble) { int error; - if (rdbSaveRio(&aof,&error,RDB_SAVE_AOF_PREAMBLE,NULL) == C_ERR) { + if (rdbSaveRio(&aof,&error,RDBFLAGS_AOF_PREAMBLE,NULL) == C_ERR) { errno = error; goto werr; } @@ -1427,7 +1604,7 @@ int rewriteAppendOnlyFile(char *filename) { if (write(server.aof_pipe_write_ack_to_parent,"!",1) != 1) goto werr; if (anetNonBlock(NULL,server.aof_pipe_read_ack_from_parent) != ANET_OK) goto werr; - /* We read the ACK from the server using a 10 seconds timeout. Normally + /* We read the ACK from the server using a 5 seconds timeout. Normally * it should reply ASAP, but just in case we lose its reply, we are sure * the child will eventually get terminated. */ if (syncRead(server.aof_pipe_read_ack_from_parent,&byte,1,5000) != 1 || @@ -1441,28 +1618,55 @@ int rewriteAppendOnlyFile(char *filename) { serverLog(LL_NOTICE, "Concatenating %.2f MB of AOF diff received from parent.", (double) sdslen(server.aof_child_diff) / (1024*1024)); - if (rioWrite(&aof,server.aof_child_diff,sdslen(server.aof_child_diff)) == 0) - goto werr; + + /* Now we write the entire AOF buffer we received from the parent + * via the pipe during the life of this fork child. + * once a second, we'll take a break and send updated COW info to the parent */ + size_t bytes_to_write = sdslen(server.aof_child_diff); + const char *buf = server.aof_child_diff; + long long cow_updated_time = mstime(); + long long key_count = dbTotalServerKeyCount(); + while (bytes_to_write) { + /* We write the AOF buffer in chunk of 8MB so that we can check the time in between them */ + size_t chunk_size = bytes_to_write < (8<<20) ? bytes_to_write : (8<<20); + + if (rioWrite(&aof,buf,chunk_size) == 0) + goto werr; + + bytes_to_write -= chunk_size; + buf += chunk_size; + + /* Update COW info */ + long long now = mstime(); + if (now - cow_updated_time >= 1000) { + sendChildInfo(CHILD_INFO_TYPE_CURRENT_INFO, key_count, "AOF rewrite"); + cow_updated_time = now; + } + } /* Make sure data will not remain on the OS's output buffers */ - if (fflush(fp) == EOF) goto werr; - if (fsync(fileno(fp)) == -1) goto werr; - if (fclose(fp) == EOF) goto werr; + if (fflush(fp)) goto werr; + if (fsync(fileno(fp))) goto werr; + if (fclose(fp)) { fp = NULL; goto werr; } + fp = NULL; /* Use RENAME to make sure the DB file is changed atomically only * if the generate DB file is ok. */ if (rename(tmpfile,filename) == -1) { serverLog(LL_WARNING,"Error moving temp append only file on the final destination: %s", strerror(errno)); unlink(tmpfile); + stopSaving(0); return C_ERR; } serverLog(LL_NOTICE,"SYNC append only file rewrite performed"); + stopSaving(1); return C_OK; werr: serverLog(LL_WARNING,"Write error writing append only file on disk: %s", strerror(errno)); - fclose(fp); + if (fp) fclose(fp); unlink(tmpfile); + stopSaving(0); return C_ERR; } @@ -1558,41 +1762,25 @@ void aofClosePipes(void) { */ int rewriteAppendOnlyFileBackground(void) { pid_t childpid; - long long start; - if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) return C_ERR; + if (hasActiveChildProcess()) return C_ERR; if (aofCreatePipes() != C_OK) return C_ERR; - openChildInfoPipe(); - start = ustime(); - if ((childpid = fork()) == 0) { + if ((childpid = redisFork(CHILD_TYPE_AOF)) == 0) { char tmpfile[256]; /* Child */ - closeListeningSockets(0); redisSetProcTitle("redis-aof-rewrite"); + redisSetCpuAffinity(server.aof_rewrite_cpulist); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid()); if (rewriteAppendOnlyFile(tmpfile) == C_OK) { - size_t private_dirty = zmalloc_get_private_dirty(-1); - - if (private_dirty) { - serverLog(LL_NOTICE, - "AOF rewrite: %zu MB of memory used by copy-on-write", - private_dirty/(1024*1024)); - } - - server.child_info_data.cow_size = private_dirty; - sendChildInfo(CHILD_INFO_TYPE_AOF); + sendChildCowInfo(CHILD_INFO_TYPE_AOF_COW_SIZE, "AOF rewrite"); exitFromChild(0); } else { exitFromChild(1); } } else { /* Parent */ - server.stat_fork_time = ustime()-start; - server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ - latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); if (childpid == -1) { - closeChildInfoPipe(); serverLog(LL_WARNING, "Can't rewrite append only file in background: fork: %s", strerror(errno)); @@ -1600,11 +1788,10 @@ int rewriteAppendOnlyFileBackground(void) { return C_ERR; } serverLog(LL_NOTICE, - "Background append only file rewriting started by pid %d",childpid); + "Background append only file rewriting started by pid %ld",(long) childpid); server.aof_rewrite_scheduled = 0; server.aof_rewrite_time_start = time(NULL); - server.aof_child_pid = childpid; - updateDictResizePolicy(); + /* We set appendseldb to -1 in order to force the next call to the * feedAppendOnlyFile() to issue a SELECT command, so the differences * accumulated by the parent into server.aof_rewrite_buf will start @@ -1617,15 +1804,16 @@ int rewriteAppendOnlyFileBackground(void) { } void bgrewriteaofCommand(client *c) { - if (server.aof_child_pid != -1) { + if (server.child_type == CHILD_TYPE_AOF) { addReplyError(c,"Background append only file rewriting already in progress"); - } else if (server.rdb_child_pid != -1) { + } else if (hasActiveChildProcess()) { server.aof_rewrite_scheduled = 1; addReplyStatus(c,"Background append only file rewriting scheduled"); } else if (rewriteAppendOnlyFileBackground() == C_OK) { addReplyStatus(c,"Background append only file rewriting started"); } else { - addReply(c,shared.err); + addReplyError(c,"Can't execute an AOF background rewriting. " + "Please check the server logs for more information."); } } @@ -1633,7 +1821,10 @@ void aofRemoveTempFile(pid_t childpid) { char tmpfile[256]; snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) childpid); - unlink(tmpfile); + bg_unlink(tmpfile); + + snprintf(tmpfile,256,"temp-rewriteaof-%d.aof", (int) childpid); + bg_unlink(tmpfile); } /* Update the server.aof_current_size field explicitly using stat(2) @@ -1671,7 +1862,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { * rewritten AOF. */ latencyStartMonitor(latency); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", - (int)server.aof_child_pid); + (int)server.child_pid); newfd = open(tmpfile,O_WRONLY|O_APPEND); if (newfd == -1) { serverLog(LL_WARNING, @@ -1687,6 +1878,20 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { } latencyEndMonitor(latency); latencyAddSampleIfNeeded("aof-rewrite-diff-write",latency); + + if (server.aof_fsync == AOF_FSYNC_EVERYSEC) { + aof_background_fsync(newfd); + } else if (server.aof_fsync == AOF_FSYNC_ALWAYS) { + latencyStartMonitor(latency); + if (redis_fsync(newfd) == -1) { + serverLog(LL_WARNING, + "Error trying to fsync the parent diff to the rewritten AOF: %s", strerror(errno)); + close(newfd); + goto cleanup; + } + latencyEndMonitor(latency); + latencyAddSampleIfNeeded("aof-rewrite-done-fsync",latency); + } serverLog(LL_NOTICE, "Residual parent diff successfully flushed to the rewritten AOF (%.2f MB)", (double) aofRewriteBufferSize() / (1024*1024)); @@ -1754,14 +1959,11 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { /* AOF enabled, replace the old fd with the new one. */ oldfd = server.aof_fd; server.aof_fd = newfd; - if (server.aof_fsync == AOF_FSYNC_ALWAYS) - redis_fsync(newfd); - else if (server.aof_fsync == AOF_FSYNC_EVERYSEC) - aof_background_fsync(newfd); server.aof_selected_db = -1; /* Make sure SELECT is re-issued */ aofUpdateCurrentSize(); server.aof_rewrite_base_size = server.aof_current_size; - server.aof_current_size = server.aof_current_size; + server.aof_fsync_offset = server.aof_current_size; + server.aof_last_fsync = server.unixtime; /* Clear regular AOF buffer since its contents was just written to * the new AOF from the background rewrite buffer. */ @@ -1777,19 +1979,20 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { server.aof_state = AOF_ON; /* Asynchronously close the overwritten AOF. */ - if (oldfd != -1) bioCreateBackgroundJob(BIO_CLOSE_FILE,(void*)(long)oldfd,NULL,NULL); + if (oldfd != -1) bioCreateCloseJob(oldfd); serverLog(LL_VERBOSE, "Background AOF rewrite signal handler took %lldus", ustime()-now); } else if (!bysignal && exitcode != 0) { - /* SIGUSR1 is whitelisted, so we have a way to kill a child without - * tirggering an error condition. */ - if (bysignal != SIGUSR1) - server.aof_lastbgrewrite_status = C_ERR; + server.aof_lastbgrewrite_status = C_ERR; + serverLog(LL_WARNING, "Background AOF rewrite terminated with error"); } else { - server.aof_lastbgrewrite_status = C_ERR; + /* SIGUSR1 is whitelisted, so we have a way to kill a child without + * triggering an error condition. */ + if (bysignal != SIGUSR1) + server.aof_lastbgrewrite_status = C_ERR; serverLog(LL_WARNING, "Background AOF rewrite terminated by signal %d", bysignal); @@ -1798,8 +2001,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { cleanup: aofClosePipes(); aofRewriteBufferReset(); - aofRemoveTempFile(server.aof_child_pid); - server.aof_child_pid = -1; + aofRemoveTempFile(server.child_pid); server.aof_rewrite_time_last = time(NULL)-server.aof_rewrite_time_start; server.aof_rewrite_time_start = -1; /* Schedule a new rewrite if we are waiting for it to switch the AOF ON. */ diff --git a/redis.submodule/src/asciilogo.h b/redis.submodule/src/asciilogo.h index 83c538b..a62f68c 100644 --- a/redis.submodule/src/asciilogo.h +++ b/redis.submodule/src/asciilogo.h @@ -27,17 +27,17 @@ * POSSIBILITY OF SUCH DAMAGE. */ -char *ascii_logo = +const char *ascii_logo = " _._ \n" " _.-``__ ''-._ \n" " _.-`` `. `_. ''-._ Redis %s (%s/%d) %s bit\n" -" .-`` .-```. ```\\/ _.,_ ''-._ \n" +" .-`` .-```. ```\\/ _.,_ ''-._ \n" " ( ' , .-` | `, ) Running in %s mode\n" " |`-._`-...-` __...-.``-._|'` _.-'| Port: %d\n" " | `-._ `._ / _.-' | PID: %ld\n" " `-._ `-._ `-./ _.-' _.-' \n" " |`-._`-._ `-.__.-' _.-'_.-'| \n" -" | `-._`-._ _.-'_.-' | http://redis.io \n" +" | `-._`-._ _.-'_.-' | https://redis.io \n" " `-._ `-._`-.__.-'_.-' _.-' \n" " |`-._`-._ `-.__.-' _.-'_.-'| \n" " | `-._`-._ _.-'_.-' | \n" diff --git a/redis.submodule/src/atomicvar.h b/redis.submodule/src/atomicvar.h index 160056c..222b826 100644 --- a/redis.submodule/src/atomicvar.h +++ b/redis.submodule/src/atomicvar.h @@ -1,5 +1,5 @@ -/* This file implements atomic counters using __atomic or __sync macros if - * available, otherwise synchronizing different threads using a mutex. +/* This file implements atomic counters using c11 _Atomic, __atomic or __sync + * macros if available, otherwise we will throw an error when compile. * * The exported interface is composed of three macros: * @@ -8,20 +8,12 @@ * atomicDecr(var,count) -- Decrement the atomic counter * atomicGet(var,dstvar) -- Fetch the atomic counter value * atomicSet(var,value) -- Set the atomic counter value - * - * The variable 'var' should also have a declared mutex with the same - * name and the "_mutex" postfix, for instance: - * - * long myvar; - * pthread_mutex_t myvar_mutex; - * atomicSet(myvar,12345); - * - * If atomic primitives are available (tested in config.h) the mutex - * is not used. + * atomicGetWithSync(var,value) -- 'atomicGet' with inter-thread synchronization + * atomicSetWithSync(var,value) -- 'atomicSet' with inter-thread synchronization * * Never use return value from the macros, instead use the AtomicGetIncr() * if you need to get the current value and increment it atomically, like - * in the followign example: + * in the following example: * * long oldvalue; * atomicGetIncr(myvar,oldvalue,1); @@ -58,17 +50,64 @@ */ #include +#include "config.h" #ifndef __ATOMIC_VAR_H #define __ATOMIC_VAR_H +/* Define redisAtomic for atomic variable. */ +#define redisAtomic + /* To test Redis with Helgrind (a Valgrind tool) it is useful to define * the following macro, so that __sync macros are used: those can be detected * by Helgrind (even if they are less efficient) so that no false positive * is reported. */ // #define __ATOMIC_VAR_FORCE_SYNC_MACROS -#if !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && defined(__ATOMIC_RELAXED) && !defined(__sun) && (!defined(__clang__) || !defined(__APPLE__) || __apple_build_version__ > 4210057) +/* There will be many false positives if we test Redis with Helgrind, since + * Helgrind can't understand we have imposed ordering on the program, so + * we use macros in helgrind.h to tell Helgrind inter-thread happens-before + * relationship explicitly for avoiding false positives. + * + * For more details, please see: valgrind/helgrind.h and + * https://www.valgrind.org/docs/manual/hg-manual.html#hg-manual.effective-use + * + * These macros take effect only when 'make helgrind', and you must first + * install Valgrind in the default path configuration. */ +#ifdef __ATOMIC_VAR_FORCE_SYNC_MACROS +#include +#else +#define ANNOTATE_HAPPENS_BEFORE(v) ((void) v) +#define ANNOTATE_HAPPENS_AFTER(v) ((void) v) +#endif + +#if !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && defined(__STDC_VERSION__) && \ + (__STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_ATOMICS__) +/* Use '_Atomic' keyword if the compiler supports. */ +#undef redisAtomic +#define redisAtomic _Atomic +/* Implementation using _Atomic in C11. */ + +#include +#define atomicIncr(var,count) atomic_fetch_add_explicit(&var,(count),memory_order_relaxed) +#define atomicGetIncr(var,oldvalue_var,count) do { \ + oldvalue_var = atomic_fetch_add_explicit(&var,(count),memory_order_relaxed); \ +} while(0) +#define atomicDecr(var,count) atomic_fetch_sub_explicit(&var,(count),memory_order_relaxed) +#define atomicGet(var,dstvar) do { \ + dstvar = atomic_load_explicit(&var,memory_order_relaxed); \ +} while(0) +#define atomicSet(var,value) atomic_store_explicit(&var,value,memory_order_relaxed) +#define atomicGetWithSync(var,dstvar) do { \ + dstvar = atomic_load_explicit(&var,memory_order_seq_cst); \ +} while(0) +#define atomicSetWithSync(var,value) \ + atomic_store_explicit(&var,value,memory_order_seq_cst) +#define REDIS_ATOMIC_API "c11-builtin" + +#elif !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && \ + (!defined(__clang__) || !defined(__APPLE__) || __apple_build_version__ > 4210057) && \ + defined(__ATOMIC_RELAXED) && defined(__ATOMIC_SEQ_CST) /* Implementation using __atomic macros. */ #define atomicIncr(var,count) __atomic_add_fetch(&var,(count),__ATOMIC_RELAXED) @@ -80,6 +119,11 @@ dstvar = __atomic_load_n(&var,__ATOMIC_RELAXED); \ } while(0) #define atomicSet(var,value) __atomic_store_n(&var,value,__ATOMIC_RELAXED) +#define atomicGetWithSync(var,dstvar) do { \ + dstvar = __atomic_load_n(&var,__ATOMIC_SEQ_CST); \ +} while(0) +#define atomicSetWithSync(var,value) \ + __atomic_store_n(&var,value,__ATOMIC_SEQ_CST) #define REDIS_ATOMIC_API "atomic-builtin" #elif defined(HAVE_ATOMIC) @@ -96,38 +140,19 @@ #define atomicSet(var,value) do { \ while(!__sync_bool_compare_and_swap(&var,var,value)); \ } while(0) +/* Actually the builtin issues a full memory barrier by default. */ +#define atomicGetWithSync(var,dstvar) { \ + dstvar = __sync_sub_and_fetch(&var,0,__sync_synchronize); \ + ANNOTATE_HAPPENS_AFTER(&var); \ +} while(0) +#define atomicSetWithSync(var,value) do { \ + ANNOTATE_HAPPENS_BEFORE(&var); \ + while(!__sync_bool_compare_and_swap(&var,var,value,__sync_synchronize)); \ +} while(0) #define REDIS_ATOMIC_API "sync-builtin" #else -/* Implementation using pthread mutex. */ - -#define atomicIncr(var,count) do { \ - pthread_mutex_lock(&var ## _mutex); \ - var += (count); \ - pthread_mutex_unlock(&var ## _mutex); \ -} while(0) -#define atomicGetIncr(var,oldvalue_var,count) do { \ - pthread_mutex_lock(&var ## _mutex); \ - oldvalue_var = var; \ - var += (count); \ - pthread_mutex_unlock(&var ## _mutex); \ -} while(0) -#define atomicDecr(var,count) do { \ - pthread_mutex_lock(&var ## _mutex); \ - var -= (count); \ - pthread_mutex_unlock(&var ## _mutex); \ -} while(0) -#define atomicGet(var,dstvar) do { \ - pthread_mutex_lock(&var ## _mutex); \ - dstvar = var; \ - pthread_mutex_unlock(&var ## _mutex); \ -} while(0) -#define atomicSet(var,value) do { \ - pthread_mutex_lock(&var ## _mutex); \ - var = value; \ - pthread_mutex_unlock(&var ## _mutex); \ -} while(0) -#define REDIS_ATOMIC_API "pthread-mutex" +#error "Unable to determine atomic operations for your platform" #endif #endif /* __ATOMIC_VAR_H */ diff --git a/redis.submodule/src/bio.c b/redis.submodule/src/bio.c index 2af6845..ff18c4f 100644 --- a/redis.submodule/src/bio.c +++ b/redis.submodule/src/bio.c @@ -78,15 +78,13 @@ static unsigned long long bio_pending[BIO_NUM_OPS]; * file as the API does not expose the internals at all. */ struct bio_job { time_t time; /* Time at which the job was created. */ - /* Job specific arguments pointers. If we need to pass more than three - * arguments we can just pass a pointer to a structure or alike. */ - void *arg1, *arg2, *arg3; + /* Job specific arguments.*/ + int fd; /* Fd for file based background jobs */ + lazy_free_fn *free_fn; /* Function that will free the provided arguments */ + void *free_args[]; /* List of arguments to be passed to the free function */ }; void *bioProcessBackgroundJobs(void *arg); -void lazyfreeFreeObjectFromBioThread(robj *o); -void lazyfreeFreeDatabaseFromBioThread(dict *ht1, dict *ht2); -void lazyfreeFreeSlotsMapFromBioThread(zskiplist *sl); /* Make sure we have enough stack to perform all the things we do in the * main thread. */ @@ -128,13 +126,8 @@ void bioInit(void) { } } -void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3) { - struct bio_job *job = zmalloc(sizeof(*job)); - +void bioSubmitJob(int type, struct bio_job *job) { job->time = time(NULL); - job->arg1 = arg1; - job->arg2 = arg2; - job->arg3 = arg3; pthread_mutex_lock(&bio_mutex[type]); listAddNodeTail(bio_jobs[type],job); bio_pending[type]++; @@ -142,6 +135,35 @@ void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3) { pthread_mutex_unlock(&bio_mutex[type]); } +void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...) { + va_list valist; + /* Allocate memory for the job structure and all required + * arguments */ + struct bio_job *job = zmalloc(sizeof(*job) + sizeof(void *) * (arg_count)); + job->free_fn = free_fn; + + va_start(valist, arg_count); + for (int i = 0; i < arg_count; i++) { + job->free_args[i] = va_arg(valist, void *); + } + va_end(valist); + bioSubmitJob(BIO_LAZY_FREE, job); +} + +void bioCreateCloseJob(int fd) { + struct bio_job *job = zmalloc(sizeof(*job)); + job->fd = fd; + + bioSubmitJob(BIO_CLOSE_FILE, job); +} + +void bioCreateFsyncJob(int fd) { + struct bio_job *job = zmalloc(sizeof(*job)); + job->fd = fd; + + bioSubmitJob(BIO_AOF_FSYNC, job); +} + void *bioProcessBackgroundJobs(void *arg) { struct bio_job *job; unsigned long type = (unsigned long) arg; @@ -154,10 +176,21 @@ void *bioProcessBackgroundJobs(void *arg) { return NULL; } - /* Make the thread killable at any time, so that bioKillThreads() - * can work reliably. */ - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); - pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); + switch (type) { + case BIO_CLOSE_FILE: + redis_set_thread_title("bio_close_file"); + break; + case BIO_AOF_FSYNC: + redis_set_thread_title("bio_aof_fsync"); + break; + case BIO_LAZY_FREE: + redis_set_thread_title("bio_lazy_free"); + break; + } + + redisSetCpuAffinity(server.bio_cpulist); + + makeThreadKillable(); pthread_mutex_lock(&bio_mutex[type]); /* Block SIGALRM so we are sure that only the main thread will @@ -185,20 +218,27 @@ void *bioProcessBackgroundJobs(void *arg) { /* Process the job accordingly to its type. */ if (type == BIO_CLOSE_FILE) { - close((long)job->arg1); + close(job->fd); } else if (type == BIO_AOF_FSYNC) { - redis_fsync((long)job->arg1); + /* The fd may be closed by main thread and reused for another + * socket, pipe, or file. We just ignore these errno because + * aof fsync did not really fail. */ + if (redis_fsync(job->fd) == -1 && + errno != EBADF && errno != EINVAL) + { + int last_status; + atomicGet(server.aof_bio_fsync_status,last_status); + atomicSet(server.aof_bio_fsync_status,C_ERR); + atomicSet(server.aof_bio_fsync_errno,errno); + if (last_status == C_OK) { + serverLog(LL_WARNING, + "Fail to fsync the AOF file: %s",strerror(errno)); + } + } else { + atomicSet(server.aof_bio_fsync_status,C_OK); + } } else if (type == BIO_LAZY_FREE) { - /* What we free changes depending on what arguments are set: - * arg1 -> free the object at pointer. - * arg2 & arg3 -> free two dictionaries (a Redis DB). - * only arg3 -> free the skiplist. */ - if (job->arg1) - lazyfreeFreeObjectFromBioThread(job->arg1); - else if (job->arg2 && job->arg3) - lazyfreeFreeDatabaseFromBioThread(job->arg2,job->arg3); - else if (job->arg3) - lazyfreeFreeSlotsMapFromBioThread(job->arg3); + job->free_fn(job->free_args); } else { serverPanic("Wrong job type in bioProcessBackgroundJobs()."); } @@ -254,10 +294,11 @@ void bioKillThreads(void) { int err, j; for (j = 0; j < BIO_NUM_OPS; j++) { - if (pthread_cancel(bio_threads[j]) == 0) { + if (bio_threads[j] == pthread_self()) continue; + if (bio_threads[j] && pthread_cancel(bio_threads[j]) == 0) { if ((err = pthread_join(bio_threads[j],NULL)) != 0) { serverLog(LL_WARNING, - "Bio thread for job type #%d can be joined: %s", + "Bio thread for job type #%d can not be joined: %s", j, strerror(err)); } else { serverLog(LL_WARNING, diff --git a/redis.submodule/src/bio.h b/redis.submodule/src/bio.h index 4b15d1c..1e6e972 100644 --- a/redis.submodule/src/bio.h +++ b/redis.submodule/src/bio.h @@ -27,16 +27,25 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#ifndef __BIO_H +#define __BIO_H + +typedef void lazy_free_fn(void *args[]); + /* Exported API */ void bioInit(void); -void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3); unsigned long long bioPendingJobsOfType(int type); unsigned long long bioWaitStepOfType(int type); time_t bioOlderJobOfType(int type); void bioKillThreads(void); +void bioCreateCloseJob(int fd); +void bioCreateFsyncJob(int fd); +void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...); /* Background job opcodes */ #define BIO_CLOSE_FILE 0 /* Deferred close(2) syscall. */ #define BIO_AOF_FSYNC 1 /* Deferred AOF fsync. */ #define BIO_LAZY_FREE 2 /* Deferred objects freeing. */ #define BIO_NUM_OPS 3 + +#endif diff --git a/redis.submodule/src/bitops.c b/redis.submodule/src/bitops.c index 7328d63..f1c563a 100644 --- a/redis.submodule/src/bitops.c +++ b/redis.submodule/src/bitops.c @@ -36,9 +36,9 @@ /* Count number of bits set in the binary array pointed by 's' and long * 'count' bytes. The implementation of this function is required to - * work with a input string length up to 512 MB. */ -size_t redisPopcount(void *s, long count) { - size_t bits = 0; + * work with an input string length up to 512 MB or more (server.proto_max_bulk_len) */ +long long redisPopcount(void *s, long count) { + long long bits = 0; unsigned char *p = s; uint32_t *p4; static const unsigned char bitsinbyte[256] = {0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,4,5,5,6,5,6,6,7,5,6,6,7,6,7,7,8}; @@ -98,16 +98,16 @@ size_t redisPopcount(void *s, long count) { * no zero bit is found, it returns count*8 assuming the string is zero * padded on the right. However if 'bit' is 1 it is possible that there is * not a single set bit in the bitmap. In this special case -1 is returned. */ -long redisBitpos(void *s, unsigned long count, int bit) { +long long redisBitpos(void *s, unsigned long count, int bit) { unsigned long *l; unsigned char *c; unsigned long skipval, word = 0, one; - long pos = 0; /* Position of bit, to return to the caller. */ + long long pos = 0; /* Position of bit, to return to the caller. */ unsigned long j; int found; /* Process whole words first, seeking for first word that is not - * all ones or all zeros respectively if we are lookig for zeros + * all ones or all zeros respectively if we are looking for zeros * or ones. This is much faster with large strings having contiguous * blocks of 1 or 0 bits compared to the vanilla bit per bit processing. * @@ -257,7 +257,7 @@ int64_t getSignedBitfield(unsigned char *p, uint64_t offset, uint64_t bits) { /* If the top significant bit is 1, propagate it to all the * higher bits for two's complement representation of signed * integers. */ - if (value & ((uint64_t)1 << (bits-1))) + if (bits < 64 && (value & ((uint64_t)1 << (bits-1)))) value |= ((uint64_t)-1) << bits; return value; } @@ -269,7 +269,7 @@ int64_t getSignedBitfield(unsigned char *p, uint64_t offset, uint64_t bits) { * then zero is returned, otherwise in case of overflow, 1 is returned, * otherwise in case of underflow, -1 is returned. * - * When non-zero is returned (oferflow or underflow), if not NULL, *limit is + * When non-zero is returned (overflow or underflow), if not NULL, *limit is * set to the value the operation should result when an overflow happens, * depending on the specified overflow semantics: * @@ -356,7 +356,6 @@ int checkSignedBitfieldOverflow(int64_t value, int64_t incr, uint64_t bits, int handle_wrap: { - uint64_t mask = ((uint64_t)-1) << bits; uint64_t msb = (uint64_t)1 << (bits-1); uint64_t a = value, b = incr, c; c = a+b; /* Perform addition as unsigned so that's defined. */ @@ -364,10 +363,13 @@ int checkSignedBitfieldOverflow(int64_t value, int64_t incr, uint64_t bits, int /* If the sign bit is set, propagate to all the higher order * bits, to cap the negative value. If it's clear, mask to * the positive integer limit. */ - if (c & msb) { - c |= mask; - } else { - c &= ~mask; + if (bits < 64) { + uint64_t mask = ((uint64_t)-1) << bits; + if (c & msb) { + c |= mask; + } else { + c &= ~mask; + } } *limit = c; } @@ -403,12 +405,12 @@ void printBits(unsigned char *p, unsigned long count) { /* This helper function used by GETBIT / SETBIT parses the bit offset argument * making sure an error is returned if it is negative or if it overflows - * Redis 512 MB limit for the string value. + * Redis 512 MB limit for the string value or more (server.proto_max_bulk_len). * * If the 'hash' argument is true, and 'bits is positive, then the command * will also parse bit offsets prefixed by "#". In such a case the offset * is multiplied by 'bits'. This is useful for the BITFIELD command. */ -int getBitOffsetFromArgument(client *c, robj *o, size_t *offset, int hash, int bits) { +int getBitOffsetFromArgument(client *c, robj *o, uint64_t *offset, int hash, int bits) { long long loffset; char *err = "bit offset is not an integer or out of range"; char *p = o->ptr; @@ -426,14 +428,14 @@ int getBitOffsetFromArgument(client *c, robj *o, size_t *offset, int hash, int b /* Adjust the offset by 'bits' for # form. */ if (usehash) loffset *= bits; - /* Limit offset to 512MB in bytes */ - if ((loffset < 0) || ((unsigned long long)loffset >> 3) >= (512*1024*1024)) + /* Limit offset to server.proto_max_bulk_len (512MB in bytes by default) */ + if ((loffset < 0) || (loffset >> 3) >= server.proto_max_bulk_len) { addReplyError(c,err); return C_ERR; } - *offset = (size_t)loffset; + *offset = loffset; return C_OK; } @@ -475,15 +477,15 @@ int getBitfieldTypeFromArgument(client *c, robj *o, int *sign, int *bits) { * so that the 'maxbit' bit can be addressed. The object is finally * returned. Otherwise if the key holds a wrong type NULL is returned and * an error is sent to the client. */ -robj *lookupStringForBitCommand(client *c, size_t maxbit) { +robj *lookupStringForBitCommand(client *c, uint64_t maxbit) { size_t byte = maxbit >> 3; robj *o = lookupKeyWrite(c->db,c->argv[1]); + if (checkType(c,o,OBJ_STRING)) return NULL; if (o == NULL) { o = createObject(OBJ_STRING,sdsnewlen(NULL, byte+1)); dbAdd(c->db,c->argv[1],o); } else { - if (checkType(c,o,OBJ_STRING)) return NULL; o = dbUnshareStringValue(c->db,c->argv[1],o); o->ptr = sdsgrowzero(o->ptr,byte+1); } @@ -494,7 +496,7 @@ robj *lookupStringForBitCommand(client *c, size_t maxbit) { * in 'len'. The user is required to pass (likely stack allocated) buffer * 'llbuf' of at least LONG_STR_SIZE bytes. Such a buffer is used in the case * the object is integer encoded in order to provide the representation - * without usign heap allocation. + * without using heap allocation. * * The function returns the pointer to the object array of bytes representing * the string it contains, that may be a pointer to 'llbuf' or to the @@ -525,7 +527,7 @@ unsigned char *getObjectReadOnlyString(robj *o, long *len, char *llbuf) { void setbitCommand(client *c) { robj *o; char *err = "bit is not an integer or out of range"; - size_t bitoffset; + uint64_t bitoffset; ssize_t byte, bit; int byteval, bitval; long on; @@ -554,7 +556,7 @@ void setbitCommand(client *c) { byteval &= ~(1 << bit); byteval |= ((on & 0x1) << bit); ((uint8_t*)o->ptr)[byte] = byteval; - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id); server.dirty++; addReply(c, bitval ? shared.cone : shared.czero); @@ -564,7 +566,7 @@ void setbitCommand(client *c) { void getbitCommand(client *c) { robj *o; char llbuf[32]; - size_t bitoffset; + uint64_t bitoffset; size_t byte, bit; size_t bitval = 0; @@ -609,7 +611,7 @@ void bitopCommand(client *c) { else if((opname[0] == 'n' || opname[0] == 'N') && !strcasecmp(opname,"not")) op = BITOP_NOT; else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } @@ -733,12 +735,23 @@ void bitopCommand(client *c) { output = (len[0] <= j) ? 0 : src[0][j]; if (op == BITOP_NOT) output = ~output; for (i = 1; i < numkeys; i++) { + int skip = 0; byte = (len[i] <= j) ? 0 : src[i][j]; switch(op) { - case BITOP_AND: output &= byte; break; - case BITOP_OR: output |= byte; break; + case BITOP_AND: + output &= byte; + skip = (output == 0); + break; + case BITOP_OR: + output |= byte; + skip = (output == 0xff); + break; case BITOP_XOR: output ^= byte; break; } + + if (skip) { + break; + } } res[j] = output; } @@ -754,14 +767,15 @@ void bitopCommand(client *c) { /* Store the computed value into the target key */ if (maxlen) { o = createObject(OBJ_STRING,res); - setKey(c->db,targetkey,o); + setKey(c,c->db,targetkey,o); notifyKeyspaceEvent(NOTIFY_STRING,"set",targetkey,c->db->id); decrRefCount(o); + server.dirty++; } else if (dbDelete(c->db,targetkey)) { - signalModifiedKey(c->db,targetkey); + signalModifiedKey(c,c->db,targetkey); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",targetkey,c->db->id); + server.dirty++; } - server.dirty++; addReplyLongLong(c,maxlen); /* Return the output string length in bytes. */ } @@ -799,7 +813,7 @@ void bitcountCommand(client *c) { end = strlen-1; } else { /* Syntax error. */ - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } @@ -864,7 +878,7 @@ void bitposCommand(client *c) { end = strlen-1; } else { /* Syntax error. */ - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } @@ -874,7 +888,7 @@ void bitposCommand(client *c) { addReplyLongLong(c, -1); } else { long bytes = end-start+1; - long pos = redisBitpos(p+start,bytes,bit); + long long pos = redisBitpos(p+start,bytes,bit); /* If we are looking for clear bits, and the user specified an exact * range with start-end, we can't consider the right of the range as @@ -883,11 +897,11 @@ void bitposCommand(client *c) { * So if redisBitpos() returns the first bit outside the range, * we return -1 to the caller, to mean, in the specified range there * is not a single "0" bit. */ - if (end_given && bit == 0 && pos == bytes*8) { + if (end_given && bit == 0 && pos == (long long)bytes<<3) { addReplyLongLong(c,-1); return; } - if (pos != -1) pos += start*8; /* Adjust for the bytes we skipped. */ + if (pos != -1) pos += (long long)start<<3; /* Adjust for the bytes we skipped. */ addReplyLongLong(c,pos); } } @@ -902,6 +916,9 @@ void bitposCommand(client *c) { * OVERFLOW [WRAP|SAT|FAIL] */ +#define BITFIELD_FLAG_NONE 0 +#define BITFIELD_FLAG_READONLY (1<<0) + struct bitfieldOp { uint64_t offset; /* Bitfield offset. */ int64_t i64; /* Increment amount (INCRBY) or SET value */ @@ -911,14 +928,17 @@ struct bitfieldOp { int sign; /* True if signed, otherwise unsigned op. */ }; -void bitfieldCommand(client *c) { +/* This implements both the BITFIELD command and the BITFIELD_RO command + * when flags is set to BITFIELD_FLAG_READONLY: in this case only the + * GET subcommand is allowed, other subcommands will return an error. */ +void bitfieldGeneric(client *c, int flags) { robj *o; - size_t bitoffset; + uint64_t bitoffset; int j, numops = 0, changes = 0; struct bitfieldOp *ops = NULL; /* Array of ops to execute at end. */ int owtype = BFOVERFLOW_WRAP; /* Overflow type. */ int readonly = 1; - size_t highest_write_offset = 0; + uint64_t highest_write_offset = 0; for (j = 2; j < c->argc; j++) { int remargs = c->argc-j-1; /* Remaining args other than current. */ @@ -950,7 +970,7 @@ void bitfieldCommand(client *c) { } continue; } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); zfree(ops); return; } @@ -999,6 +1019,12 @@ void bitfieldCommand(client *c) { return; } } else { + if (flags & BITFIELD_FLAG_READONLY) { + zfree(ops); + addReplyError(c, "BITFIELD_RO only supports the GET subcommand"); + return; + } + /* Lookup by making room up to the farest bit reached by * this operation. */ if ((o = lookupStringForBitCommand(c, @@ -1008,7 +1034,7 @@ void bitfieldCommand(client *c) { } } - addReplyMultiBulkLen(c,numops); + addReplyArrayLen(c,numops); /* Actually process the operations. */ for (j = 0; j < numops; j++) { @@ -1053,7 +1079,7 @@ void bitfieldCommand(client *c) { setSignedBitfield(o->ptr,thisop->offset, thisop->bits,newval); } else { - addReply(c,shared.nullbulk); + addReplyNull(c); } } else { uint64_t oldval, newval, wrapped, retval; @@ -1082,7 +1108,7 @@ void bitfieldCommand(client *c) { setUnsignedBitfield(o->ptr,thisop->offset, thisop->bits,newval); } else { - addReply(c,shared.nullbulk); + addReplyNull(c); } } changes++; @@ -1102,9 +1128,9 @@ void bitfieldCommand(client *c) { * object boundaries. */ memset(buf,0,9); int i; - size_t byte = thisop->offset >> 3; + uint64_t byte = thisop->offset >> 3; for (i = 0; i < 9; i++) { - if (src == NULL || i+byte >= (size_t)strlen) break; + if (src == NULL || i+byte >= (uint64_t)strlen) break; buf[i] = src[i+byte]; } @@ -1123,9 +1149,17 @@ void bitfieldCommand(client *c) { } if (changes) { - signalModifiedKey(c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id); server.dirty += changes; } zfree(ops); } + +void bitfieldCommand(client *c) { + bitfieldGeneric(c, BITFIELD_FLAG_NONE); +} + +void bitfieldroCommand(client *c) { + bitfieldGeneric(c, BITFIELD_FLAG_READONLY); +} diff --git a/redis.submodule/src/blocked.c b/redis.submodule/src/blocked.c index 2b43f2b..df3b6a9 100644 --- a/redis.submodule/src/blocked.c +++ b/redis.submodule/src/blocked.c @@ -31,9 +31,6 @@ * * API: * - * getTimeoutFromObjectOrReply() is just an utility function to parse a - * timeout argument since blocking operations usually require a timeout. - * * blockClient() set the CLIENT_BLOCKED flag in the client, and set the * specified block type 'btype' filed to one of BLOCKED_* macros. * @@ -56,7 +53,7 @@ * to 0, no timeout is processed). * It usually just needs to send a reply to the client. * - * When implementing a new type of blocking opeation, the implementation + * When implementing a new type of blocking operation, the implementation * should modify unblockClient() and replyToBlockedClientTimedOut() in order * to handle the btype-specific behavior of this two functions. * If the blocking operation waits for certain keys to change state, the @@ -64,46 +61,61 @@ */ #include "server.h" - -int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int where); - -/* Get a timeout value from an object and store it into 'timeout'. - * The final timeout is always stored as milliseconds as a time where the - * timeout will expire, however the parsing is performed according to - * the 'unit' that can be seconds or milliseconds. - * - * Note that if the timeout is zero (usually from the point of view of - * commands API this means no timeout) the value stored into 'timeout' - * is zero. */ -int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int unit) { - long long tval; - - if (getLongLongFromObjectOrReply(c,object,&tval, - "timeout is not an integer or out of range") != C_OK) - return C_ERR; - - if (tval < 0) { - addReplyError(c,"timeout is negative"); - return C_ERR; - } - - if (tval > 0) { - if (unit == UNIT_SECONDS) tval *= 1000; - tval += mstime(); - } - *timeout = tval; - - return C_OK; -} +#include "slowlog.h" +#include "latency.h" +#include "monotonic.h" + +int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int wherefrom, int whereto); +int getListPositionFromObjectOrReply(client *c, robj *arg, int *position); + +/* This structure represents the blocked key information that we store + * in the client structure. Each client blocked on keys, has a + * client->bpop.keys hash table. The keys of the hash table are Redis + * keys pointers to 'robj' structures. The value is this structure. + * The structure has two goals: firstly we store the list node that this + * client uses to be listed in the database "blocked clients for this key" + * list, so we can later unblock in O(1) without a list scan. + * Secondly for certain blocking types, we have additional info. Right now + * the only use for additional info we have is when clients are blocked + * on streams, as we have to remember the ID it blocked for. */ +typedef struct bkinfo { + listNode *listnode; /* List node for db->blocking_keys[key] list. */ + streamID stream_id; /* Stream ID if we blocked in a stream. */ +} bkinfo; /* Block a client for the specific operation type. Once the CLIENT_BLOCKED * flag is set client query buffer is not longer processed, but accumulated, * and will be processed when the client is unblocked. */ void blockClient(client *c, int btype) { + /* Master client should never be blocked unless pause or module */ + serverAssert(!(c->flags & CLIENT_MASTER && + btype != BLOCKED_MODULE && + btype != BLOCKED_PAUSE)); + c->flags |= CLIENT_BLOCKED; c->btype = btype; server.blocked_clients++; server.blocked_clients_by_type[btype]++; + addClientToTimeoutTable(c); + if (btype == BLOCKED_PAUSE) { + listAddNodeTail(server.paused_clients, c); + c->paused_list_node = listLast(server.paused_clients); + /* Mark this client to execute its command */ + c->flags |= CLIENT_PENDING_COMMAND; + } +} + +/* This function is called after a client has finished a blocking operation + * in order to update the total command duration, log the command into + * the Slow log if needed, and log the reply duration event if needed. */ +void updateStatsOnUnblock(client *c, long blocked_us, long reply_us){ + const ustime_t total_cmd_duration = c->duration + blocked_us + reply_us; + c->lastcmd->microseconds += total_cmd_duration; + + /* Log the command into the Slow log if needed. */ + slowlogPushCurrentCommand(c, c->lastcmd, total_cmd_duration); + /* Log the reply duration event. */ + latencyAddSampleIfNeeded("command-unblocking",reply_us/1000); } /* This function is called in the beforeSleep() function of the event loop @@ -125,8 +137,13 @@ void processUnblockedClients(void) { * client is not blocked before to proceed, but things may change and * the code is conceptually more correct this way. */ if (!(c->flags & CLIENT_BLOCKED)) { + /* If we have a queued command, execute it now. */ + if (processPendingCommandsAndResetClient(c) == C_ERR) { + continue; + } + /* Then process client if it has more data in it's buffer. */ if (c->querybuf && sdslen(c->querybuf) > 0) { - processInputBufferAndReplicate(c); + processInputBuffer(c); } } } @@ -134,7 +151,7 @@ void processUnblockedClients(void) { /* This function will schedule the client for reprocessing at a safe time. * - * This is useful when a client was blocked for some reason (blocking opeation, + * This is useful when a client was blocked for some reason (blocking operation, * CLIENT PAUSE, or whatever), because it may end with some accumulated query * buffer that needs to be processed ASAP: * @@ -167,16 +184,31 @@ void unblockClient(client *c) { } else if (c->btype == BLOCKED_WAIT) { unblockClientWaitingReplicas(c); } else if (c->btype == BLOCKED_MODULE) { + if (moduleClientIsBlockedOnKeys(c)) unblockClientWaitingData(c); unblockClientFromModule(c); + } else if (c->btype == BLOCKED_PAUSE) { + listDelNode(server.paused_clients,c->paused_list_node); + c->paused_list_node = NULL; } else { serverPanic("Unknown btype in unblockClient()."); } + + /* Reset the client for a new query since, for blocking commands + * we do not do it immediately after the command returns (when the + * client got blocked) in order to be still able to access the argument + * vector from module callbacks and updateStatsOnUnblock. */ + if (c->btype != BLOCKED_PAUSE) { + freeClientOriginalArgv(c); + resetClient(c); + } + /* Clear the flags, and put the client in the unblocked list so that * we'll process new commands in its query buffer ASAP. */ server.blocked_clients--; server.blocked_clients_by_type[c->btype]--; c->flags &= ~CLIENT_BLOCKED; c->btype = BLOCKED_NONE; + removeClientFromTimeoutTable(c); queueClientForReprocessing(c); } @@ -187,7 +219,7 @@ void replyToBlockedClientTimedOut(client *c) { if (c->btype == BLOCKED_LIST || c->btype == BLOCKED_ZSET || c->btype == BLOCKED_STREAM) { - addReply(c,shared.nullmultibulk); + addReplyNullArray(c); } else if (c->btype == BLOCKED_WAIT) { addReplyLongLong(c,replicationCountAcksByOffset(c->bpop.reploffset)); } else if (c->btype == BLOCKED_MODULE) { @@ -213,15 +245,286 @@ void disconnectAllBlockedClients(void) { client *c = listNodeValue(ln); if (c->flags & CLIENT_BLOCKED) { - addReplySds(c,sdsnew( + /* PAUSED clients are an exception, when they'll be unblocked, the + * command processing will start from scratch, and the command will + * be either executed or rejected. (unlike LIST blocked clients for + * which the command is already in progress in a way. */ + if (c->btype == BLOCKED_PAUSE) + continue; + + addReplyError(c, "-UNBLOCKED force unblock from blocking operation, " - "instance state changed (master -> replica?)\r\n")); + "instance state changed (master -> replica?)"); unblockClient(c); c->flags |= CLIENT_CLOSE_AFTER_REPLY; } } } +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * when there may be clients blocked on a list key, and there may be new + * data to fetch (the key is ready). */ +void serveClientsBlockedOnListKey(robj *o, readyList *rl) { + /* We serve clients in the same order they blocked for + * this key, from the first blocked to the last. */ + dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); + if (de) { + list *clients = dictGetVal(de); + int numclients = listLength(clients); + + while(numclients--) { + listNode *clientnode = listFirst(clients); + client *receiver = clientnode->value; + + if (receiver->btype != BLOCKED_LIST) { + /* Put at the tail, so that at the next call + * we'll not run into it again. */ + listRotateHeadToTail(clients); + continue; + } + + robj *dstkey = receiver->bpop.target; + int wherefrom = receiver->bpop.listpos.wherefrom; + int whereto = receiver->bpop.listpos.whereto; + robj *value = listTypePop(o, wherefrom); + + if (value) { + /* Protect receiver->bpop.target, that will be + * freed by the next unblockClient() + * call. */ + if (dstkey) incrRefCount(dstkey); + + monotime replyTimer; + elapsedStart(&replyTimer); + if (serveClientBlockedOnList(receiver, + rl->key,dstkey,rl->db,value, + wherefrom, whereto) == C_ERR) + { + /* If we failed serving the client we need + * to also undo the POP operation. */ + listTypePush(o,value,wherefrom); + } + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); + unblockClient(receiver); + + if (dstkey) decrRefCount(dstkey); + decrRefCount(value); + } else { + break; + } + } + } + + if (listTypeLength(o) == 0) { + dbDelete(rl->db,rl->key); + notifyKeyspaceEvent(NOTIFY_GENERIC,"del",rl->key,rl->db->id); + } + /* We don't call signalModifiedKey() as it was already called + * when an element was pushed on the list. */ +} + +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * when there may be clients blocked on a sorted set key, and there may be new + * data to fetch (the key is ready). */ +void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) { + /* We serve clients in the same order they blocked for + * this key, from the first blocked to the last. */ + dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); + if (de) { + list *clients = dictGetVal(de); + int numclients = listLength(clients); + unsigned long zcard = zsetLength(o); + + while(numclients-- && zcard) { + listNode *clientnode = listFirst(clients); + client *receiver = clientnode->value; + + if (receiver->btype != BLOCKED_ZSET) { + /* Put at the tail, so that at the next call + * we'll not run into it again. */ + listRotateHeadToTail(clients); + continue; + } + + int where = (receiver->lastcmd && + receiver->lastcmd->proc == bzpopminCommand) + ? ZSET_MIN : ZSET_MAX; + monotime replyTimer; + elapsedStart(&replyTimer); + genericZpopCommand(receiver,&rl->key,1,where,1,NULL); + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); + unblockClient(receiver); + zcard--; + + /* Replicate the command. */ + robj *argv[2]; + struct redisCommand *cmd = where == ZSET_MIN ? + server.zpopminCommand : + server.zpopmaxCommand; + argv[0] = createStringObject(cmd->name,strlen(cmd->name)); + argv[1] = rl->key; + incrRefCount(rl->key); + propagate(cmd,receiver->db->id, + argv,2,PROPAGATE_AOF|PROPAGATE_REPL); + decrRefCount(argv[0]); + decrRefCount(argv[1]); + } + } +} + +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * when there may be clients blocked on a stream key, and there may be new + * data to fetch (the key is ready). */ +void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { + dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); + stream *s = o->ptr; + + /* We need to provide the new data arrived on the stream + * to all the clients that are waiting for an offset smaller + * than the current top item. */ + if (de) { + list *clients = dictGetVal(de); + listNode *ln; + listIter li; + listRewind(clients,&li); + + while((ln = listNext(&li))) { + client *receiver = listNodeValue(ln); + if (receiver->btype != BLOCKED_STREAM) continue; + bkinfo *bki = dictFetchValue(receiver->bpop.keys,rl->key); + streamID *gt = &bki->stream_id; + + /* If we blocked in the context of a consumer + * group, we need to resolve the group and update the + * last ID the client is blocked for: this is needed + * because serving other clients in the same consumer + * group will alter the "last ID" of the consumer + * group, and clients blocked in a consumer group are + * always blocked for the ">" ID: we need to deliver + * only new messages and avoid unblocking the client + * otherwise. */ + streamCG *group = NULL; + if (receiver->bpop.xread_group) { + group = streamLookupCG(s, + receiver->bpop.xread_group->ptr); + /* If the group was not found, send an error + * to the consumer. */ + if (!group) { + addReplyError(receiver, + "-NOGROUP the consumer group this client " + "was blocked on no longer exists"); + unblockClient(receiver); + continue; + } else { + *gt = group->last_id; + } + } + + if (streamCompareID(&s->last_id, gt) > 0) { + streamID start = *gt; + streamIncrID(&start); + + /* Lookup the consumer for the group, if any. */ + streamConsumer *consumer = NULL; + int noack = 0; + + if (group) { + int created = 0; + consumer = + streamLookupConsumer(group, + receiver->bpop.xread_consumer->ptr, + SLC_NONE, + &created); + noack = receiver->bpop.xread_group_noack; + if (created && noack) { + streamPropagateConsumerCreation(receiver,rl->key, + receiver->bpop.xread_group, + consumer->name); + } + } + + monotime replyTimer; + elapsedStart(&replyTimer); + /* Emit the two elements sub-array consisting of + * the name of the stream and the data we + * extracted from it. Wrapped in a single-item + * array, since we have just one key. */ + if (receiver->resp == 2) { + addReplyArrayLen(receiver,1); + addReplyArrayLen(receiver,2); + } else { + addReplyMapLen(receiver,1); + } + addReplyBulk(receiver,rl->key); + + streamPropInfo pi = { + rl->key, + receiver->bpop.xread_group + }; + streamReplyWithRange(receiver,s,&start,NULL, + receiver->bpop.xread_count, + 0, group, consumer, noack, &pi); + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); + + /* Note that after we unblock the client, 'gt' + * and other receiver->bpop stuff are no longer + * valid, so we must do the setup above before + * this call. */ + unblockClient(receiver); + } + } + } +} + +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * in order to check if we can serve clients blocked by modules using + * RM_BlockClientOnKeys(), when the corresponding key was signaled as ready: + * our goal here is to call the RedisModuleBlockedClient reply() callback to + * see if the key is really able to serve the client, and in that case, + * unblock it. */ +void serveClientsBlockedOnKeyByModule(readyList *rl) { + dictEntry *de; + + /* Optimization: If no clients are in type BLOCKED_MODULE, + * we can skip this loop. */ + if (!server.blocked_clients_by_type[BLOCKED_MODULE]) return; + + /* We serve clients in the same order they blocked for + * this key, from the first blocked to the last. */ + de = dictFind(rl->db->blocking_keys,rl->key); + if (de) { + list *clients = dictGetVal(de); + int numclients = listLength(clients); + + while(numclients--) { + listNode *clientnode = listFirst(clients); + client *receiver = clientnode->value; + + /* Put at the tail, so that at the next call + * we'll not run into it again: clients here may not be + * ready to be served, so they'll remain in the list + * sometimes. We want also be able to skip clients that are + * not blocked for the MODULE type safely. */ + listRotateHeadToTail(clients); + + if (receiver->btype != BLOCKED_MODULE) continue; + + /* Note that if *this* client cannot be served by this key, + * it does not mean that another client that is next into the + * list cannot be served as well: they may be blocked by + * different modules with different triggers to consider if a key + * is ready or not. This means we can't exit the loop but need + * to continue after the first failure. */ + monotime replyTimer; + elapsedStart(&replyTimer); + if (!moduleTryServeClientBlockedOnKey(receiver, rl->key)) continue; + updateStatsOnUnblock(receiver, 0, elapsedUs(replyTimer)); + + moduleUnblockClient(receiver); + } + } +} + /* This function should be called by Redis every time a single command, * a MULTI/EXEC block, or a Lua script, terminated its execution after * being called by a client. It handles serving clients blocked in @@ -231,8 +534,8 @@ void disconnectAllBlockedClients(void) { * one new element via some write operation are accumulated into * the server.ready_keys list. This function will run the list and will * serve clients accordingly. Note that the function will iterate again and - * again as a result of serving BRPOPLPUSH we can have new blocking clients - * to serve because of the PUSH side of BRPOPLPUSH. + * again as a result of serving BLMOVE we can have new blocking clients + * to serve because of the PUSH side of BLMOVE. * * This function is normally "fair", that is, it will server clients * using a FIFO behavior. However this fairness is violated in certain @@ -250,7 +553,7 @@ void handleClientsBlockedOnKeys(void) { /* Point server.ready_keys to a fresh list and save the current one * locally. This way as we run the old list we are free to call * signalKeyAsReady() that may push new elements in server.ready_keys - * when handling clients blocked into BRPOPLPUSH. */ + * when handling clients blocked into BLMOVE. */ l = server.ready_keys; server.ready_keys = listCreate(); @@ -262,201 +565,32 @@ void handleClientsBlockedOnKeys(void) { * we can safely call signalKeyAsReady() against this key. */ dictDelete(rl->db->ready_keys,rl->key); - /* Serve clients blocked on list key. */ + /* Even if we are not inside call(), increment the call depth + * in order to make sure that keys are expired against a fixed + * reference time, and not against the wallclock time. This + * way we can lookup an object multiple times (BLMOVE does + * that) without the risk of it being freed in the second + * lookup, invalidating the first one. + * See https://github.com/redis/redis/pull/6554. */ + server.fixed_time_expire++; + updateCachedTime(0); + + /* Serve clients blocked on the key. */ robj *o = lookupKeyWrite(rl->db,rl->key); - if (o != NULL && o->type == OBJ_LIST) { - dictEntry *de; - - /* We serve clients in the same order they blocked for - * this key, from the first blocked to the last. */ - de = dictFind(rl->db->blocking_keys,rl->key); - if (de) { - list *clients = dictGetVal(de); - int numclients = listLength(clients); - - while(numclients--) { - listNode *clientnode = listFirst(clients); - client *receiver = clientnode->value; - - if (receiver->btype != BLOCKED_LIST) { - /* Put at the tail, so that at the next call - * we'll not run into it again. */ - listDelNode(clients,clientnode); - listAddNodeTail(clients,receiver); - continue; - } - - robj *dstkey = receiver->bpop.target; - int where = (receiver->lastcmd && - receiver->lastcmd->proc == blpopCommand) ? - LIST_HEAD : LIST_TAIL; - robj *value = listTypePop(o,where); - - if (value) { - /* Protect receiver->bpop.target, that will be - * freed by the next unblockClient() - * call. */ - if (dstkey) incrRefCount(dstkey); - unblockClient(receiver); - - if (serveClientBlockedOnList(receiver, - rl->key,dstkey,rl->db,value, - where) == C_ERR) - { - /* If we failed serving the client we need - * to also undo the POP operation. */ - listTypePush(o,value,where); - } - - if (dstkey) decrRefCount(dstkey); - decrRefCount(value); - } else { - break; - } - } - } - if (listTypeLength(o) == 0) { - dbDelete(rl->db,rl->key); - notifyKeyspaceEvent(NOTIFY_GENERIC,"del",rl->key,rl->db->id); - } - /* We don't call signalModifiedKey() as it was already called - * when an element was pushed on the list. */ - } - - /* Serve clients blocked on sorted set key. */ - else if (o != NULL && o->type == OBJ_ZSET) { - dictEntry *de; - - /* We serve clients in the same order they blocked for - * this key, from the first blocked to the last. */ - de = dictFind(rl->db->blocking_keys,rl->key); - if (de) { - list *clients = dictGetVal(de); - int numclients = listLength(clients); - unsigned long zcard = zsetLength(o); - - while(numclients-- && zcard) { - listNode *clientnode = listFirst(clients); - client *receiver = clientnode->value; - - if (receiver->btype != BLOCKED_ZSET) { - /* Put at the tail, so that at the next call - * we'll not run into it again. */ - listDelNode(clients,clientnode); - listAddNodeTail(clients,receiver); - continue; - } - - int where = (receiver->lastcmd && - receiver->lastcmd->proc == bzpopminCommand) - ? ZSET_MIN : ZSET_MAX; - unblockClient(receiver); - genericZpopCommand(receiver,&rl->key,1,where,1,NULL); - zcard--; - - /* Replicate the command. */ - robj *argv[2]; - struct redisCommand *cmd = where == ZSET_MIN ? - server.zpopminCommand : - server.zpopmaxCommand; - argv[0] = createStringObject(cmd->name,strlen(cmd->name)); - argv[1] = rl->key; - incrRefCount(rl->key); - propagate(cmd,receiver->db->id, - argv,2,PROPAGATE_AOF|PROPAGATE_REPL); - decrRefCount(argv[0]); - decrRefCount(argv[1]); - } - } - } - - /* Serve clients blocked on stream key. */ - else if (o != NULL && o->type == OBJ_STREAM) { - dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); - stream *s = o->ptr; - - /* We need to provide the new data arrived on the stream - * to all the clients that are waiting for an offset smaller - * than the current top item. */ - if (de) { - list *clients = dictGetVal(de); - listNode *ln; - listIter li; - listRewind(clients,&li); - - while((ln = listNext(&li))) { - client *receiver = listNodeValue(ln); - if (receiver->btype != BLOCKED_STREAM) continue; - streamID *gt = dictFetchValue(receiver->bpop.keys, - rl->key); - - /* If we blocked in the context of a consumer - * group, we need to resolve the group and update the - * last ID the client is blocked for: this is needed - * because serving other clients in the same consumer - * group will alter the "last ID" of the consumer - * group, and clients blocked in a consumer group are - * always blocked for the ">" ID: we need to deliver - * only new messages and avoid unblocking the client - * otherwise. */ - streamCG *group = NULL; - if (receiver->bpop.xread_group) { - group = streamLookupCG(s, - receiver->bpop.xread_group->ptr); - /* If the group was not found, send an error - * to the consumer. */ - if (!group) { - addReplyError(receiver, - "-NOGROUP the consumer group this client " - "was blocked on no longer exists"); - unblockClient(receiver); - continue; - } else { - *gt = group->last_id; - } - } - - if (streamCompareID(&s->last_id, gt) > 0) { - streamID start = *gt; - start.seq++; /* Can't overflow, it's an uint64_t */ - - /* Lookup the consumer for the group, if any. */ - streamConsumer *consumer = NULL; - int noack = 0; - - if (group) { - consumer = streamLookupConsumer(group, - receiver->bpop.xread_consumer->ptr, - 1); - noack = receiver->bpop.xread_group_noack; - } - - /* Emit the two elements sub-array consisting of - * the name of the stream and the data we - * extracted from it. Wrapped in a single-item - * array, since we have just one key. */ - addReplyMultiBulkLen(receiver,1); - addReplyMultiBulkLen(receiver,2); - addReplyBulk(receiver,rl->key); - - streamPropInfo pi = { - rl->key, - receiver->bpop.xread_group - }; - streamReplyWithRange(receiver,s,&start,NULL, - receiver->bpop.xread_count, - 0, group, consumer, noack, &pi); - - /* Note that after we unblock the client, 'gt' - * and other receiver->bpop stuff are no longer - * valid, so we must do the setup above before - * this call. */ - unblockClient(receiver); - } - } - } + if (o != NULL) { + if (o->type == OBJ_LIST) + serveClientsBlockedOnListKey(o,rl); + else if (o->type == OBJ_ZSET) + serveClientsBlockedOnSortedSetKey(o,rl); + else if (o->type == OBJ_STREAM) + serveClientsBlockedOnStreamKey(o,rl); + /* We want to serve clients blocked on module keys + * regardless of the object type: we don't know what the + * module is trying to accomplish right now. */ + serveClientsBlockedOnKeyByModule(rl); } + server.fixed_time_expire--; /* Free this item. */ decrRefCount(rl->key); @@ -494,7 +628,7 @@ void handleClientsBlockedOnKeys(void) { * stream keys, we also provide an array of streamID structures: clients will * be unblocked only when items with an ID greater or equal to the specified * one is appended to the stream. */ -void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, streamID *ids) { +void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeout, robj *target, struct listPos *listpos, streamID *ids) { dictEntry *de; list *l; int j; @@ -502,20 +636,20 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo c->bpop.timeout = timeout; c->bpop.target = target; + if (listpos != NULL) c->bpop.listpos = *listpos; + if (target != NULL) incrRefCount(target); for (j = 0; j < numkeys; j++) { - /* The value associated with the key name in the bpop.keys dictionary - * is NULL for lists and sorted sets, or the stream ID for streams. */ - void *key_data = NULL; - if (btype == BLOCKED_STREAM) { - key_data = zmalloc(sizeof(streamID)); - memcpy(key_data,ids+j,sizeof(streamID)); - } + /* Allocate our bkinfo structure, associated to each key the client + * is blocked for. */ + bkinfo *bki = zmalloc(sizeof(*bki)); + if (btype == BLOCKED_STREAM) + bki->stream_id = ids[j]; /* If the key already exists in the dictionary ignore it. */ - if (dictAdd(c->bpop.keys,keys[j],key_data) != DICT_OK) { - zfree(key_data); + if (dictAdd(c->bpop.keys,keys[j],bki) != DICT_OK) { + zfree(bki); continue; } incrRefCount(keys[j]); @@ -534,6 +668,7 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo l = dictGetVal(de); } listAddNodeTail(l,c); + bki->listnode = listLast(l); } blockClient(c,btype); } @@ -550,11 +685,12 @@ void unblockClientWaitingData(client *c) { /* The client may wait for multiple keys, so unblock it for every key. */ while((de = dictNext(di)) != NULL) { robj *key = dictGetKey(de); + bkinfo *bki = dictGetVal(de); /* Remove this client from the list of clients waiting for this key. */ l = dictFetchValue(c->db->blocking_keys,key); serverAssertWithInfo(c,key,l != NULL); - listDelNode(l,listSearchKey(l,c)); + listDelNode(l,bki->listnode); /* If the list is empty we need to remove it to avoid wasting memory */ if (listLength(l) == 0) dictDelete(c->db->blocking_keys,key); @@ -575,16 +711,41 @@ void unblockClientWaitingData(client *c) { } } +static int getBlockedTypeByType(int type) { + switch (type) { + case OBJ_LIST: return BLOCKED_LIST; + case OBJ_ZSET: return BLOCKED_ZSET; + case OBJ_MODULE: return BLOCKED_MODULE; + case OBJ_STREAM: return BLOCKED_STREAM; + default: return BLOCKED_NONE; + } +} + /* If the specified key has clients blocked waiting for list pushes, this * function will put the key reference into the server.ready_keys list. * Note that db->ready_keys is a hash table that allows us to avoid putting * the same key again and again in the list in case of multiple pushes * made by a script or in the context of MULTI/EXEC. * - * The list will be finally processed by handleClientsBlockedOnLists() */ -void signalKeyAsReady(redisDb *db, robj *key) { + * The list will be finally processed by handleClientsBlockedOnKeys() */ +void signalKeyAsReady(redisDb *db, robj *key, int type) { readyList *rl; + /* Quick returns. */ + int btype = getBlockedTypeByType(type); + if (btype == BLOCKED_NONE) { + /* The type can never block. */ + return; + } + if (!server.blocked_clients_by_type[btype] && + !server.blocked_clients_by_type[BLOCKED_MODULE]) { + /* No clients block on this type. Note: Blocked modules are represented + * by BLOCKED_MODULE, even if the intention is to wake up by normal + * types (list, zset, stream), so we need to check that there are no + * blocked modules before we do a quick return here. */ + return; + } + /* No clients blocking for this key? No need to queue it. */ if (dictFind(db->blocking_keys,key) == NULL) return; @@ -605,4 +766,3 @@ void signalKeyAsReady(redisDb *db, robj *key) { serverAssert(dictAdd(db->ready_keys,key,NULL) == DICT_OK); } - diff --git a/redis.submodule/src/childinfo.c b/redis.submodule/src/childinfo.c index 719025e..5e94a4f 100644 --- a/redis.submodule/src/childinfo.c +++ b/redis.submodule/src/childinfo.c @@ -30,18 +30,26 @@ #include "server.h" #include +typedef struct { + size_t keys; + size_t cow; + monotime cow_updated; + double progress; + childInfoType information_type; /* Type of information */ +} child_info_data; + /* Open a child-parent channel used in order to move information about the * RDB / AOF saving process from the child to the parent (for instance * the amount of copy on write memory used) */ void openChildInfoPipe(void) { if (pipe(server.child_info_pipe) == -1) { /* On error our two file descriptors should be still set to -1, - * but we call anyway cloesChildInfoPipe() since can't hurt. */ + * but we call anyway closeChildInfoPipe() since can't hurt. */ closeChildInfoPipe(); } else if (anetNonBlock(NULL,server.child_info_pipe[0]) != ANET_OK) { closeChildInfoPipe(); } else { - memset(&server.child_info_data,0,sizeof(server.child_info_data)); + server.child_info_nread = 0; } } @@ -54,32 +62,112 @@ void closeChildInfoPipe(void) { close(server.child_info_pipe[1]); server.child_info_pipe[0] = -1; server.child_info_pipe[1] = -1; + server.child_info_nread = 0; } } -/* Send COW data to parent. The child should call this function after populating - * the corresponding fields it want to sent (according to the process type). */ -void sendChildInfo(int ptype) { +/* Send save data to parent. */ +void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress, char *pname) { if (server.child_info_pipe[1] == -1) return; - server.child_info_data.magic = CHILD_INFO_MAGIC; - server.child_info_data.process_type = ptype; - ssize_t wlen = sizeof(server.child_info_data); - if (write(server.child_info_pipe[1],&server.child_info_data,wlen) != wlen) { + + static monotime cow_updated = 0; + static uint64_t cow_update_cost = 0; + static size_t cow = 0; + + child_info_data data = {0}; /* zero everything, including padding to satisfy valgrind */ + + /* When called to report current info, we need to throttle down CoW updates as they + * can be very expensive. To do that, we measure the time it takes to get a reading + * and schedule the next reading to happen not before time*CHILD_COW_COST_FACTOR + * passes. */ + + monotime now = getMonotonicUs(); + if (info_type != CHILD_INFO_TYPE_CURRENT_INFO || + !cow_updated || + now - cow_updated > cow_update_cost * CHILD_COW_DUTY_CYCLE) + { + cow = zmalloc_get_private_dirty(-1); + cow_updated = getMonotonicUs(); + cow_update_cost = cow_updated - now; + + if (cow) { + serverLog((info_type == CHILD_INFO_TYPE_CURRENT_INFO) ? LL_VERBOSE : LL_NOTICE, + "%s: %zu MB of memory used by copy-on-write", + pname, cow / (1024 * 1024)); + } + } + + data.information_type = info_type; + data.keys = keys; + data.cow = cow; + data.cow_updated = cow_updated; + data.progress = progress; + + ssize_t wlen = sizeof(data); + + if (write(server.child_info_pipe[1], &data, wlen) != wlen) { /* Nothing to do on error, this will be detected by the other side. */ } } -/* Receive COW data from parent. */ +/* Update Child info. */ +void updateChildInfo(childInfoType information_type, size_t cow, monotime cow_updated, size_t keys, double progress) { + if (information_type == CHILD_INFO_TYPE_CURRENT_INFO) { + server.stat_current_cow_bytes = cow; + server.stat_current_cow_updated = cow_updated; + server.stat_current_save_keys_processed = keys; + if (progress != -1) server.stat_module_progress = progress; + } else if (information_type == CHILD_INFO_TYPE_AOF_COW_SIZE) { + server.stat_aof_cow_bytes = cow; + } else if (information_type == CHILD_INFO_TYPE_RDB_COW_SIZE) { + server.stat_rdb_cow_bytes = cow; + } else if (information_type == CHILD_INFO_TYPE_MODULE_COW_SIZE) { + server.stat_module_cow_bytes = cow; + } +} + +/* Read child info data from the pipe. + * if complete data read into the buffer, + * data is stored into *buffer, and returns 1. + * otherwise, the partial data is left in the buffer, waiting for the next read, and returns 0. */ +int readChildInfo(childInfoType *information_type, size_t *cow, monotime *cow_updated, size_t *keys, double* progress) { + /* We are using here a static buffer in combination with the server.child_info_nread to handle short reads */ + static child_info_data buffer; + ssize_t wlen = sizeof(buffer); + + /* Do not overlap */ + if (server.child_info_nread == wlen) server.child_info_nread = 0; + + int nread = read(server.child_info_pipe[0], (char *)&buffer + server.child_info_nread, wlen - server.child_info_nread); + if (nread > 0) { + server.child_info_nread += nread; + } + + /* We have complete child info */ + if (server.child_info_nread == wlen) { + *information_type = buffer.information_type; + *cow = buffer.cow; + *cow_updated = buffer.cow_updated; + *keys = buffer.keys; + *progress = buffer.progress; + return 1; + } else { + return 0; + } +} + +/* Receive info data from child. */ void receiveChildInfo(void) { if (server.child_info_pipe[0] == -1) return; - ssize_t wlen = sizeof(server.child_info_data); - if (read(server.child_info_pipe[0],&server.child_info_data,wlen) == wlen && - server.child_info_data.magic == CHILD_INFO_MAGIC) - { - if (server.child_info_data.process_type == CHILD_INFO_TYPE_RDB) { - server.stat_rdb_cow_bytes = server.child_info_data.cow_size; - } else if (server.child_info_data.process_type == CHILD_INFO_TYPE_AOF) { - server.stat_aof_cow_bytes = server.child_info_data.cow_size; - } + + size_t cow; + monotime cow_updated; + size_t keys; + double progress; + childInfoType information_type; + + /* Drain the pipe and update child info so that we get the final message. */ + while (readChildInfo(&information_type, &cow, &cow_updated, &keys, &progress)) { + updateChildInfo(information_type, cow, cow_updated, keys, progress); } } diff --git a/redis.submodule/src/cli_common.c b/redis.submodule/src/cli_common.c new file mode 100644 index 0000000..e88327a --- /dev/null +++ b/redis.submodule/src/cli_common.c @@ -0,0 +1,194 @@ +/* CLI (command line interface) common methods + * + * Copyright (c) 2020, Redis Labs + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "cli_common.h" +#include +#include +#include /* Use hiredis' sds compat header that maps sds calls to their hi_ variants */ +#include /* use sds.h from hiredis, so that only one set of sds functions will be present in the binary */ +#ifdef USE_OPENSSL +#include +#include +#include +#endif + + +/* Wrapper around redisSecureConnection to avoid hiredis_ssl dependencies if + * not building with TLS support. + */ +int cliSecureConnection(redisContext *c, cliSSLconfig config, const char **err) { +#ifdef USE_OPENSSL + static SSL_CTX *ssl_ctx = NULL; + + if (!ssl_ctx) { + ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + if (!ssl_ctx) { + *err = "Failed to create SSL_CTX"; + goto error; + } + SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); + SSL_CTX_set_verify(ssl_ctx, config.skip_cert_verify ? SSL_VERIFY_NONE : SSL_VERIFY_PEER, NULL); + + if (config.cacert || config.cacertdir) { + if (!SSL_CTX_load_verify_locations(ssl_ctx, config.cacert, config.cacertdir)) { + *err = "Invalid CA Certificate File/Directory"; + goto error; + } + } else { + if (!SSL_CTX_set_default_verify_paths(ssl_ctx)) { + *err = "Failed to use default CA paths"; + goto error; + } + } + + if (config.cert && !SSL_CTX_use_certificate_chain_file(ssl_ctx, config.cert)) { + *err = "Invalid client certificate"; + goto error; + } + + if (config.key && !SSL_CTX_use_PrivateKey_file(ssl_ctx, config.key, SSL_FILETYPE_PEM)) { + *err = "Invalid private key"; + goto error; + } + if (config.ciphers && !SSL_CTX_set_cipher_list(ssl_ctx, config.ciphers)) { + *err = "Error while configuring ciphers"; + goto error; + } +#ifdef TLS1_3_VERSION + if (config.ciphersuites && !SSL_CTX_set_ciphersuites(ssl_ctx, config.ciphersuites)) { + *err = "Error while setting cypher suites"; + goto error; + } +#endif + } + + SSL *ssl = SSL_new(ssl_ctx); + if (!ssl) { + *err = "Failed to create SSL object"; + return REDIS_ERR; + } + + if (config.sni && !SSL_set_tlsext_host_name(ssl, config.sni)) { + *err = "Failed to configure SNI"; + SSL_free(ssl); + return REDIS_ERR; + } + + return redisInitiateSSL(c, ssl); + +error: + SSL_CTX_free(ssl_ctx); + ssl_ctx = NULL; + return REDIS_ERR; +#else + (void) config; + (void) c; + (void) err; + return REDIS_OK; +#endif +} + +/* Wrapper around hiredis to allow arbitrary reads and writes. + * + * We piggybacks on top of hiredis to achieve transparent TLS support, + * and use its internal buffers so it can co-exist with commands + * previously/later issued on the connection. + * + * Interface is close to enough to read()/write() so things should mostly + * work transparently. + */ + +/* Write a raw buffer through a redisContext. If we already have something + * in the buffer (leftovers from hiredis operations) it will be written + * as well. + */ +ssize_t cliWriteConn(redisContext *c, const char *buf, size_t buf_len) +{ + int done = 0; + + /* Append data to buffer which is *usually* expected to be empty + * but we don't assume that, and write. + */ + c->obuf = sdscatlen(c->obuf, buf, buf_len); + if (redisBufferWrite(c, &done) == REDIS_ERR) { + if (!(c->flags & REDIS_BLOCK)) + errno = EAGAIN; + + /* On error, we assume nothing was written and we roll back the + * buffer to its original state. + */ + if (sdslen(c->obuf) > buf_len) + sdsrange(c->obuf, 0, -(buf_len+1)); + else + sdsclear(c->obuf); + + return -1; + } + + /* If we're done, free up everything. We may have written more than + * buf_len (if c->obuf was not initially empty) but we don't have to + * tell. + */ + if (done) { + sdsclear(c->obuf); + return buf_len; + } + + /* Write was successful but we have some leftovers which we should + * remove from the buffer. + * + * Do we still have data that was there prior to our buf? If so, + * restore buffer to it's original state and report no new data was + * writen. + */ + if (sdslen(c->obuf) > buf_len) { + sdsrange(c->obuf, 0, -(buf_len+1)); + return 0; + } + + /* At this point we're sure no prior data is left. We flush the buffer + * and report how much we've written. + */ + size_t left = sdslen(c->obuf); + sdsclear(c->obuf); + return buf_len - left; +} + +/* Wrapper around OpenSSL (libssl and libcrypto) initialisation + */ +int cliSecureInit() +{ +#ifdef USE_OPENSSL + ERR_load_crypto_strings(); + SSL_load_error_strings(); + SSL_library_init(); +#endif + return REDIS_OK; +} diff --git a/redis.submodule/src/cli_common.h b/redis.submodule/src/cli_common.h new file mode 100644 index 0000000..16d6ec2 --- /dev/null +++ b/redis.submodule/src/cli_common.h @@ -0,0 +1,50 @@ +#ifndef __CLICOMMON_H +#define __CLICOMMON_H + +#include + +typedef struct cliSSLconfig { + /* Requested SNI, or NULL */ + char *sni; + /* CA Certificate file, or NULL */ + char *cacert; + /* Directory where trusted CA certificates are stored, or NULL */ + char *cacertdir; + /* Skip server certificate verification. */ + int skip_cert_verify; + /* Client certificate to authenticate with, or NULL */ + char *cert; + /* Private key file to authenticate with, or NULL */ + char *key; + /* Prefered cipher list, or NULL (applies only to <= TLSv1.2) */ + char* ciphers; + /* Prefered ciphersuites list, or NULL (applies only to TLSv1.3) */ + char* ciphersuites; +} cliSSLconfig; + +/* Wrapper around redisSecureConnection to avoid hiredis_ssl dependencies if + * not building with TLS support. + */ +int cliSecureConnection(redisContext *c, cliSSLconfig config, const char **err); + +/* Wrapper around hiredis to allow arbitrary reads and writes. + * + * We piggybacks on top of hiredis to achieve transparent TLS support, + * and use its internal buffers so it can co-exist with commands + * previously/later issued on the connection. + * + * Interface is close to enough to read()/write() so things should mostly + * work transparently. + */ + +/* Write a raw buffer through a redisContext. If we already have something + * in the buffer (leftovers from hiredis operations) it will be written + * as well. + */ +ssize_t cliWriteConn(redisContext *c, const char *buf, size_t buf_len); + +/* Wrapper around OpenSSL (libssl and libcrypto) initialisation. + */ +int cliSecureInit(); + +#endif /* __CLICOMMON_H */ diff --git a/redis.submodule/src/cluster.c b/redis.submodule/src/cluster.c index 858c0c9..b305353 100644 --- a/redis.submodule/src/cluster.c +++ b/redis.submodule/src/cluster.c @@ -47,15 +47,15 @@ clusterNode *myself = NULL; clusterNode *createClusterNode(char *nodename, int flags); -int clusterAddNode(clusterNode *node); +void clusterAddNode(clusterNode *node); void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask); -void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask); +void clusterReadHandler(connection *conn); void clusterSendPing(clusterLink *link, int type); void clusterSendFail(char *nodename); void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request); void clusterUpdateState(void); int clusterNodeGetSlotBit(clusterNode *n, int slot); -sds clusterGenNodesDescription(int filter); +sds clusterGenNodesDescription(int filter, int use_pport); clusterNode *clusterLookupNode(const char *name); int clusterNodeAddSlave(clusterNode *master, clusterNode *slave); int clusterAddSlot(clusterNode *n, int slot); @@ -77,6 +77,9 @@ uint64_t clusterGetMaxEpoch(void); int clusterBumpConfigEpochWithoutConsensus(void); void moduleCallClusterReceivers(const char *sender_id, uint64_t module_id, uint8_t type, const unsigned char *payload, uint32_t len); +#define RCVBUF_INIT_LEN 1024 +#define RCVBUF_MAX_PREALLOC (1<<20) /* 1MB */ + /* ----------------------------------------------------------------------------- * Initialization * -------------------------------------------------------------------------- */ @@ -157,7 +160,10 @@ int clusterLoadConfig(char *filename) { } /* Regular config lines have at least eight fields */ - if (argc < 8) goto fmterr; + if (argc < 8) { + sdsfreesplitres(argv,argc); + goto fmterr; + } /* Create this node if it does not exist */ n = clusterLookupNode(argv[0]); @@ -166,7 +172,10 @@ int clusterLoadConfig(char *filename) { clusterAddNode(n); } /* Address and port */ - if ((p = strrchr(argv[1],':')) == NULL) goto fmterr; + if ((p = strrchr(argv[1],':')) == NULL) { + sdsfreesplitres(argv,argc); + goto fmterr; + } *p = '\0'; memcpy(n->ip,argv[1],strlen(argv[1])+1); char *port = p+1; @@ -181,6 +190,9 @@ int clusterLoadConfig(char *filename) { * base port. */ n->cport = busp ? atoi(busp) : n->port + CLUSTER_PORT_INCR; + /* The plaintext port for client in a TLS cluster (n->pport) is not + * stored in nodes.conf. It is received later over the bus protocol. */ + /* Parse flags */ p = s = argv[2]; while(p) { @@ -247,7 +259,10 @@ int clusterLoadConfig(char *filename) { *p = '\0'; direction = p[1]; /* Either '>' or '<' */ slot = atoi(argv[j]+1); - if (slot < 0 || slot >= CLUSTER_SLOTS) goto fmterr; + if (slot < 0 || slot >= CLUSTER_SLOTS) { + sdsfreesplitres(argv,argc); + goto fmterr; + } p += 3; cn = clusterLookupNode(p); if (!cn) { @@ -267,8 +282,12 @@ int clusterLoadConfig(char *filename) { } else { start = stop = atoi(argv[j]); } - if (start < 0 || start >= CLUSTER_SLOTS) goto fmterr; - if (stop < 0 || stop >= CLUSTER_SLOTS) goto fmterr; + if (start < 0 || start >= CLUSTER_SLOTS || + stop < 0 || stop >= CLUSTER_SLOTS) + { + sdsfreesplitres(argv,argc); + goto fmterr; + } while(start <= stop) clusterAddSlot(n, start++); } @@ -320,7 +339,7 @@ int clusterSaveConfig(int do_fsync) { /* Get the nodes description and concatenate our "vars" directive to * save currentEpoch and lastVoteEpoch. */ - ci = clusterGenNodesDescription(CLUSTER_NODE_HANDSHAKE); + ci = clusterGenNodesDescription(CLUSTER_NODE_HANDSHAKE, 0); ci = sdscatprintf(ci,"vars currentEpoch %llu lastVoteEpoch %llu\n", (unsigned long long) server.cluster->currentEpoch, (unsigned long long) server.cluster->lastVoteEpoch); @@ -339,7 +358,7 @@ int clusterSaveConfig(int do_fsync) { if (write(fd,ci,sdslen(ci)) != (ssize_t)sdslen(ci)) goto err; if (do_fsync) { server.cluster->todo_before_sleep &= ~CLUSTER_TODO_FSYNC_CONFIG; - fsync(fd); + if (fsync(fd) == -1) goto err; } /* Truncate the file if needed to remove the final \n padding that @@ -364,7 +383,7 @@ void clusterSaveConfigOrDie(int do_fsync) { } } -/* Lock the cluster config using flock(), and leaks the file descritor used to +/* Lock the cluster config using flock(), and leaks the file descriptor used to * acquire the lock so that the file will be locked forever. * * This works because we always update nodes.conf with a new version @@ -382,7 +401,7 @@ int clusterLockConfig(char *filename) { /* To lock it, we need to open the file in a way it is created if * it does not exist, otherwise there is a race condition with other * processes. */ - int fd = open(filename,O_WRONLY|O_CREAT,0644); + int fd = open(filename,O_WRONLY|O_CREAT|O_CLOEXEC,0644); if (fd == -1) { serverLog(LL_WARNING, "Can't open %s in order to acquire a lock: %s", @@ -405,12 +424,42 @@ int clusterLockConfig(char *filename) { return C_ERR; } /* Lock acquired: leak the 'fd' by not closing it, so that we'll retain the - * lock to the file as long as the process exists. */ + * lock to the file as long as the process exists. + * + * After fork, the child process will get the fd opened by the parent process, + * we need save `fd` to `cluster_config_file_lock_fd`, so that in redisFork(), + * it will be closed in the child process. + * If it is not closed, when the main process is killed -9, but the child process + * (redis-aof-rewrite) is still alive, the fd(lock) will still be held by the + * child process, and the main process will fail to get lock, means fail to start. */ + server.cluster_config_file_lock_fd = fd; +#else + UNUSED(filename); #endif /* __sun */ return C_OK; } +/* Derives our ports to be announced in the cluster bus. */ +void deriveAnnouncedPorts(int *announced_port, int *announced_pport, + int *announced_cport) { + int port = server.tls_cluster ? server.tls_port : server.port; + /* Default announced ports. */ + *announced_port = port; + *announced_pport = server.tls_cluster ? server.port : 0; + *announced_cport = port + CLUSTER_PORT_INCR; + /* Config overriding announced ports. */ + if (server.tls_cluster && server.cluster_announce_tls_port) { + *announced_port = server.cluster_announce_tls_port; + *announced_pport = server.cluster_announce_port; + } else if (server.cluster_announce_port) { + *announced_port = server.cluster_announce_port; + } + if (server.cluster_announce_bus_port) { + *announced_cport = server.cluster_announce_bus_port; + } +} + /* Some flags (currently just the NOFAILOVER flag) may need to be updated * in the "myself" node based on the current configuration of the node, * that may change at runtime via CONFIG SET. This function changes the @@ -455,6 +504,7 @@ void clusterInit(void) { /* Lock the cluster config file to make sure every node uses * its own nodes.conf. */ + server.cluster_config_file_lock_fd = -1; if (clusterLockConfig(server.cluster_configfile) == C_ERR) exit(1); @@ -472,33 +522,24 @@ void clusterInit(void) { if (saveconf) clusterSaveConfigOrDie(1); /* We need a listening TCP port for our cluster messaging needs. */ - server.cfd_count = 0; + server.cfd.count = 0; /* Port sanity check II * The other handshake port check is triggered too late to stop * us from trying to use a too-high cluster port number. */ - if (server.port > (65535-CLUSTER_PORT_INCR)) { + int port = server.tls_cluster ? server.tls_port : server.port; + if (port > (65535-CLUSTER_PORT_INCR)) { serverLog(LL_WARNING, "Redis port number too high. " "Cluster communication port is 10,000 port " "numbers higher than your Redis port. " - "Your Redis port number must be " - "lower than 55535."); + "Your Redis port number must be 55535 or less."); exit(1); } - - if (listenToPort(server.port+CLUSTER_PORT_INCR, - server.cfd,&server.cfd_count) == C_ERR) - { + if (listenToPort(port+CLUSTER_PORT_INCR, &server.cfd) == C_ERR) { exit(1); - } else { - int j; - - for (j = 0; j < server.cfd_count; j++) { - if (aeCreateFileEvent(server.el, server.cfd[j], AE_READABLE, - clusterAcceptHandler, NULL) == AE_ERR) - serverPanic("Unrecoverable error creating Redis Cluster " - "file event."); - } + } + if (createSocketAcceptHandler(&server.cfd, clusterAcceptHandler) != C_OK) { + serverPanic("Unrecoverable error creating Redis Cluster socket accept handler."); } /* The slots -> keys map is a radix tree. Initialize it here. */ @@ -506,14 +547,9 @@ void clusterInit(void) { memset(server.cluster->slots_keys_count,0, sizeof(server.cluster->slots_keys_count)); - /* Set myself->port / cport to my listening ports, we'll just need to + /* Set myself->port/cport/pport to my listening ports, we'll just need to * discover the IP address via MEET messages. */ - myself->port = server.port; - myself->cport = server.port+CLUSTER_PORT_INCR; - if (server.cluster_announce_port) - myself->port = server.cluster_announce_port; - if (server.cluster_announce_bus_port) - myself->cport = server.cluster_announce_bus_port; + deriveAnnouncedPorts(&myself->port, &myself->pport, &myself->cport); server.cluster->mf_end = 0; resetManualFailover(); @@ -522,13 +558,13 @@ void clusterInit(void) { /* Reset a node performing a soft or hard reset: * - * 1) All other nodes are forget. + * 1) All other nodes are forgotten. * 2) All the assigned / open slots are released. * 3) If the node is a slave, it turns into a master. - * 5) Only for hard reset: a new Node ID is generated. - * 6) Only for hard reset: currentEpoch and configEpoch are set to 0. - * 7) The new configuration is saved and the cluster state updated. - * 8) If the node was a slave, the whole data set is flushed away. */ + * 4) Only for hard reset: a new Node ID is generated. + * 5) Only for hard reset: currentEpoch and configEpoch are set to 0. + * 6) The new configuration is saved and the cluster state updated. + * 7) If the node was a slave, the whole data set is flushed away. */ void clusterReset(int hard) { dictIterator *di; dictEntry *de; @@ -591,9 +627,10 @@ clusterLink *createClusterLink(clusterNode *node) { clusterLink *link = zmalloc(sizeof(*link)); link->ctime = mstime(); link->sndbuf = sdsempty(); - link->rcvbuf = sdsempty(); + link->rcvbuf = zmalloc(link->rcvbuf_alloc = RCVBUF_INIT_LEN); + link->rcvbuf_len = 0; link->node = node; - link->fd = -1; + link->conn = NULL; return link; } @@ -601,23 +638,45 @@ clusterLink *createClusterLink(clusterNode *node) { * This function will just make sure that the original node associated * with this link will have the 'link' field set to NULL. */ void freeClusterLink(clusterLink *link) { - if (link->fd != -1) { - aeDeleteFileEvent(server.el, link->fd, AE_READABLE|AE_WRITABLE); + if (link->conn) { + connClose(link->conn); + link->conn = NULL; } sdsfree(link->sndbuf); - sdsfree(link->rcvbuf); + zfree(link->rcvbuf); if (link->node) link->node->link = NULL; - close(link->fd); zfree(link); } +static void clusterConnAcceptHandler(connection *conn) { + clusterLink *link; + + if (connGetState(conn) != CONN_STATE_CONNECTED) { + serverLog(LL_VERBOSE, + "Error accepting cluster node connection: %s", connGetLastError(conn)); + connClose(conn); + return; + } + + /* Create a link object we use to handle the connection. + * It gets passed to the readable handler when data is available. + * Initially the link->node pointer is set to NULL as we don't know + * which node is, but the right node is references once we know the + * node identity. */ + link = createClusterLink(NULL); + link->conn = conn; + connSetPrivateData(conn, link); + + /* Register read handler */ + connSetReadHandler(conn, clusterReadHandler); +} + #define MAX_CLUSTER_ACCEPTS_PER_CALL 1000 void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { int cport, cfd; int max = MAX_CLUSTER_ACCEPTS_PER_CALL; char cip[NET_IP_STR_LEN]; - clusterLink *link; UNUSED(el); UNUSED(mask); UNUSED(privdata); @@ -634,22 +693,49 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { "Error accepting cluster node: %s", server.neterr); return; } - anetNonBlock(NULL,cfd); - anetEnableTcpNoDelay(NULL,cfd); + + connection *conn = server.tls_cluster ? + connCreateAcceptedTLS(cfd, TLS_CLIENT_AUTH_YES) : connCreateAcceptedSocket(cfd); + + /* Make sure connection is not in an error state */ + if (connGetState(conn) != CONN_STATE_ACCEPTING) { + serverLog(LL_VERBOSE, + "Error creating an accepting connection for cluster node: %s", + connGetLastError(conn)); + connClose(conn); + return; + } + connNonBlock(conn); + connEnableTcpNoDelay(conn); + connKeepAlive(conn,server.cluster_node_timeout * 2); /* Use non-blocking I/O for cluster messages. */ - serverLog(LL_VERBOSE,"Accepted cluster node %s:%d", cip, cport); - /* Create a link object we use to handle the connection. - * It gets passed to the readable handler when data is available. - * Initiallly the link->node pointer is set to NULL as we don't know - * which node is, but the right node is references once we know the - * node identity. */ - link = createClusterLink(NULL); - link->fd = cfd; - aeCreateFileEvent(server.el,cfd,AE_READABLE,clusterReadHandler,link); + serverLog(LL_VERBOSE,"Accepting cluster node connection from %s:%d", cip, cport); + + /* Accept the connection now. connAccept() may call our handler directly + * or schedule it for later depending on connection implementation. + */ + if (connAccept(conn, clusterConnAcceptHandler) == C_ERR) { + if (connGetState(conn) == CONN_STATE_ERROR) + serverLog(LL_VERBOSE, + "Error accepting cluster node connection: %s", + connGetLastError(conn)); + connClose(conn); + return; + } } } +/* Return the approximated number of sockets we are using in order to + * take the cluster bus connections. */ +unsigned long getClusterConnectionsCount(void) { + /* We decrement the number of nodes by one, since there is the + * "myself" node too in the list. Each node uses two file descriptors, + * one incoming and one outgoing, thus the multiplication by 2. */ + return server.cluster_enabled ? + ((dictSize(server.cluster->nodes)-1)*2) : 0; +} + /* ----------------------------------------------------------------------------- * Key space handling * -------------------------------------------------------------------------- */ @@ -703,16 +789,19 @@ clusterNode *createClusterNode(char *nodename, int flags) { node->configEpoch = 0; node->flags = flags; memset(node->slots,0,sizeof(node->slots)); + node->slots_info = NULL; node->numslots = 0; node->numslaves = 0; node->slaves = NULL; node->slaveof = NULL; node->ping_sent = node->pong_received = 0; + node->data_received = 0; node->fail_time = 0; node->link = NULL; memset(node->ip,0,sizeof(node->ip)); node->port = 0; node->cport = 0; + node->pport = 0; node->fail_reports = listCreate(); node->voted_time = 0; node->orphaned_time = 0; @@ -884,15 +973,15 @@ void freeClusterNode(clusterNode *n) { } /* Add a node to the nodes hash table */ -int clusterAddNode(clusterNode *node) { +void clusterAddNode(clusterNode *node) { int retval; retval = dictAdd(server.cluster->nodes, sdsnewlen(node->name,CLUSTER_NAMELEN), node); - return (retval == DICT_OK) ? C_OK : C_ERR; + serverAssert(retval == DICT_OK); } -/* Remove a node from the cluster. The functio performs the high level +/* Remove a node from the cluster. The function performs the high level * cleanup, calling freeClusterNode() for the low level cleanup. * Here we do the following: * @@ -989,7 +1078,7 @@ uint64_t clusterGetMaxEpoch(void) { * 3) Persist the configuration on disk before sending packets with the * new configuration. * - * If the new config epoch is generated and assigend, C_OK is returned, + * If the new config epoch is generated and assigned, C_OK is returned, * otherwise C_ERR is returned (since the node has already the greatest * configuration around) and no operation is performed. * @@ -1062,7 +1151,7 @@ int clusterBumpConfigEpochWithoutConsensus(void) { * * In general we want a system that eventually always ends with different * masters having different configuration epochs whatever happened, since - * nothign is worse than a split-brain condition in a distributed system. + * nothing is worse than a split-brain condition in a distributed system. * * BEHAVIOR * @@ -1121,7 +1210,7 @@ void clusterHandleConfigEpochCollision(clusterNode *sender) { * entries from the black list. This is an O(N) operation but it is not a * problem since add / exists operations are called very infrequently and * the hash table is supposed to contain very little elements at max. - * However without the cleanup during long uptimes and with some automated + * However without the cleanup during long uptime and with some automated * node add/removal procedures, entries could accumulate. */ void clusterBlacklistCleanup(void) { dictIterator *di; @@ -1212,8 +1301,11 @@ void markNodeAsFailingIfNeeded(clusterNode *node) { node->fail_time = mstime(); /* Broadcast the failing node name to everybody, forcing all the other - * reachable nodes to flag the node as FAIL. */ - if (nodeIsMaster(myself)) clusterSendFail(node->name); + * reachable nodes to flag the node as FAIL. + * We do that even if this node is a replica and not a master: anyway + * the failing state is triggered collecting failure reports from masters, + * so here the replica is only helping propagating this status. */ + clusterSendFail(node->name); clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG); } @@ -1272,12 +1364,12 @@ int clusterHandshakeInProgress(char *ip, int port, int cport) { return de != NULL; } -/* Start an handshake with the specified address if there is not one +/* Start a handshake with the specified address if there is not one * already in progress. Returns non-zero if the handshake was actually * started. On error zero is returned and errno is set to one of the * following values: * - * EAGAIN - There is already an handshake in progress for this address. + * EAGAIN - There is already a handshake in progress for this address. * EINVAL - IP or port are not valid. */ int clusterStartHandshake(char *ip, int port, int cport) { clusterNode *n; @@ -1416,12 +1508,16 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) { if (node->link) freeClusterLink(node->link); memcpy(node->ip,g->ip,NET_IP_STR_LEN); node->port = ntohs(g->port); + node->pport = ntohs(g->pport); node->cport = ntohs(g->cport); node->flags &= ~CLUSTER_NODE_NOADDR; } } else { /* If it's not in NOADDR state and we don't have it, we - * start a handshake process against this IP/PORT pairs. + * add it to our trusted dict with exact nodeid and flag. + * Note that we cannot simply start a handshake against + * this IP/PORT pairs, since IP/PORT can be reused already, + * otherwise we risk joining another cluster. * * Note that we require that the sender of this gossip message * is a well known node in our cluster, otherwise we risk @@ -1430,7 +1526,13 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) { !(flags & CLUSTER_NODE_NOADDR) && !clusterBlacklistExists(g->nodename)) { - clusterStartHandshake(g->ip,ntohs(g->port),ntohs(g->cport)); + clusterNode *node; + node = createClusterNode(g->nodename, flags); + memcpy(node->ip,g->ip,NET_IP_STR_LEN); + node->port = ntohs(g->port); + node->pport = ntohs(g->pport); + node->cport = ntohs(g->cport); + clusterAddNode(node); } } @@ -1447,7 +1549,7 @@ void nodeIp2String(char *buf, clusterLink *link, char *announced_ip) { memcpy(buf,announced_ip,NET_IP_STR_LEN); buf[NET_IP_STR_LEN-1] = '\0'; /* We are not sure the input is sane. */ } else { - anetPeerToString(link->fd, buf, NET_IP_STR_LEN, NULL); + connPeerToString(link->conn, buf, NET_IP_STR_LEN, NULL); } } @@ -1468,6 +1570,7 @@ int nodeUpdateAddressIfNeeded(clusterNode *node, clusterLink *link, { char ip[NET_IP_STR_LEN] = {0}; int port = ntohs(hdr->port); + int pport = ntohs(hdr->pport); int cport = ntohs(hdr->cport); /* We don't proceed if the link is the same as the sender link, as this @@ -1479,12 +1582,13 @@ int nodeUpdateAddressIfNeeded(clusterNode *node, clusterLink *link, if (link == node->link) return 0; nodeIp2String(ip,link,hdr->myip); - if (node->port == port && node->cport == cport && + if (node->port == port && node->cport == cport && node->pport == pport && strcmp(ip,node->ip) == 0) return 0; /* IP / port is different, update it. */ memcpy(node->ip,ip,sizeof(ip)); node->port = port; + node->pport = pport; node->cport = cport; if (node->link) freeClusterLink(node->link); node->flags &= ~CLUSTER_NODE_NOADDR; @@ -1530,7 +1634,7 @@ void clusterSetNodeAsMaster(clusterNode *n) { * case we receive the info via an UPDATE packet. */ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoch, unsigned char *slots) { int j; - clusterNode *curmaster, *newmaster = NULL; + clusterNode *curmaster = NULL, *newmaster = NULL; /* The dirty slots list is a list of slots for which we lose the ownership * while having still keys inside. This usually happens after a failover * or after a manual cluster reconfiguration operated by the admin. @@ -1541,6 +1645,12 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc uint16_t dirty_slots[CLUSTER_SLOTS]; int dirty_slots_count = 0; + /* We should detect if sender is new master of our shard. + * We will know it if all our slots were migrated to sender, and sender + * has no slots except ours */ + int sender_slots = 0; + int migrated_our_slots = 0; + /* Here we set curmaster to this node or the node this node * replicates to if it's a slave. In the for loop we are * interested to check if slots are taken away from curmaster. */ @@ -1553,6 +1663,8 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc for (j = 0; j < CLUSTER_SLOTS; j++) { if (bitmapTestBit(slots,j)) { + sender_slots++; + /* The slot is already bound to the sender of this message. */ if (server.cluster->slots[j] == sender) continue; @@ -1579,8 +1691,10 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc dirty_slots_count++; } - if (server.cluster->slots[j] == curmaster) + if (server.cluster->slots[j] == curmaster) { newmaster = sender; + migrated_our_slots++; + } clusterDelSlot(j); clusterAddSlot(sender,j); clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG| @@ -1603,7 +1717,9 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc * master. * 2) We are a slave and our master is left without slots. We need * to replicate to the new slots owner. */ - if (newmaster && curmaster->numslots == 0) { + if (newmaster && curmaster->numslots == 0 && + (server.cluster_allow_replica_migration || + sender_slots == migrated_our_slots)) { serverLog(LL_WARNING, "Configuration change detected. Reconfiguring myself " "as a replica of %.40s", sender->name); @@ -1637,6 +1753,7 @@ int clusterProcessPacket(clusterLink *link) { clusterMsg *hdr = (clusterMsg*) link->rcvbuf; uint32_t totlen = ntohl(hdr->totlen); uint16_t type = ntohs(hdr->type); + mstime_t now = mstime(); if (type < CLUSTERMSG_TYPE_COUNT) server.cluster->stats_bus_messages_received[type]++; @@ -1645,7 +1762,7 @@ int clusterProcessPacket(clusterLink *link) { /* Perform sanity checks */ if (totlen < 16) return 1; /* At least signature, version, totlen, count. */ - if (totlen > sdslen(link->rcvbuf)) return 1; + if (totlen > link->rcvbuf_len) return 1; if (ntohs(hdr->ver) != CLUSTER_PROTO_VER) { /* Can't handle messages of different versions. */ @@ -1693,15 +1810,24 @@ int clusterProcessPacket(clusterLink *link) { } else if (type == CLUSTERMSG_TYPE_MODULE) { uint32_t explen = sizeof(clusterMsg)-sizeof(union clusterMsgData); - explen += sizeof(clusterMsgDataPublish) - + explen += sizeof(clusterMsgModule) - 3 + ntohl(hdr->data.module.msg.len); if (totlen != explen) return 1; } - /* Check if the sender is a known node. */ + /* Check if the sender is a known node. Note that for incoming connections + * we don't store link->node information, but resolve the node by the + * ID in the header each time in the current implementation. */ sender = clusterLookupNode(hdr->sender); + + /* Update the last time we saw any data from this node. We + * use this in order to avoid detecting a timeout from a node that + * is just sending a lot of data in the cluster bus, for instance + * because of Pub/Sub. */ + if (sender) sender->data_received = now; + if (sender && !nodeInHandshake(sender)) { - /* Update our curretEpoch if we see a newer epoch in the cluster. */ + /* Update our currentEpoch if we see a newer epoch in the cluster. */ senderCurrentEpoch = ntohu64(hdr->currentEpoch); senderConfigEpoch = ntohu64(hdr->configEpoch); if (senderCurrentEpoch > server.cluster->currentEpoch) @@ -1714,16 +1840,17 @@ int clusterProcessPacket(clusterLink *link) { } /* Update the replication offset info for this node. */ sender->repl_offset = ntohu64(hdr->offset); - sender->repl_offset_time = mstime(); + sender->repl_offset_time = now; /* If we are a slave performing a manual failover and our master * sent its offset while already paused, populate the MF state. */ if (server.cluster->mf_end && nodeIsSlave(myself) && myself->slaveof == sender && hdr->mflags[0] & CLUSTERMSG_FLAG0_PAUSED && - server.cluster->mf_master_offset == 0) + server.cluster->mf_master_offset == -1) { server.cluster->mf_master_offset = sender->repl_offset; + clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_MANUALFAILOVER); serverLog(LL_WARNING, "Received replication offset for paused " "master manual failover: %lld", @@ -1751,7 +1878,7 @@ int clusterProcessPacket(clusterLink *link) { { char ip[NET_IP_STR_LEN]; - if (anetSockName(link->fd,ip,sizeof(ip),NULL) != -1 && + if (connSockName(link->conn,ip,sizeof(ip),NULL) != -1 && strcmp(ip,myself->ip)) { memcpy(myself->ip,ip,NET_IP_STR_LEN); @@ -1771,6 +1898,7 @@ int clusterProcessPacket(clusterLink *link) { node = createClusterNode(NULL,CLUSTER_NODE_HANDSHAKE); nodeIp2String(node->ip,link,hdr->myip); node->port = ntohs(hdr->port); + node->pport = ntohs(hdr->pport); node->cport = ntohs(hdr->cport); clusterAddNode(node); clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG); @@ -1828,11 +1956,12 @@ int clusterProcessPacket(clusterLink *link) { * address. */ serverLog(LL_DEBUG,"PONG contains mismatching sender ID. About node %.40s added %d ms ago, having flags %d", link->node->name, - (int)(mstime()-(link->node->ctime)), + (int)(now-(link->node->ctime)), link->node->flags); link->node->flags |= CLUSTER_NODE_NOADDR; link->node->ip[0] = '\0'; link->node->port = 0; + link->node->pport = 0; link->node->cport = 0; freeClusterLink(link); clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG); @@ -1863,7 +1992,7 @@ int clusterProcessPacket(clusterLink *link) { /* Update our info about the node */ if (link->node && type == CLUSTERMSG_TYPE_PONG) { - link->node->pong_received = mstime(); + link->node->pong_received = now; link->node->ping_sent = 0; /* The PFAIL condition can be reversed without external @@ -2010,7 +2139,7 @@ int clusterProcessPacket(clusterLink *link) { "FAIL message received from %.40s about %.40s", hdr->sender, hdr->data.fail.about.nodename); failing->flags |= CLUSTER_NODE_FAIL; - failing->fail_time = mstime(); + failing->fail_time = now; failing->flags &= ~CLUSTER_NODE_PFAIL; clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG| CLUSTER_TODO_UPDATE_STATE); @@ -2027,7 +2156,7 @@ int clusterProcessPacket(clusterLink *link) { /* Don't bother creating useless objects if there are no * Pub/Sub subscribers. */ if (dictSize(server.pubsub_channels) || - listLength(server.pubsub_patterns)) + dictSize(server.pubsub_patterns)) { channel_len = ntohl(hdr->data.publish.msg.channel_len); message_len = ntohl(hdr->data.publish.msg.message_len); @@ -2063,11 +2192,17 @@ int clusterProcessPacket(clusterLink *link) { /* Manual failover requested from slaves. Initialize the state * accordingly. */ resetManualFailover(); - server.cluster->mf_end = mstime() + CLUSTER_MF_TIMEOUT; + server.cluster->mf_end = now + CLUSTER_MF_TIMEOUT; server.cluster->mf_slave = sender; - pauseClients(mstime()+(CLUSTER_MF_TIMEOUT*2)); + pauseClients(now+(CLUSTER_MF_TIMEOUT*CLUSTER_MF_PAUSE_MULT),CLIENT_PAUSE_WRITE); serverLog(LL_WARNING,"Manual failover requested by replica %.40s.", sender->name); + /* We need to send a ping message to the replica, as it would carry + * `server.cluster->mf_master_offset`, which means the master paused clients + * at offset `server.cluster->mf_master_offset`, so that the replica would + * know that it is safe to set its `server.cluster->mf_can_start` to 1 so as + * to complete failover as quickly as possible. */ + clusterSendPing(link, CLUSTERMSG_TYPE_PING); } else if (type == CLUSTERMSG_TYPE_UPDATE) { clusterNode *n; /* The node the update is about. */ uint64_t reportedConfigEpoch = @@ -2118,38 +2253,79 @@ void handleLinkIOError(clusterLink *link) { /* Send data. This is handled using a trivial send buffer that gets * consumed by write(). We don't try to optimize this for speed too much * as this is a very low traffic channel. */ -void clusterWriteHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - clusterLink *link = (clusterLink*) privdata; +void clusterWriteHandler(connection *conn) { + clusterLink *link = connGetPrivateData(conn); ssize_t nwritten; - UNUSED(el); - UNUSED(mask); - nwritten = write(fd, link->sndbuf, sdslen(link->sndbuf)); + nwritten = connWrite(conn, link->sndbuf, sdslen(link->sndbuf)); if (nwritten <= 0) { serverLog(LL_DEBUG,"I/O error writing to node link: %s", - (nwritten == -1) ? strerror(errno) : "short write"); + (nwritten == -1) ? connGetLastError(conn) : "short write"); handleLinkIOError(link); return; } sdsrange(link->sndbuf,nwritten,-1); if (sdslen(link->sndbuf) == 0) - aeDeleteFileEvent(server.el, link->fd, AE_WRITABLE); + connSetWriteHandler(link->conn, NULL); +} + +/* A connect handler that gets called when a connection to another node + * gets established. + */ +void clusterLinkConnectHandler(connection *conn) { + clusterLink *link = connGetPrivateData(conn); + clusterNode *node = link->node; + + /* Check if connection succeeded */ + if (connGetState(conn) != CONN_STATE_CONNECTED) { + serverLog(LL_VERBOSE, "Connection with Node %.40s at %s:%d failed: %s", + node->name, node->ip, node->cport, + connGetLastError(conn)); + freeClusterLink(link); + return; + } + + /* Register a read handler from now on */ + connSetReadHandler(conn, clusterReadHandler); + + /* Queue a PING in the new connection ASAP: this is crucial + * to avoid false positives in failure detection. + * + * If the node is flagged as MEET, we send a MEET message instead + * of a PING one, to force the receiver to add us in its node + * table. */ + mstime_t old_ping_sent = node->ping_sent; + clusterSendPing(link, node->flags & CLUSTER_NODE_MEET ? + CLUSTERMSG_TYPE_MEET : CLUSTERMSG_TYPE_PING); + if (old_ping_sent) { + /* If there was an active ping before the link was + * disconnected, we want to restore the ping time, otherwise + * replaced by the clusterSendPing() call. */ + node->ping_sent = old_ping_sent; + } + /* We can clear the flag after the first packet is sent. + * If we'll never receive a PONG, we'll never send new packets + * to this node. Instead after the PONG is received and we + * are no longer in meet/handshake status, we want to send + * normal PING packets. */ + node->flags &= ~CLUSTER_NODE_MEET; + + serverLog(LL_DEBUG,"Connecting with Node %.40s at %s:%d", + node->name, node->ip, node->cport); } /* Read data. Try to read the first field of the header first to check the * full length of the packet. When a whole packet is in memory this function * will call the function to process the packet. And so forth. */ -void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - char buf[sizeof(clusterMsg)]; +void clusterReadHandler(connection *conn) { + clusterMsg buf[1]; ssize_t nread; clusterMsg *hdr; - clusterLink *link = (clusterLink*) privdata; + clusterLink *link = connGetPrivateData(conn); unsigned int readlen, rcvbuflen; - UNUSED(el); - UNUSED(mask); while(1) { /* Read as long as there is data to read. */ - rcvbuflen = sdslen(link->rcvbuf); + rcvbuflen = link->rcvbuf_len; if (rcvbuflen < 8) { /* First, obtain the first 8 bytes to get the full message * length. */ @@ -2174,18 +2350,26 @@ void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { if (readlen > sizeof(buf)) readlen = sizeof(buf); } - nread = read(fd,buf,readlen); - if (nread == -1 && errno == EAGAIN) return; /* No more data ready. */ + nread = connRead(conn,buf,readlen); + if (nread == -1 && (connGetState(conn) == CONN_STATE_CONNECTED)) return; /* No more data ready. */ if (nread <= 0) { /* I/O error... */ serverLog(LL_DEBUG,"I/O error reading from node link: %s", - (nread == 0) ? "connection closed" : strerror(errno)); + (nread == 0) ? "connection closed" : connGetLastError(conn)); handleLinkIOError(link); return; } else { /* Read data and recast the pointer to the new buffer. */ - link->rcvbuf = sdscatlen(link->rcvbuf,buf,nread); + size_t unused = link->rcvbuf_alloc - link->rcvbuf_len; + if ((size_t)nread > unused) { + size_t required = link->rcvbuf_len + nread; + /* If less than 1mb, grow to twice the needed size, if larger grow by 1mb. */ + link->rcvbuf_alloc = required < RCVBUF_MAX_PREALLOC ? required * 2: required + RCVBUF_MAX_PREALLOC; + link->rcvbuf = zrealloc(link->rcvbuf, link->rcvbuf_alloc); + } + memcpy(link->rcvbuf + link->rcvbuf_len, buf, nread); + link->rcvbuf_len += nread; hdr = (clusterMsg*) link->rcvbuf; rcvbuflen += nread; } @@ -2193,8 +2377,11 @@ void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { /* Total length obtained? Process this packet. */ if (rcvbuflen >= 8 && rcvbuflen == ntohl(hdr->totlen)) { if (clusterProcessPacket(link)) { - sdsfree(link->rcvbuf); - link->rcvbuf = sdsempty(); + if (link->rcvbuf_alloc > RCVBUF_INIT_LEN) { + zfree(link->rcvbuf); + link->rcvbuf = zmalloc(link->rcvbuf_alloc = RCVBUF_INIT_LEN); + } + link->rcvbuf_len = 0; } else { return; /* Link no longer valid. */ } @@ -2209,8 +2396,7 @@ void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { * from event handlers that will do stuff with the same link later. */ void clusterSendMessage(clusterLink *link, unsigned char *msg, size_t msglen) { if (sdslen(link->sndbuf) == 0 && msglen != 0) - aeCreateFileEvent(server.el,link->fd,AE_WRITABLE|AE_BARRIER, - clusterWriteHandler,link); + connSetWriteHandlerWithBarrier(link->conn, clusterWriteHandler, 1); link->sndbuf = sdscatlen(link->sndbuf, msg, msglen); @@ -2275,18 +2461,16 @@ void clusterBuildMessageHdr(clusterMsg *hdr, int type) { hdr->myip[NET_IP_STR_LEN-1] = '\0'; } - /* Handle cluster-announce-port as well. */ - int announced_port = server.cluster_announce_port ? - server.cluster_announce_port : server.port; - int announced_cport = server.cluster_announce_bus_port ? - server.cluster_announce_bus_port : - (server.port + CLUSTER_PORT_INCR); + /* Handle cluster-announce-[tls-|bus-]port. */ + int announced_port, announced_pport, announced_cport; + deriveAnnouncedPorts(&announced_port, &announced_pport, &announced_cport); memcpy(hdr->myslots,master->slots,sizeof(hdr->myslots)); memset(hdr->slaveof,0,CLUSTER_NAMELEN); if (myself->slaveof != NULL) memcpy(hdr->slaveof,myself->slaveof->name, CLUSTER_NAMELEN); hdr->port = htons(announced_port); + hdr->pport = htons(announced_pport); hdr->cport = htons(announced_cport); hdr->flags = htons(myself->flags); hdr->state = server.cluster->state; @@ -2343,11 +2527,12 @@ void clusterSetGossipEntry(clusterMsg *hdr, int i, clusterNode *n) { gossip->port = htons(n->port); gossip->cport = htons(n->cport); gossip->flags = htons(n->flags); + gossip->pport = htons(n->pport); gossip->notused1 = 0; } /* Send a PING or PONG packet to the specified node, making sure to add enough - * gossip informations. */ + * gossip information. */ void clusterSendPing(clusterLink *link, int type) { unsigned char *buf; clusterMsg *hdr; @@ -2367,7 +2552,7 @@ void clusterSendPing(clusterLink *link, int type) { * node_timeout we exchange with each other node at least 4 packets * (we ping in the worst case in node_timeout/2 time, and we also * receive two pings from the host), we have a total of 8 packets - * in the node_timeout*2 falure reports validity time. So we have + * in the node_timeout*2 failure reports validity time. So we have * that, for a single PFAIL node, we can expect to receive the following * number of failure reports (in the specified window of time): * @@ -2394,7 +2579,7 @@ void clusterSendPing(clusterLink *link, int type) { * faster to propagate to go from PFAIL to FAIL state. */ int pfail_wanted = server.cluster->stats_pfail_nodes; - /* Compute the maxium totlen to allocate our buffer. We'll fix the totlen + /* Compute the maximum totlen to allocate our buffer. We'll fix the totlen * later according to the number of gossip sections we really were able * to put inside the packet. */ totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); @@ -2431,7 +2616,7 @@ void clusterSendPing(clusterLink *link, int type) { if (this->flags & (CLUSTER_NODE_HANDSHAKE|CLUSTER_NODE_NOADDR) || (this->link == NULL && this->numslots == 0)) { - freshnodes--; /* Tecnically not correct, but saves CPU. */ + freshnodes--; /* Technically not correct, but saves CPU. */ continue; } @@ -2517,7 +2702,8 @@ void clusterBroadcastPong(int target) { * * If link is NULL, then the message is broadcasted to the whole cluster. */ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { - unsigned char buf[sizeof(clusterMsg)], *payload; + unsigned char *payload; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; uint32_t channel_len, message_len; @@ -2537,7 +2723,7 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { /* Try to use the local buffer if possible */ if (totlen < sizeof(buf)) { - payload = buf; + payload = (unsigned char*)buf; } else { payload = zmalloc(totlen); memcpy(payload,hdr,sizeof(*hdr)); @@ -2554,7 +2740,7 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { decrRefCount(channel); decrRefCount(message); - if (payload != buf) zfree(payload); + if (payload != (unsigned char*)buf) zfree(payload); } /* Send a FAIL message to all the nodes we are able to contact. @@ -2563,7 +2749,7 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { * we switch the node state to CLUSTER_NODE_FAIL and ask all the other * nodes to do the same ASAP. */ void clusterSendFail(char *nodename) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_FAIL); @@ -2575,7 +2761,7 @@ void clusterSendFail(char *nodename) { * slots configuration. The node name, slots bitmap, and configEpoch info * are included. */ void clusterSendUpdate(clusterLink *link, clusterNode *node) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; if (link == NULL) return; @@ -2583,7 +2769,7 @@ void clusterSendUpdate(clusterLink *link, clusterNode *node) { memcpy(hdr->data.update.nodecfg.nodename,node->name,CLUSTER_NAMELEN); hdr->data.update.nodecfg.configEpoch = htonu64(node->configEpoch); memcpy(hdr->data.update.nodecfg.slots,node->slots,sizeof(node->slots)); - clusterSendMessage(link,buf,ntohl(hdr->totlen)); + clusterSendMessage(link,(unsigned char*)buf,ntohl(hdr->totlen)); } /* Send a MODULE message. @@ -2591,7 +2777,8 @@ void clusterSendUpdate(clusterLink *link, clusterNode *node) { * If link is NULL, then the message is broadcasted to the whole cluster. */ void clusterSendModule(clusterLink *link, uint64_t module_id, uint8_t type, unsigned char *payload, uint32_t len) { - unsigned char buf[sizeof(clusterMsg)], *heapbuf; + unsigned char *heapbuf; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2606,7 +2793,7 @@ void clusterSendModule(clusterLink *link, uint64_t module_id, uint8_t type, /* Try to use the local buffer if possible */ if (totlen < sizeof(buf)) { - heapbuf = buf; + heapbuf = (unsigned char*)buf; } else { heapbuf = zmalloc(totlen); memcpy(heapbuf,hdr,sizeof(*hdr)); @@ -2619,7 +2806,7 @@ void clusterSendModule(clusterLink *link, uint64_t module_id, uint8_t type, else clusterBroadcastMessage(heapbuf,totlen); - if (heapbuf != buf) zfree(heapbuf); + if (heapbuf != (unsigned char*)buf) zfree(heapbuf); } /* This function gets a cluster node ID string as target, the same way the nodes @@ -2656,14 +2843,14 @@ void clusterPropagatePublish(robj *channel, robj *message) { * SLAVE node specific functions * -------------------------------------------------------------------------- */ -/* This function sends a FAILOVE_AUTH_REQUEST message to every node in order to +/* This function sends a FAILOVER_AUTH_REQUEST message to every node in order to * see if there is the quorum for this slave instance to failover its failing * master. * * Note that we send the failover request to everybody, master and slave nodes, * but only the masters are supposed to reply to our query. */ void clusterRequestFailoverAuth(void) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2679,7 +2866,7 @@ void clusterRequestFailoverAuth(void) { /* Send a FAILOVER_AUTH_ACK message to the specified node. */ void clusterSendFailoverAuth(clusterNode *node) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2687,12 +2874,12 @@ void clusterSendFailoverAuth(clusterNode *node) { clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK); totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); hdr->totlen = htonl(totlen); - clusterSendMessage(node->link,buf,totlen); + clusterSendMessage(node->link,(unsigned char*)buf,totlen); } /* Send a MFSTART message to the specified node. */ void clusterSendMFStart(clusterNode *node) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2700,7 +2887,7 @@ void clusterSendMFStart(clusterNode *node) { clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_MFSTART); totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); hdr->totlen = htonl(totlen); - clusterSendMessage(node->link,buf,totlen); + clusterSendMessage(node->link,(unsigned char*)buf,totlen); } /* Vote for the node asking for our vote if there are the conditions. */ @@ -2939,7 +3126,7 @@ void clusterFailoverReplaceYourMaster(void) { /* This function is called if we are a slave node and our master serving * a non-zero amount of hash slots is in FAIL state. * - * The gaol of this function is: + * The goal of this function is: * 1) To check if we are able to perform a failover, is our data updated? * 2) Try to get elected by masters. * 3) Perform the failover informing all the other nodes. @@ -2984,7 +3171,7 @@ void clusterHandleSlaveFailover(void) { return; } - /* Set data_age to the number of seconds we are disconnected from + /* Set data_age to the number of milliseconds we are disconnected from * the master. */ if (server.repl_state == REPL_STATE_CONNECTED) { data_age = (mstime_t)(server.unixtime - server.master->lastinteraction) @@ -3014,7 +3201,7 @@ void clusterHandleSlaveFailover(void) { } } - /* If the previous failover attempt timedout and the retry time has + /* If the previous failover attempt timeout and the retry time has * elapsed, we can setup a new one. */ if (auth_age > auth_retry_time) { server.cluster->failover_auth_time = mstime() + @@ -3120,7 +3307,7 @@ void clusterHandleSlaveFailover(void) { * * Slave migration is the process that allows a slave of a master that is * already covered by at least another slave, to "migrate" to a master that - * is orpaned, that is, left with no working slaves. + * is orphaned, that is, left with no working slaves. * ------------------------------------------------------------------------- */ /* This function is responsible to decide if this replica should be migrated @@ -3137,7 +3324,7 @@ void clusterHandleSlaveFailover(void) { * the nodes anyway, so we spend time into clusterHandleSlaveMigration() * if definitely needed. * - * The fuction is called with a pre-computed max_slaves, that is the max + * The function is called with a pre-computed max_slaves, that is the max * number of working (not in FAIL state) slaves for a single master. * * Additional conditions for migration are examined inside the function. @@ -3256,20 +3443,19 @@ void clusterHandleSlaveMigration(int max_slaves) { * data loss due to the asynchronous master-slave replication. * -------------------------------------------------------------------------- */ -/* Reset the manual failover state. This works for both masters and slavesa +/* Reset the manual failover state. This works for both masters and slaves * as all the state about manual failover is cleared. * * The function can be used both to initialize the manual failover state at * startup or to abort a manual failover in progress. */ void resetManualFailover(void) { - if (server.cluster->mf_end && clientsArePaused()) { - server.clients_pause_end_time = 0; - clientsArePaused(); /* Just use the side effect of the function. */ + if (server.cluster->mf_end) { + checkClientPauseTimeoutAndReturnIfPaused(); } server.cluster->mf_end = 0; /* No manual failover in progress. */ server.cluster->mf_can_start = 0; server.cluster->mf_slave = NULL; - server.cluster->mf_master_offset = 0; + server.cluster->mf_master_offset = -1; } /* If a manual failover timed out, abort it. */ @@ -3290,7 +3476,7 @@ void clusterHandleManualFailover(void) { * next steps are performed by clusterHandleSlaveFailover(). */ if (server.cluster->mf_can_start) return; - if (server.cluster->mf_master_offset == 0) return; /* Wait for offset... */ + if (server.cluster->mf_master_offset == -1) return; /* Wait for offset... */ if (server.cluster->mf_master_offset == replicationGetSlaveOffset()) { /* Our replication offset matches the master replication offset @@ -3299,7 +3485,10 @@ void clusterHandleManualFailover(void) { serverLog(LL_WARNING, "All master replication stream processed, " "manual failover can start."); + clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_FAILOVER); + return; } + clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_MANUALFAILOVER); } /* ----------------------------------------------------------------------------- @@ -3383,13 +3572,11 @@ void clusterCron(void) { } if (node->link == NULL) { - int fd; - mstime_t old_ping_sent; - clusterLink *link; - - fd = anetTcpNonBlockBindConnect(server.neterr, node->ip, - node->cport, NET_FIRST_BIND_ADDR); - if (fd == -1) { + clusterLink *link = createClusterLink(node); + link->conn = server.tls_cluster ? connCreateTLS() : connCreateSocket(); + connSetPrivateData(link->conn, link); + if (connConnect(link->conn, node->ip, node->cport, NET_FIRST_BIND_ADDR, + clusterLinkConnectHandler) == -1) { /* We got a synchronous error from connect before * clusterSendPing() had a chance to be called. * If node->ping_sent is zero, failure detection can't work, @@ -3399,37 +3586,11 @@ void clusterCron(void) { serverLog(LL_DEBUG, "Unable to connect to " "Cluster Node [%s]:%d -> %s", node->ip, node->cport, server.neterr); + + freeClusterLink(link); continue; } - link = createClusterLink(node); - link->fd = fd; node->link = link; - aeCreateFileEvent(server.el,link->fd,AE_READABLE, - clusterReadHandler,link); - /* Queue a PING in the new connection ASAP: this is crucial - * to avoid false positives in failure detection. - * - * If the node is flagged as MEET, we send a MEET message instead - * of a PING one, to force the receiver to add us in its node - * table. */ - old_ping_sent = node->ping_sent; - clusterSendPing(link, node->flags & CLUSTER_NODE_MEET ? - CLUSTERMSG_TYPE_MEET : CLUSTERMSG_TYPE_PING); - if (old_ping_sent) { - /* If there was an active ping before the link was - * disconnected, we want to restore the ping time, otherwise - * replaced by the clusterSendPing() call. */ - node->ping_sent = old_ping_sent; - } - /* We can clear the flag after the first packet is sent. - * If we'll never receive a PONG, we'll never send new packets - * to this node. Instead after the PONG is received and we - * are no longer in meet/handshake status, we want to send - * normal PING packets. */ - node->flags &= ~CLUSTER_NODE_MEET; - - serverLog(LL_DEBUG,"Connecting with Node %.40s at %s:%d", - node->name, node->ip, node->cport); } } dictReleaseIterator(di); @@ -3473,7 +3634,6 @@ void clusterCron(void) { while((de = dictNext(di)) != NULL) { clusterNode *node = dictGetVal(de); now = mstime(); /* Use an updated time at every iteration. */ - mstime_t delay; if (node->flags & (CLUSTER_NODE_MYSELF|CLUSTER_NODE_NOADDR|CLUSTER_NODE_HANDSHAKE)) @@ -3497,16 +3657,19 @@ void clusterCron(void) { this_slaves = okslaves; } - /* If we are waiting for the PONG more than half the cluster + /* If we are not receiving any data for more than half the cluster * timeout, reconnect the link: maybe there is a connection * issue even if the node is alive. */ + mstime_t ping_delay = now - node->ping_sent; + mstime_t data_delay = now - node->data_received; if (node->link && /* is connected */ now - node->link->ctime > server.cluster_node_timeout && /* was not already reconnected */ node->ping_sent && /* we already sent a ping */ - node->pong_received < node->ping_sent && /* still waiting pong */ /* and we are waiting for the pong more than timeout/2 */ - now - node->ping_sent > server.cluster_node_timeout/2) + ping_delay > server.cluster_node_timeout/2 && + /* and in such interval we are not seeing any traffic at all. */ + data_delay > server.cluster_node_timeout/2) { /* Disconnect the link, it will be reconnected automatically. */ freeClusterLink(node->link); @@ -3538,12 +3701,18 @@ void clusterCron(void) { /* Check only if we have an active ping for this instance. */ if (node->ping_sent == 0) continue; - /* Compute the delay of the PONG. Note that if we already received - * the PONG, then node->ping_sent is zero, so can't reach this - * code at all. */ - delay = now - node->ping_sent; + /* Check if this node looks unreachable. + * Note that if we already received the PONG, then node->ping_sent + * is zero, so can't reach this code at all, so we don't risk of + * checking for a PONG delay if we didn't sent the PING. + * + * We also consider every incoming data as proof of liveness, since + * our cluster bus link is also used for data: under heavy data + * load pong delays are possible. */ + mstime_t node_delay = (ping_delay < data_delay) ? ping_delay : + data_delay; - if (delay > server.cluster_node_timeout) { + if (node_delay > server.cluster_node_timeout) { /* Timeout reached. Set the node as possibly failing if it is * not already in this state. */ if (!(node->flags & (CLUSTER_NODE_PFAIL|CLUSTER_NODE_FAIL))) { @@ -3567,7 +3736,7 @@ void clusterCron(void) { replicationSetMaster(myself->slaveof->ip, myself->slaveof->port); } - /* Abourt a manual failover if the timeout is reached. */ + /* Abort a manual failover if the timeout is reached. */ manualFailoverCheckTimeout(); if (nodeIsSlave(myself)) { @@ -3579,7 +3748,8 @@ void clusterCron(void) { * the orphaned masters. Note that it does not make sense to try * a migration if there is no master with at least *two* working * slaves. */ - if (orphaned_masters && max_slaves >= 2 && this_slaves == max_slaves) + if (orphaned_masters && max_slaves >= 2 && this_slaves == max_slaves && + server.cluster_allow_replica_migration) clusterHandleSlaveMigration(max_slaves); } @@ -3593,25 +3763,35 @@ void clusterCron(void) { * handlers, or to perform potentially expansive tasks that we need to do * a single time before replying to clients. */ void clusterBeforeSleep(void) { - /* Handle failover, this is needed when it is likely that there is already - * the quorum from masters in order to react fast. */ - if (server.cluster->todo_before_sleep & CLUSTER_TODO_HANDLE_FAILOVER) + int flags = server.cluster->todo_before_sleep; + + /* Reset our flags (not strictly needed since every single function + * called for flags set should be able to clear its flag). */ + server.cluster->todo_before_sleep = 0; + + if (flags & CLUSTER_TODO_HANDLE_MANUALFAILOVER) { + /* Handle manual failover as soon as possible so that won't have a 100ms + * as it was handled only in clusterCron */ + if(nodeIsSlave(myself)) { + clusterHandleManualFailover(); + if (!(server.cluster_module_flags & CLUSTER_MODULE_FLAG_NO_FAILOVER)) + clusterHandleSlaveFailover(); + } + } else if (flags & CLUSTER_TODO_HANDLE_FAILOVER) { + /* Handle failover, this is needed when it is likely that there is already + * the quorum from masters in order to react fast. */ clusterHandleSlaveFailover(); + } /* Update the cluster state. */ - if (server.cluster->todo_before_sleep & CLUSTER_TODO_UPDATE_STATE) + if (flags & CLUSTER_TODO_UPDATE_STATE) clusterUpdateState(); /* Save the config, possibly using fsync. */ - if (server.cluster->todo_before_sleep & CLUSTER_TODO_SAVE_CONFIG) { - int fsync = server.cluster->todo_before_sleep & - CLUSTER_TODO_FSYNC_CONFIG; + if (flags & CLUSTER_TODO_SAVE_CONFIG) { + int fsync = flags & CLUSTER_TODO_FSYNC_CONFIG; clusterSaveConfigOrDie(fsync); } - - /* Reset our flags (not strictly needed since every single function - * called for flags set should be able to clear its flag). */ - server.cluster->todo_before_sleep = 0; } void clusterDoBeforeSleep(int flags) { @@ -3672,14 +3852,14 @@ int clusterNodeSetSlotBit(clusterNode *n, int slot) { * target for replicas migration, if and only if at least one of * the other masters has slaves right now. * - * Normally masters are valid targerts of replica migration if: + * Normally masters are valid targets of replica migration if: * 1. The used to have slaves (but no longer have). * 2. They are slaves failing over a master that used to have slaves. * * However new masters with slots assigned are considered valid - * migration tagets if the rest of the cluster is not a slave-less. + * migration targets if the rest of the cluster is not a slave-less. * - * See https://github.com/antirez/redis/issues/3043 for more info. */ + * See https://github.com/redis/redis/issues/3043 for more info. */ if (n->numslots == 1 && clusterMastersHaveSlaves()) n->flags |= CLUSTER_NODE_MIGRATE_TO; } @@ -3861,7 +4041,7 @@ void clusterUpdateState(void) { * A) If no other node is in charge according to the current cluster * configuration, we add these slots to our node. * B) If according to our config other nodes are already in charge for - * this lots, we set the slots as IMPORTING from our point of view + * this slots, we set the slots as IMPORTING from our point of view * in order to justify we have those slots, and in order to make * redis-trib aware of the issue, so that it can try to fix it. * 2) If we find data in a DB different than DB0 we return C_ERR to @@ -3987,51 +4167,62 @@ sds representClusterNodeFlags(sds ci, uint16_t flags) { * See clusterGenNodesDescription() top comment for more information. * * The function returns the string representation as an SDS string. */ -sds clusterGenNodeDescription(clusterNode *node) { +sds clusterGenNodeDescription(clusterNode *node, int use_pport) { int j, start; sds ci; + int port = use_pport && node->pport ? node->pport : node->port; /* Node coordinates */ - ci = sdscatprintf(sdsempty(),"%.40s %s:%d@%d ", - node->name, + ci = sdscatlen(sdsempty(),node->name,CLUSTER_NAMELEN); + ci = sdscatfmt(ci," %s:%i@%i ", node->ip, - node->port, + port, node->cport); /* Flags */ ci = representClusterNodeFlags(ci, node->flags); /* Slave of... or just "-" */ + ci = sdscatlen(ci," ",1); if (node->slaveof) - ci = sdscatprintf(ci," %.40s ",node->slaveof->name); + ci = sdscatlen(ci,node->slaveof->name,CLUSTER_NAMELEN); else - ci = sdscatlen(ci," - ",3); + ci = sdscatlen(ci,"-",1); + unsigned long long nodeEpoch = node->configEpoch; + if (nodeIsSlave(node) && node->slaveof) { + nodeEpoch = node->slaveof->configEpoch; + } /* Latency from the POV of this node, config epoch, link status */ - ci = sdscatprintf(ci,"%lld %lld %llu %s", + ci = sdscatfmt(ci," %I %I %U %s", (long long) node->ping_sent, (long long) node->pong_received, - (unsigned long long) node->configEpoch, + nodeEpoch, (node->link || node->flags & CLUSTER_NODE_MYSELF) ? "connected" : "disconnected"); - /* Slots served by this instance */ - start = -1; - for (j = 0; j < CLUSTER_SLOTS; j++) { - int bit; + /* Slots served by this instance. If we already have slots info, + * append it diretly, otherwise, generate slots only if it has. */ + if (node->slots_info) { + ci = sdscatsds(ci, node->slots_info); + } else if (node->numslots > 0) { + start = -1; + for (j = 0; j < CLUSTER_SLOTS; j++) { + int bit; - if ((bit = clusterNodeGetSlotBit(node,j)) != 0) { - if (start == -1) start = j; - } - if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) { - if (bit && j == CLUSTER_SLOTS-1) j++; + if ((bit = clusterNodeGetSlotBit(node,j)) != 0) { + if (start == -1) start = j; + } + if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) { + if (bit && j == CLUSTER_SLOTS-1) j++; - if (start == j-1) { - ci = sdscatprintf(ci," %d",start); - } else { - ci = sdscatprintf(ci," %d-%d",start,j-1); + if (start == j-1) { + ci = sdscatfmt(ci," %i",start); + } else { + ci = sdscatfmt(ci," %i-%i",start,j-1); + } + start = -1; } - start = -1; } } @@ -4052,6 +4243,41 @@ sds clusterGenNodeDescription(clusterNode *node) { return ci; } +/* Generate the slot topology for all nodes and store the string representation + * in the slots_info struct on the node. This is used to improve the efficiency + * of clusterGenNodesDescription() because it removes looping of the slot space + * for generating the slot info for each node individually. */ +void clusterGenNodesSlotsInfo(int filter) { + clusterNode *n = NULL; + int start = -1; + + for (int i = 0; i <= CLUSTER_SLOTS; i++) { + /* Find start node and slot id. */ + if (n == NULL) { + if (i == CLUSTER_SLOTS) break; + n = server.cluster->slots[i]; + start = i; + continue; + } + + /* Generate slots info when occur different node with start + * or end of slot. */ + if (i == CLUSTER_SLOTS || n != server.cluster->slots[i]) { + if (!(n->flags & filter)) { + if (n->slots_info == NULL) n->slots_info = sdsempty(); + if (start == i-1) { + n->slots_info = sdscatfmt(n->slots_info," %i",start); + } else { + n->slots_info = sdscatfmt(n->slots_info," %i-%i",start,i-1); + } + } + if (i == CLUSTER_SLOTS) break; + n = server.cluster->slots[i]; + start = i; + } + } +} + /* Generate a csv-alike representation of the nodes we are aware of, * including the "myself" node, and return an SDS string containing the * representation (it is up to the caller to free it). @@ -4061,23 +4287,35 @@ sds clusterGenNodeDescription(clusterNode *node) { * include all the known nodes in the representation, including nodes in * the HANDSHAKE state. * + * Setting use_pport to 1 in a TLS cluster makes the result contain the + * plaintext client port rather then the TLS client port of each node. + * * The representation obtained using this function is used for the output * of the CLUSTER NODES function, and as format for the cluster * configuration file (nodes.conf) for a given node. */ -sds clusterGenNodesDescription(int filter) { +sds clusterGenNodesDescription(int filter, int use_pport) { sds ci = sdsempty(), ni; dictIterator *di; dictEntry *de; + /* Generate all nodes slots info firstly. */ + clusterGenNodesSlotsInfo(filter); + di = dictGetSafeIterator(server.cluster->nodes); while((de = dictNext(di)) != NULL) { clusterNode *node = dictGetVal(de); if (node->flags & filter) continue; - ni = clusterGenNodeDescription(node); + ni = clusterGenNodeDescription(node, use_pport); ci = sdscatsds(ci,ni); sdsfree(ni); ci = sdscatlen(ci,"\n",1); + + /* Release slots info. */ + if (node->slots_info) { + sdsfree(node->slots_info); + node->slots_info = NULL; + } } dictReleaseIterator(di); return ci; @@ -4115,7 +4353,37 @@ int getSlotOrReply(client *c, robj *o) { return (int) slot; } -void clusterReplyMultiBulkSlots(client *c) { +void addNodeReplyForClusterSlot(client *c, clusterNode *node, int start_slot, int end_slot) { + int i, nested_elements = 3; /* slots (2) + master addr (1) */ + void *nested_replylen = addReplyDeferredLen(c); + addReplyLongLong(c, start_slot); + addReplyLongLong(c, end_slot); + addReplyArrayLen(c, 3); + addReplyBulkCString(c, node->ip); + /* Report non-TLS ports to non-TLS client in TLS cluster if available. */ + int use_pport = (server.tls_cluster && + c->conn && connGetType(c->conn) != CONN_TYPE_TLS); + addReplyLongLong(c, use_pport && node->pport ? node->pport : node->port); + addReplyBulkCBuffer(c, node->name, CLUSTER_NAMELEN); + + /* Remaining nodes in reply are replicas for slot range */ + for (i = 0; i < node->numslaves; i++) { + /* This loop is copy/pasted from clusterGenNodeDescription() + * with modifications for per-slot node aggregation. */ + if (nodeFailed(node->slaves[i])) continue; + addReplyArrayLen(c, 3); + addReplyBulkCString(c, node->slaves[i]->ip); + /* Report slave's non-TLS port to non-TLS client in TLS cluster */ + addReplyLongLong(c, (use_pport && node->slaves[i]->pport ? + node->slaves[i]->pport : + node->slaves[i]->port)); + addReplyBulkCBuffer(c, node->slaves[i]->name, CLUSTER_NAMELEN); + nested_elements++; + } + setDeferredArrayLen(c, nested_replylen, nested_elements); +} + +void clusterReplyMultiBulkSlots(client * c) { /* Format: 1) 1) start slot * 2) end slot * 3) 1) master IP @@ -4126,67 +4394,30 @@ void clusterReplyMultiBulkSlots(client *c) { * 3) node ID * ... continued until done */ + clusterNode *n = NULL; + int num_masters = 0, start = -1; + void *slot_replylen = addReplyDeferredLen(c); + + for (int i = 0; i <= CLUSTER_SLOTS; i++) { + /* Find start node and slot id. */ + if (n == NULL) { + if (i == CLUSTER_SLOTS) break; + n = server.cluster->slots[i]; + start = i; + continue; + } - int num_masters = 0; - void *slot_replylen = addDeferredMultiBulkLength(c); - - dictEntry *de; - dictIterator *di = dictGetSafeIterator(server.cluster->nodes); - while((de = dictNext(di)) != NULL) { - clusterNode *node = dictGetVal(de); - int j = 0, start = -1; - - /* Skip slaves (that are iterated when producing the output of their - * master) and masters not serving any slot. */ - if (!nodeIsMaster(node) || node->numslots == 0) continue; - - for (j = 0; j < CLUSTER_SLOTS; j++) { - int bit, i; - - if ((bit = clusterNodeGetSlotBit(node,j)) != 0) { - if (start == -1) start = j; - } - if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) { - int nested_elements = 3; /* slots (2) + master addr (1). */ - void *nested_replylen = addDeferredMultiBulkLength(c); - - if (bit && j == CLUSTER_SLOTS-1) j++; - - /* If slot exists in output map, add to it's list. - * else, create a new output map for this slot */ - if (start == j-1) { - addReplyLongLong(c, start); /* only one slot; low==high */ - addReplyLongLong(c, start); - } else { - addReplyLongLong(c, start); /* low */ - addReplyLongLong(c, j-1); /* high */ - } - start = -1; - - /* First node reply position is always the master */ - addReplyMultiBulkLen(c, 3); - addReplyBulkCString(c, node->ip); - addReplyLongLong(c, node->port); - addReplyBulkCBuffer(c, node->name, CLUSTER_NAMELEN); - - /* Remaining nodes in reply are replicas for slot range */ - for (i = 0; i < node->numslaves; i++) { - /* This loop is copy/pasted from clusterGenNodeDescription() - * with modifications for per-slot node aggregation */ - if (nodeFailed(node->slaves[i])) continue; - addReplyMultiBulkLen(c, 3); - addReplyBulkCString(c, node->slaves[i]->ip); - addReplyLongLong(c, node->slaves[i]->port); - addReplyBulkCBuffer(c, node->slaves[i]->name, CLUSTER_NAMELEN); - nested_elements++; - } - setDeferredMultiBulkLength(c, nested_replylen, nested_elements); - num_masters++; - } + /* Add cluster slots info when occur different node with start + * or end of slot. */ + if (i == CLUSTER_SLOTS || n != server.cluster->slots[i]) { + addNodeReplyForClusterSlot(c, n, start, i-1); + num_masters++; + if (i == CLUSTER_SLOTS) break; + n = server.cluster->slots[i]; + start = i; } } - dictReleaseIterator(di); - setDeferredMultiBulkLength(c, slot_replylen, num_masters); + setDeferredArrayLen(c, slot_replylen, num_masters); } void clusterCommand(client *c) { @@ -4197,27 +4428,49 @@ void clusterCommand(client *c) { if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { const char *help[] = { -"ADDSLOTS [slot ...] -- Assign slots to current node.", -"BUMPEPOCH -- Advance the cluster config epoch.", -"COUNT-failure-reports -- Return number of failure reports for .", -"COUNTKEYSINSLOT - Return the number of keys in .", -"DELSLOTS [slot ...] -- Delete slots information from current node.", -"FAILOVER [force|takeover] -- Promote current replica node to being a master.", -"FORGET -- Remove a node from the cluster.", -"GETKEYSINSLOT -- Return key names stored by current node in a slot.", -"FLUSHSLOTS -- Delete current node own slots information.", -"INFO - Return onformation about the cluster.", -"KEYSLOT -- Return the hash slot for .", -"MEET [bus-port] -- Connect nodes into a working cluster.", -"MYID -- Return the node id.", -"NODES -- Return cluster configuration seen by node. Output format:", -" ... ", -"REPLICATE -- Configure current node as replica to .", -"RESET [hard|soft] -- Reset current node (default: soft).", -"SET-config-epoch - Set config epoch of current node.", -"SETSLOT (importing|migrating|stable|node ) -- Set slot state.", -"REPLICAS -- Return replicas.", -"SLOTS -- Return information about slots range mappings. Each range is made of:", +"ADDSLOTS [ ...]", +" Assign slots to current node.", +"BUMPEPOCH", +" Advance the cluster config epoch.", +"COUNT-FAILURE-REPORTS ", +" Return number of failure reports for .", +"COUNTKEYSINSLOT ", +" Return the number of keys in .", +"DELSLOTS [ ...]", +" Delete slots information from current node.", +"FAILOVER [FORCE|TAKEOVER]", +" Promote current replica node to being a master.", +"FORGET ", +" Remove a node from the cluster.", +"GETKEYSINSLOT ", +" Return key names stored by current node in a slot.", +"FLUSHSLOTS", +" Delete current node own slots information.", +"INFO", +" Return information about the cluster.", +"KEYSLOT ", +" Return the hash slot for .", +"MEET []", +" Connect nodes into a working cluster.", +"MYID", +" Return the node id.", +"NODES", +" Return cluster configuration seen by node. Output format:", +" ...", +"REPLICATE ", +" Configure current node as replica to .", +"RESET [HARD|SOFT]", +" Reset current node (default: soft).", +"SET-CONFIG-EPOCH ", +" Set config epoch of current node.", +"SETSLOT (IMPORTING|MIGRATING|STABLE|NODE )", +" Set slot state.", +"REPLICAS ", +" Return replicas.", +"SAVECONFIG", +" Force saving cluster configuration on disk.", +"SLOTS", +" Return information about slots range mappings. Each range is made of:", " start, end, master and replicas IP addresses, ports and ids", NULL }; @@ -4252,12 +4505,13 @@ NULL } } else if (!strcasecmp(c->argv[1]->ptr,"nodes") && c->argc == 2) { /* CLUSTER NODES */ - robj *o; - sds ci = clusterGenNodesDescription(0); - - o = createObject(OBJ_STRING,ci); - addReplyBulk(c,o); - decrRefCount(o); + /* Report plaintext ports, only if cluster is TLS but client is known to + * be non-TLS). */ + int use_pport = (server.tls_cluster && + c->conn && connGetType(c->conn) != CONN_TYPE_TLS); + sds nodes = clusterGenNodesDescription(0, use_pport); + addReplyVerbatim(c,nodes,sdslen(nodes),"txt"); + sdsfree(nodes); } else if (!strcasecmp(c->argv[1]->ptr,"myid") && c->argc == 2) { /* CLUSTER MYID */ addReplyBulkCBuffer(c,myself->name, CLUSTER_NAMELEN); @@ -4386,11 +4640,14 @@ NULL } /* If this slot is in migrating status but we have no keys * for it assigning the slot to another node will clear - * the migratig status. */ + * the migrating status. */ if (countKeysInSlot(slot) == 0 && server.cluster->migrating_slots_to[slot]) server.cluster->migrating_slots_to[slot] = NULL; + clusterDelSlot(slot); + clusterAddSlot(n,slot); + /* If this node was importing this slot, assigning the slot to * itself also clears the importing status. */ if (n == myself && @@ -4410,9 +4667,10 @@ NULL "configEpoch updated after importing slot %d", slot); } server.cluster->importing_slots_from[slot] = NULL; + /* After importing this slot, let the other nodes know as + * soon as possible. */ + clusterBroadcastPong(CLUSTER_BROADCAST_ALL); } - clusterDelSlot(slot); - clusterAddSlot(n,slot); } else { addReplyError(c, "Invalid CLUSTER SETSLOT action or number of arguments. Try CLUSTER HELP"); @@ -4499,10 +4757,8 @@ NULL "cluster_stats_messages_received:%lld\r\n", tot_msg_received); /* Produce the reply protocol. */ - addReplySds(c,sdscatprintf(sdsempty(),"$%lu\r\n", - (unsigned long)sdslen(info))); - addReplySds(c,info); - addReply(c,shared.crlf); + addReplyVerbatim(c,info,sdslen(info),"txt"); + sdsfree(info); } else if (!strcasecmp(c->argv[1]->ptr,"saveconfig") && c->argc == 2) { int retval = clusterSaveConfig(1); @@ -4550,7 +4806,7 @@ NULL keys = zmalloc(sizeof(robj*)*maxkeys); numkeys = getKeysInSlot(slot, keys, maxkeys); - addReplyMultiBulkLen(c,numkeys); + addReplyArrayLen(c,numkeys); for (j = 0; j < numkeys; j++) { addReplyBulk(c,keys[j]); decrRefCount(keys[j]); @@ -4629,9 +4885,12 @@ NULL return; } - addReplyMultiBulkLen(c,n->numslaves); + /* Use plaintext port if cluster is TLS but client is non-TLS. */ + int use_pport = (server.tls_cluster && + c->conn && connGetType(c->conn) != CONN_TYPE_TLS); + addReplyArrayLen(c,n->numslaves); for (j = 0; j < n->numslaves; j++) { - sds ni = clusterGenNodeDescription(n->slaves[j]); + sds ni = clusterGenNodeDescription(n->slaves[j], use_pport); addReplyBulkCString(c,ni); sdsfree(ni); } @@ -4660,7 +4919,7 @@ NULL takeover = 1; force = 1; /* Takeover also implies force. */ } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } } @@ -4733,7 +4992,7 @@ NULL server.cluster->currentEpoch = epoch; /* No need to fsync the config here since in the unlucky event * of a failure to persist the config, the conflict resolution code - * will assign an unique config to this node. */ + * will assign a unique config to this node. */ clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE| CLUSTER_TODO_SAVE_CONFIG); addReply(c,shared.ok); @@ -4751,7 +5010,7 @@ NULL } else if (!strcasecmp(c->argv[2]->ptr,"soft")) { hard = 0; } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } } @@ -4781,7 +5040,7 @@ void createDumpPayload(rio *payload, robj *o, robj *key) { unsigned char buf[2]; uint64_t crc; - /* Serialize the object in a RDB-like format. It consist of an object type + /* Serialize the object in an RDB-like format. It consist of an object type * byte followed by the serialized object. This is understood by RESTORE. */ rioInitWithBuffer(payload,sdsempty()); serverAssert(rdbSaveObjectType(payload,o)); @@ -4823,6 +5082,9 @@ int verifyDumpPayload(unsigned char *p, size_t len) { rdbver = (footer[1] << 8) | footer[0]; if (rdbver > RDB_VERSION) return C_ERR; + if (server.skip_checksum_validation) + return C_OK; + /* Verify CRC64 */ crc = crc64(0,p,len-8); memrev64ifbe(&crc); @@ -4833,12 +5095,12 @@ int verifyDumpPayload(unsigned char *p, size_t len) { * DUMP is actually not used by Redis Cluster but it is the obvious * complement of RESTORE and can be useful for different applications. */ void dumpCommand(client *c) { - robj *o, *dumpobj; + robj *o; rio payload; /* Check if the key is here. */ if ((o = lookupKeyRead(c->db,c->argv[1])) == NULL) { - addReply(c,shared.nullbulk); + addReplyNull(c); return; } @@ -4846,9 +5108,7 @@ void dumpCommand(client *c) { createDumpPayload(&payload,o,c->argv[1]); /* Transfer to the client */ - dumpobj = createObject(OBJ_STRING,payload.io.buffer.ptr); - addReplyBulk(c,dumpobj); - decrRefCount(dumpobj); + addReplyBulkSds(c,payload.io.buffer.ptr); return; } @@ -4888,14 +5148,15 @@ void restoreCommand(client *c) { } j++; /* Consume additional arg. */ } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } } /* Make sure this key does not already exist here... */ - if (!replace && lookupKeyWrite(c->db,c->argv[1]) != NULL) { - addReply(c,shared.busykeyerr); + robj *key = c->argv[1]; + if (!replace && lookupKeyWrite(c->db,key) != NULL) { + addReplyErrorObject(c,shared.busykeyerr); return; } @@ -4916,23 +5177,38 @@ void restoreCommand(client *c) { rioInitWithBuffer(&payload,c->argv[3]->ptr); if (((type = rdbLoadObjectType(&payload)) == -1) || - ((obj = rdbLoadObject(type,&payload,c->argv[1])) == NULL)) + ((obj = rdbLoadObject(type,&payload,key->ptr,NULL)) == NULL)) { addReplyError(c,"Bad data format"); return; } /* Remove the old key if needed. */ - if (replace) dbDelete(c->db,c->argv[1]); + int deleted = 0; + if (replace) + deleted = dbDelete(c->db,key); + + if (ttl && !absttl) ttl+=mstime(); + if (ttl && checkAlreadyExpired(ttl)) { + if (deleted) { + rewriteClientCommandVector(c,2,shared.del,key); + signalModifiedKey(c,c->db,key); + notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id); + server.dirty++; + } + decrRefCount(obj); + addReply(c, shared.ok); + return; + } /* Create the key and set the TTL if any */ - dbAdd(c->db,c->argv[1],obj); + dbAdd(c->db,key,obj); if (ttl) { - if (!absttl) ttl+=mstime(); - setExpire(c,c->db,c->argv[1],ttl); + setExpire(c,c->db,key,ttl); } - objectSetLRUOrLFU(obj,lfu_freq,lru_idle,lru_clock); - signalModifiedKey(c->db,c->argv[1]); + objectSetLRUOrLFU(obj,lfu_freq,lru_idle,lru_clock,1000); + signalModifiedKey(c,c->db,key); + notifyKeyspaceEvent(NOTIFY_GENERIC,"restore",key,c->db->id); addReply(c,shared.ok); server.dirty++; } @@ -4947,7 +5223,7 @@ void restoreCommand(client *c) { #define MIGRATE_SOCKET_CACHE_TTL 10 /* close cached sockets after 10 sec. */ typedef struct migrateCachedSocket { - int fd; + connection *conn; long last_dbid; time_t last_use_time; } migrateCachedSocket; @@ -4964,7 +5240,7 @@ typedef struct migrateCachedSocket { * should be called so that the connection will be created from scratch * the next time. */ migrateCachedSocket* migrateGetSocket(client *c, robj *host, robj *port, long timeout) { - int fd; + connection *conn; sds name = sdsempty(); migrateCachedSocket *cs; @@ -4984,34 +5260,26 @@ migrateCachedSocket* migrateGetSocket(client *c, robj *host, robj *port, long ti /* Too many items, drop one at random. */ dictEntry *de = dictGetRandomKey(server.migrate_cached_sockets); cs = dictGetVal(de); - close(cs->fd); + connClose(cs->conn); zfree(cs); dictDelete(server.migrate_cached_sockets,dictGetKey(de)); } /* Create the socket */ - fd = anetTcpNonBlockConnect(server.neterr,c->argv[1]->ptr, - atoi(c->argv[2]->ptr)); - if (fd == -1) { + conn = server.tls_cluster ? connCreateTLS() : connCreateSocket(); + if (connBlockingConnect(conn, c->argv[1]->ptr, atoi(c->argv[2]->ptr), timeout) + != C_OK) { + addReplyError(c,"-IOERR error or timeout connecting to the client"); + connClose(conn); sdsfree(name); - addReplyErrorFormat(c,"Can't connect to target node: %s", - server.neterr); - return NULL; - } - anetEnableTcpNoDelay(server.neterr,fd); - - /* Check if it connects within the specified timeout. */ - if ((aeWait(fd,AE_WRITABLE,timeout) & AE_WRITABLE) == 0) { - sdsfree(name); - addReplySds(c, - sdsnew("-IOERR error or timeout connecting to the client\r\n")); - close(fd); return NULL; } + connEnableTcpNoDelay(conn); /* Add to the cache and return it to the caller. */ cs = zmalloc(sizeof(*cs)); - cs->fd = fd; + cs->conn = conn; + cs->last_dbid = -1; cs->last_use_time = server.unixtime; dictAdd(server.migrate_cached_sockets,name,cs); @@ -5032,7 +5300,7 @@ void migrateCloseSocket(robj *host, robj *port) { return; } - close(cs->fd); + connClose(cs->conn); zfree(cs); dictDelete(server.migrate_cached_sockets,name); sdsfree(name); @@ -5046,7 +5314,7 @@ void migrateCloseTimedoutSockets(void) { migrateCachedSocket *cs = dictGetVal(de); if ((server.unixtime - cs->last_use_time) > MIGRATE_SOCKET_CACHE_TTL) { - close(cs->fd); + connClose(cs->conn); zfree(cs); dictDelete(server.migrate_cached_sockets,dictGetKey(de)); } @@ -5054,15 +5322,17 @@ void migrateCloseTimedoutSockets(void) { dictReleaseIterator(di); } -/* MIGRATE host port key dbid timeout [COPY | REPLACE | AUTH password] +/* MIGRATE host port key dbid timeout [COPY | REPLACE | AUTH password | + * AUTH2 username password] * * On in the multiple keys form: * - * MIGRATE host port "" dbid timeout [COPY | REPLACE | AUTH password] KEYS key1 - * key2 ... keyN */ + * MIGRATE host port "" dbid timeout [COPY | REPLACE | AUTH password | + * AUTH2 username password] KEYS key1 key2 ... keyN */ void migrateCommand(client *c) { migrateCachedSocket *cs; int copy = 0, replace = 0, j; + char *username = NULL; char *password = NULL; long timeout; long dbid; @@ -5080,18 +5350,28 @@ void migrateCommand(client *c) { /* Parse additional options */ for (j = 6; j < c->argc; j++) { - int moreargs = j < c->argc-1; + int moreargs = (c->argc-1) - j; if (!strcasecmp(c->argv[j]->ptr,"copy")) { copy = 1; } else if (!strcasecmp(c->argv[j]->ptr,"replace")) { replace = 1; } else if (!strcasecmp(c->argv[j]->ptr,"auth")) { if (!moreargs) { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } j++; password = c->argv[j]->ptr; + redactClientCommandArgument(c,j); + } else if (!strcasecmp(c->argv[j]->ptr,"auth2")) { + if (moreargs < 2) { + addReplyErrorObject(c,shared.syntaxerr); + return; + } + username = c->argv[++j]->ptr; + redactClientCommandArgument(c,j); + password = c->argv[++j]->ptr; + redactClientCommandArgument(c,j); } else if (!strcasecmp(c->argv[j]->ptr,"keys")) { if (sdslen(c->argv[3]->ptr) != 0) { addReplyError(c, @@ -5103,7 +5383,7 @@ void migrateCommand(client *c) { num_keys = c->argc - j - 1; break; /* All the remaining args are keys. */ } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } } @@ -5152,8 +5432,13 @@ void migrateCommand(client *c) { /* Authentication */ if (password) { - serverAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',2)); + int arity = username ? 3 : 2; + serverAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',arity)); serverAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"AUTH",4)); + if (username) { + serverAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,username, + sdslen(username))); + } serverAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,password, sdslen(password))); } @@ -5184,9 +5469,10 @@ void migrateCommand(client *c) { if (ttl < 1) ttl = 1; } - /* Relocate valid (non expired) keys into the array in successive + /* Relocate valid (non expired) keys and values into the array in successive * positions to remove holes created by the keys that were present * in the first lookup but are now expired after the second lookup. */ + ov[non_expired] = ov[j]; kv[non_expired++] = kv[j]; serverAssertWithInfo(c,NULL, @@ -5228,7 +5514,7 @@ void migrateCommand(client *c) { while ((towrite = sdslen(buf)-pos) > 0) { towrite = (towrite > (64*1024) ? (64*1024) : towrite); - nwritten = syncWrite(cs->fd,buf+pos,towrite,timeout); + nwritten = connSyncWrite(cs->conn,buf+pos,towrite,timeout); if (nwritten != (signed)towrite) { write_error = 1; goto socket_err; @@ -5242,11 +5528,11 @@ void migrateCommand(client *c) { char buf2[1024]; /* Restore reply. */ /* Read the AUTH reply if needed. */ - if (password && syncReadLine(cs->fd, buf0, sizeof(buf0), timeout) <= 0) + if (password && connSyncReadLine(cs->conn, buf0, sizeof(buf0), timeout) <= 0) goto socket_err; /* Read the SELECT reply if needed. */ - if (select && syncReadLine(cs->fd, buf1, sizeof(buf1), timeout) <= 0) + if (select && connSyncReadLine(cs->conn, buf1, sizeof(buf1), timeout) <= 0) goto socket_err; /* Read the RESTORE replies. */ @@ -5261,7 +5547,7 @@ void migrateCommand(client *c) { if (!copy) newargv = zmalloc(sizeof(robj*)*(num_keys+1)); for (j = 0; j < num_keys; j++) { - if (syncReadLine(cs->fd, buf2, sizeof(buf2), timeout) <= 0) { + if (connSyncReadLine(cs->conn, buf2, sizeof(buf2), timeout) <= 0) { socket_error = 1; break; } @@ -5285,7 +5571,8 @@ void migrateCommand(client *c) { if (!copy) { /* No COPY option: remove the local key, signal the change. */ dbDelete(c->db,kv[j]); - signalModifiedKey(c->db,kv[j]); + signalModifiedKey(c,c->db,kv[j]); + notifyKeyspaceEvent(NOTIFY_GENERIC,"del",kv[j],c->db->id); server.dirty++; /* Populate the argument vector to replace the old one. */ @@ -5426,7 +5713,7 @@ void readwriteCommand(client *c) { * resharding in progress). * * On success the function returns the node that is able to serve the request. - * If the node is not 'myself' a redirection must be perfomed. The kind of + * If the node is not 'myself' a redirection must be performed. The kind of * redirection is specified setting the integer passed by reference * 'error_code', which will be set to CLUSTER_REDIR_ASK or * CLUSTER_REDIR_MOVED. @@ -5448,8 +5735,8 @@ void readwriteCommand(client *c) { * already "down" but it is fragile to rely on the update of the global state, * so we also handle it here. * - * CLUSTER_REDIR_DOWN_STATE if the cluster is down but the user attempts to - * execute a command that addresses one or more keys. */ + * CLUSTER_REDIR_DOWN_STATE and CLUSTER_REDIR_DOWN_RO_STATE if the cluster is + * down but the user attempts to execute a command that addresses one or more keys. */ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, int argc, int *hashslot, int *error_code) { clusterNode *n = NULL; robj *firstkey = NULL; @@ -5499,7 +5786,10 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in margc = ms->commands[i].argc; margv = ms->commands[i].argv; - keyindex = getKeysFromCommand(mcmd,margv,margc,&numkeys); + getKeysResult result = GETKEYS_RESULT_INIT; + numkeys = getKeysFromCommand(mcmd,margv,margc,&result); + keyindex = result.keys; + for (j = 0; j < numkeys; j++) { robj *thiskey = margv[keyindex[j]]; int thisslot = keyHashSlot((char*)thiskey->ptr, @@ -5517,7 +5807,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in * not trapped earlier in processCommand(). Report the same * error to the client. */ if (n == NULL) { - getKeysFreeResult(keyindex); + getKeysFreeResult(&result); if (error_code) *error_code = CLUSTER_REDIR_DOWN_UNBOUND; return NULL; @@ -5541,7 +5831,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in if (!equalStringObjects(firstkey,thiskey)) { if (slot != thisslot) { /* Error: multiple keys from different slots. */ - getKeysFreeResult(keyindex); + getKeysFreeResult(&result); if (error_code) *error_code = CLUSTER_REDIR_CROSS_SLOT; return NULL; @@ -5553,24 +5843,37 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in } } - /* Migarting / Improrting slot? Count keys we don't have. */ + /* Migrating / Importing slot? Count keys we don't have. */ if ((migrating_slot || importing_slot) && lookupKeyRead(&server.db[0],thiskey) == NULL) { missing_keys++; } } - getKeysFreeResult(keyindex); + getKeysFreeResult(&result); } /* No key at all in command? then we can serve the request * without redirections or errors in all the cases. */ if (n == NULL) return myself; - /* Cluster is globally down but we got keys? We can't serve the request. */ + /* Cluster is globally down but we got keys? We only serve the request + * if it is a read command and when allow_reads_when_down is enabled. */ if (server.cluster->state != CLUSTER_OK) { - if (error_code) *error_code = CLUSTER_REDIR_DOWN_STATE; - return NULL; + if (!server.cluster_allow_reads_when_down) { + /* The cluster is configured to block commands when the + * cluster is down. */ + if (error_code) *error_code = CLUSTER_REDIR_DOWN_STATE; + return NULL; + } else if (cmd->flags & CMD_WRITE) { + /* The cluster is configured to allow read only commands */ + if (error_code) *error_code = CLUSTER_REDIR_DOWN_RO_STATE; + return NULL; + } else { + /* Fall through and allow the command to be executed: + * this happens when server.cluster_allow_reads_when_down is + * true and the command is not a write command */ + } } /* Return the hashslot by reference. */ @@ -5605,11 +5908,12 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in } /* Handle the read-only client case reading from a slave: if this - * node is a slave and the request is about an hash slot our master + * node is a slave and the request is about a hash slot our master * is serving, we can reply without redirection. */ + int is_write_command = (c->cmd->flags & CMD_WRITE) || + (c->cmd->proc == execCommand && (c->mstate.cmd_flags & CMD_WRITE)); if (c->flags & CLIENT_READONLY && - (cmd->flags & CMD_READONLY || cmd->proc == evalCommand || - cmd->proc == evalShaCommand) && + !is_write_command && nodeIsSlave(myself) && myself->slaveof == n) { @@ -5617,7 +5921,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in } /* Base case: just return the right node. However if this node is not - * myself, set error_code to MOVED since we need to issue a rediretion. */ + * myself, set error_code to MOVED since we need to issue a redirection. */ if (n != myself && error_code) *error_code = CLUSTER_REDIR_MOVED; return n; } @@ -5631,23 +5935,30 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in * be set to the hash slot that caused the redirection. */ void clusterRedirectClient(client *c, clusterNode *n, int hashslot, int error_code) { if (error_code == CLUSTER_REDIR_CROSS_SLOT) { - addReplySds(c,sdsnew("-CROSSSLOT Keys in request don't hash to the same slot\r\n")); + addReplyError(c,"-CROSSSLOT Keys in request don't hash to the same slot"); } else if (error_code == CLUSTER_REDIR_UNSTABLE) { /* The request spawns multiple keys in the same slot, * but the slot is not "stable" currently as there is * a migration or import in progress. */ - addReplySds(c,sdsnew("-TRYAGAIN Multiple keys request during rehashing of slot\r\n")); + addReplyError(c,"-TRYAGAIN Multiple keys request during rehashing of slot"); } else if (error_code == CLUSTER_REDIR_DOWN_STATE) { - addReplySds(c,sdsnew("-CLUSTERDOWN The cluster is down\r\n")); + addReplyError(c,"-CLUSTERDOWN The cluster is down"); + } else if (error_code == CLUSTER_REDIR_DOWN_RO_STATE) { + addReplyError(c,"-CLUSTERDOWN The cluster is down and only accepts read commands"); } else if (error_code == CLUSTER_REDIR_DOWN_UNBOUND) { - addReplySds(c,sdsnew("-CLUSTERDOWN Hash slot not served\r\n")); + addReplyError(c,"-CLUSTERDOWN Hash slot not served"); } else if (error_code == CLUSTER_REDIR_MOVED || error_code == CLUSTER_REDIR_ASK) { - addReplySds(c,sdscatprintf(sdsempty(), - "-%s %d %s:%d\r\n", + /* Redirect to IP:port. Include plaintext port if cluster is TLS but + * client is non-TLS. */ + int use_pport = (server.tls_cluster && + c->conn && connGetType(c->conn) != CONN_TYPE_TLS); + int port = use_pport && n->pport ? n->pport : n->port; + addReplyErrorSds(c,sdscatprintf(sdsempty(), + "-%s %d %s:%d", (error_code == CLUSTER_REDIR_ASK) ? "ASK" : "MOVED", - hashslot,n->ip,n->port)); + hashslot, n->ip, port)); } else { serverPanic("getNodeByQuery() unknown error."); } @@ -5661,7 +5972,7 @@ void clusterRedirectClient(client *c, clusterNode *n, int hashslot, int error_co * 3) The client may remain blocked forever (or up to the max timeout time) * waiting for a key change that will never happen. * - * If the client is found to be blocked into an hash slot this node no + * If the client is found to be blocked into a hash slot this node no * longer handles, the client is sent a redirection error, and the function * returns 1. Otherwise 0 is returned and no operation is performed. */ int clusterRedirectBlockedClientIfNeeded(client *c) { @@ -5673,7 +5984,10 @@ int clusterRedirectBlockedClientIfNeeded(client *c) { dictEntry *de; dictIterator *di; - /* If the cluster is down, unblock the client with the right error. */ + /* If the cluster is down, unblock the client with the right error. + * If the cluster is configured to allow reads on cluster down, we + * still want to emit this error since a write will be required + * to unblock them which may never come. */ if (server.cluster->state == CLUSTER_FAIL) { clusterRedirectClient(c,NULL,0,CLUSTER_REDIR_DOWN_STATE); return 1; @@ -5686,6 +6000,15 @@ int clusterRedirectBlockedClientIfNeeded(client *c) { int slot = keyHashSlot((char*)key->ptr, sdslen(key->ptr)); clusterNode *node = server.cluster->slots[slot]; + /* if the client is read-only and attempting to access key that our + * replica can handle, allow it. */ + if ((c->flags & CLIENT_READONLY) && + !(c->lastcmd->flags & CMD_WRITE) && + nodeIsSlave(myself) && myself->slaveof == node) + { + node = myself; + } + /* We send an error and unblock the client if: * 1) The slot is unassigned, emitting a cluster down error. * 2) The slot is not handled by this node, nor being imported. */ diff --git a/redis.submodule/src/cluster.h b/redis.submodule/src/cluster.h index 571b9c5..f476a50 100644 --- a/redis.submodule/src/cluster.h +++ b/redis.submodule/src/cluster.h @@ -13,15 +13,10 @@ /* The following defines are amount of time, sometimes expressed as * multiplicators of the node timeout value (when ending with MULT). */ -#define CLUSTER_DEFAULT_NODE_TIMEOUT 15000 -#define CLUSTER_DEFAULT_SLAVE_VALIDITY 10 /* Slave max data age factor. */ -#define CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE 1 -#define CLUSTER_DEFAULT_SLAVE_NO_FAILOVER 0 /* Failover by default. */ #define CLUSTER_FAIL_REPORT_VALIDITY_MULT 2 /* Fail report validity. */ #define CLUSTER_FAIL_UNDO_TIME_MULT 2 /* Undo fail if master is back. */ #define CLUSTER_FAIL_UNDO_TIME_ADD 10 /* Some additional time. */ #define CLUSTER_FAILOVER_DELAY 5 /* Seconds */ -#define CLUSTER_DEFAULT_MIGRATION_BARRIER 1 #define CLUSTER_MF_TIMEOUT 5000 /* Milliseconds to do a manual failover. */ #define CLUSTER_MF_PAUSE_MULT 2 /* Master pause manual failover mult. */ #define CLUSTER_SLAVE_MIGRATION_DELAY 5000 /* Delay for slave migration. */ @@ -34,15 +29,18 @@ #define CLUSTER_REDIR_MOVED 4 /* -MOVED redirection required. */ #define CLUSTER_REDIR_DOWN_STATE 5 /* -CLUSTERDOWN, global state. */ #define CLUSTER_REDIR_DOWN_UNBOUND 6 /* -CLUSTERDOWN, unbound slot. */ +#define CLUSTER_REDIR_DOWN_RO_STATE 7 /* -CLUSTERDOWN, allow reads. */ struct clusterNode; /* clusterLink encapsulates everything needed to talk with a remote node. */ typedef struct clusterLink { mstime_t ctime; /* Link creation time */ - int fd; /* TCP socket file descriptor */ + connection *conn; /* Connection to remote node */ sds sndbuf; /* Packet send buffer */ - sds rcvbuf; /* Packet reception buffer */ + char *rcvbuf; /* Packet reception buffer */ + size_t rcvbuf_len; /* Used size of rcvbuf */ + size_t rcvbuf_alloc; /* Allocated size of rcvbuf */ struct clusterNode *node; /* Node related to this link if any, or NULL */ } clusterLink; @@ -55,8 +53,8 @@ typedef struct clusterLink { #define CLUSTER_NODE_HANDSHAKE 32 /* We have still to exchange the first ping */ #define CLUSTER_NODE_NOADDR 64 /* We don't know the address of this node */ #define CLUSTER_NODE_MEET 128 /* Send a MEET message to this node */ -#define CLUSTER_NODE_MIGRATE_TO 256 /* Master elegible for replica migration. */ -#define CLUSTER_NODE_NOFAILOVER 512 /* Slave will not try to failver. */ +#define CLUSTER_NODE_MIGRATE_TO 256 /* Master eligible for replica migration. */ +#define CLUSTER_NODE_NOFAILOVER 512 /* Slave will not try to failover. */ #define CLUSTER_NODE_NULL_NAME "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" #define nodeIsMaster(n) ((n)->flags & CLUSTER_NODE_MASTER) @@ -81,6 +79,7 @@ typedef struct clusterLink { #define CLUSTER_TODO_UPDATE_STATE (1<<1) #define CLUSTER_TODO_SAVE_CONFIG (1<<2) #define CLUSTER_TODO_FSYNC_CONFIG (1<<3) +#define CLUSTER_TODO_HANDLE_MANUALFAILOVER (1<<4) /* Message types. * @@ -119,6 +118,7 @@ typedef struct clusterNode { int flags; /* CLUSTER_NODE_... */ uint64_t configEpoch; /* Last configEpoch observed for this node */ unsigned char slots[CLUSTER_SLOTS/8]; /* slots handled by this node */ + sds slots_info; /* Slots info represented by string. */ int numslots; /* Number of slots handled by this node */ int numslaves; /* Number of slave nodes, if this is a master */ struct clusterNode **slaves; /* pointers to slave nodes */ @@ -128,13 +128,16 @@ typedef struct clusterNode { tables. */ mstime_t ping_sent; /* Unix time we sent latest ping */ mstime_t pong_received; /* Unix time we received the pong */ + mstime_t data_received; /* Unix time we received any data */ mstime_t fail_time; /* Unix time when FAIL flag was set */ mstime_t voted_time; /* Last time we voted for a slave of this master */ mstime_t repl_offset_time; /* Unix time we received offset for this node */ mstime_t orphaned_time; /* Starting time of orphaned master condition */ long long repl_offset; /* Last known repl offset for this node. */ char ip[NET_IP_STR_LEN]; /* Latest known IP address of this node */ - int port; /* Latest known clients port of this node */ + int port; /* Latest known clients port (TLS or plain). */ + int pport; /* Latest known clients plaintext port. Only used + if the main clients port is for TLS. */ int cport; /* Latest known cluster port of this node. */ clusterLink *link; /* TCP/IP link with this node */ list *fail_reports; /* List of nodes signaling this as failing */ @@ -167,10 +170,10 @@ typedef struct clusterState { clusterNode *mf_slave; /* Slave performing the manual failover. */ /* Manual failover state of slave. */ long long mf_master_offset; /* Master offset the slave needs to start MF - or zero if stil not received. */ + or -1 if still not received. */ int mf_can_start; /* If non-zero signal that the manual failover can start requesting masters vote. */ - /* The followign fields are used by masters to take state on elections. */ + /* The following fields are used by masters to take state on elections. */ uint64_t lastVoteEpoch; /* Epoch of the last vote granted. */ int todo_before_sleep; /* Things to do in clusterBeforeSleep(). */ /* Messages received and sent by type. */ @@ -193,7 +196,8 @@ typedef struct { uint16_t port; /* base port last time it was seen */ uint16_t cport; /* cluster port last time it was seen */ uint16_t flags; /* node->flags copy */ - uint32_t notused1; + uint16_t pport; /* plaintext-port, when base port is TLS */ + uint16_t notused1; } clusterMsgDataGossip; typedef struct { @@ -266,7 +270,8 @@ typedef struct { unsigned char myslots[CLUSTER_SLOTS/8]; char slaveof[CLUSTER_NAMELEN]; char myip[NET_IP_STR_LEN]; /* Sender IP, if not all zeroed. */ - char notused1[34]; /* 34 bytes reserved for future usage. */ + char notused1[32]; /* 32 bytes reserved for future usage. */ + uint16_t pport; /* Sender TCP plaintext port, if base port is TLS */ uint16_t cport; /* Sender TCP cluster bus port */ uint16_t flags; /* Sender node flags */ unsigned char state; /* Cluster state from the POV of the sender */ @@ -286,5 +291,6 @@ typedef struct { clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, int argc, int *hashslot, int *ask); int clusterRedirectBlockedClientIfNeeded(client *c); void clusterRedirectClient(client *c, clusterNode *n, int hashslot, int error_code); +unsigned long getClusterConnectionsCount(void); #endif /* __CLUSTER_H */ diff --git a/redis.submodule/src/config.c b/redis.submodule/src/config.c index ba8e17e..e0eec2f 100644 --- a/redis.submodule/src/config.c +++ b/redis.submodule/src/config.c @@ -91,6 +91,41 @@ configEnum aof_fsync_enum[] = { {NULL, 0} }; +configEnum repl_diskless_load_enum[] = { + {"disabled", REPL_DISKLESS_LOAD_DISABLED}, + {"on-empty-db", REPL_DISKLESS_LOAD_WHEN_DB_EMPTY}, + {"swapdb", REPL_DISKLESS_LOAD_SWAPDB}, + {NULL, 0} +}; + +configEnum tls_auth_clients_enum[] = { + {"no", TLS_CLIENT_AUTH_NO}, + {"yes", TLS_CLIENT_AUTH_YES}, + {"optional", TLS_CLIENT_AUTH_OPTIONAL}, + {NULL, 0} +}; + +configEnum oom_score_adj_enum[] = { + {"no", OOM_SCORE_ADJ_NO}, + {"yes", OOM_SCORE_RELATIVE}, + {"relative", OOM_SCORE_RELATIVE}, + {"absolute", OOM_SCORE_ADJ_ABSOLUTE}, + {NULL, 0} +}; + +configEnum acl_pubsub_default_enum[] = { + {"allchannels", USER_FLAG_ALLCHANNELS}, + {"resetchannels", 0}, + {NULL, 0} +}; + +configEnum sanitize_dump_payload_enum[] = { + {"no", SANITIZE_DUMP_NO}, + {"yes", SANITIZE_DUMP_YES}, + {"clients", SANITIZE_DUMP_CLIENTS}, + {NULL, 0} +}; + /* Output buffer limits presets. */ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { {0, 0, 0}, /* normal */ @@ -98,6 +133,126 @@ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { {1024*1024*32, 1024*1024*8, 60} /* pubsub */ }; +/* OOM Score defaults */ +int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT] = { 0, 200, 800 }; + +/* Generic config infrastructure function pointers + * int is_valid_fn(val, err) + * Return 1 when val is valid, and 0 when invalid. + * Optionally set err to a static error string. + * int update_fn(val, prev, err) + * This function is called only for CONFIG SET command (not at config file parsing) + * It is called after the actual config is applied, + * Return 1 for success, and 0 for failure. + * Optionally set err to a static error string. + * On failure the config change will be reverted. + */ + +/* Configuration values that require no special handling to set, get, load or + * rewrite. */ +typedef struct boolConfigData { + int *config; /* The pointer to the server config this value is stored in */ + const int default_value; /* The default value of the config on rewrite */ + int (*is_valid_fn)(int val, const char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(int val, int prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */ +} boolConfigData; + +typedef struct stringConfigData { + char **config; /* Pointer to the server config this value is stored in. */ + const char *default_value; /* Default value of the config on rewrite. */ + int (*is_valid_fn)(char* val, const char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(char* val, char* prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */ + int convert_empty_to_null; /* Boolean indicating if empty strings should + be stored as a NULL value. */ +} stringConfigData; + +typedef struct sdsConfigData { + sds *config; /* Pointer to the server config this value is stored in. */ + const char *default_value; /* Default value of the config on rewrite. */ + int (*is_valid_fn)(sds val, const char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(sds val, sds prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */ + int convert_empty_to_null; /* Boolean indicating if empty SDS strings should + be stored as a NULL value. */ +} sdsConfigData; + +typedef struct enumConfigData { + int *config; /* The pointer to the server config this value is stored in */ + configEnum *enum_value; /* The underlying enum type this data represents */ + const int default_value; /* The default value of the config on rewrite */ + int (*is_valid_fn)(int val, const char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(int val, int prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */ +} enumConfigData; + +typedef enum numericType { + NUMERIC_TYPE_INT, + NUMERIC_TYPE_UINT, + NUMERIC_TYPE_LONG, + NUMERIC_TYPE_ULONG, + NUMERIC_TYPE_LONG_LONG, + NUMERIC_TYPE_ULONG_LONG, + NUMERIC_TYPE_SIZE_T, + NUMERIC_TYPE_SSIZE_T, + NUMERIC_TYPE_OFF_T, + NUMERIC_TYPE_TIME_T, +} numericType; + +typedef struct numericConfigData { + union { + int *i; + unsigned int *ui; + long *l; + unsigned long *ul; + long long *ll; + unsigned long long *ull; + size_t *st; + ssize_t *sst; + off_t *ot; + time_t *tt; + } config; /* The pointer to the numeric config this value is stored in */ + int is_memory; /* Indicates if this value can be loaded as a memory value */ + numericType numeric_type; /* An enum indicating the type of this value */ + long long lower_bound; /* The lower bound of this numeric value */ + long long upper_bound; /* The upper bound of this numeric value */ + const long long default_value; /* The default value of the config on rewrite */ + int (*is_valid_fn)(long long val, const char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(long long val, long long prev, const char **err); /* Optional function to apply new value at runtime (generic doc above) */ +} numericConfigData; + +typedef union typeData { + boolConfigData yesno; + stringConfigData string; + sdsConfigData sds; + enumConfigData enumd; + numericConfigData numeric; +} typeData; + +typedef struct typeInterface { + /* Called on server start, to init the server with default value */ + void (*init)(typeData data); + /* Called on server startup and CONFIG SET, returns 1 on success, 0 on error + * and can set a verbose err string, update is true when called from CONFIG SET */ + int (*set)(typeData data, sds value, int update, const char **err); + /* Called on CONFIG GET, required to add output to the client */ + void (*get)(client *c, typeData data); + /* Called on CONFIG REWRITE, required to rewrite the config state */ + void (*rewrite)(typeData data, const char *name, struct rewriteConfigState *state); +} typeInterface; + +typedef struct standardConfig { + const char *name; /* The user visible name of this config */ + const char *alias; /* An alias that can also be used for this config */ + const unsigned int flags; /* Flags for this specific config */ + typeInterface interface; /* The function pointers that define the type interface */ + typeData data; /* The type specific data exposed used by the interface */ +} standardConfig; + +#define MODIFIABLE_CONFIG 0 /* This is the implied default for a standard + * config, which is mutable. */ +#define IMMUTABLE_CONFIG (1ULL<<0) /* Can this value only be set at startup? */ +#define SENSITIVE_CONFIG (1ULL<<1) /* Does this value contain sensitive information */ + +standardConfig configs[]; + /*----------------------------------------------------------------------------- * Enum access functions *----------------------------------------------------------------------------*/ @@ -169,11 +324,75 @@ void queueLoadModule(sds path, sds *argv, int argc) { listAddNodeTail(server.loadmodule_queue,loadmod); } +/* Parse an array of CONFIG_OOM_COUNT sds strings, validate and populate + * server.oom_score_adj_values if valid. + */ + +static int updateOOMScoreAdjValues(sds *args, const char **err, int apply) { + int i; + int values[CONFIG_OOM_COUNT]; + + for (i = 0; i < CONFIG_OOM_COUNT; i++) { + char *eptr; + long long val = strtoll(args[i], &eptr, 10); + + if (*eptr != '\0' || val < -2000 || val > 2000) { + if (err) *err = "Invalid oom-score-adj-values, elements must be between -2000 and 2000."; + return C_ERR; + } + + values[i] = val; + } + + /* Verify that the values make sense. If they don't omit a warning but + * keep the configuration, which may still be valid for privileged processes. + */ + + if (values[CONFIG_OOM_REPLICA] < values[CONFIG_OOM_MASTER] || + values[CONFIG_OOM_BGCHILD] < values[CONFIG_OOM_REPLICA]) { + serverLog(LOG_WARNING, + "The oom-score-adj-values configuration may not work for non-privileged processes! " + "Please consult the documentation."); + } + + /* Store values, retain previous config for rollback in case we fail. */ + int old_values[CONFIG_OOM_COUNT]; + for (i = 0; i < CONFIG_OOM_COUNT; i++) { + old_values[i] = server.oom_score_adj_values[i]; + server.oom_score_adj_values[i] = values[i]; + } + + /* When parsing the config file, we want to apply only when all is done. */ + if (!apply) + return C_OK; + + /* Update */ + if (setOOMScoreAdj(-1) == C_ERR) { + /* Roll back */ + for (i = 0; i < CONFIG_OOM_COUNT; i++) + server.oom_score_adj_values[i] = old_values[i]; + + if (err) + *err = "Failed to apply oom-score-adj-values configuration, check server logs."; + + return C_ERR; + } + + return C_OK; +} + +void initConfigValues() { + for (standardConfig *config = configs; config->name != NULL; config++) { + config->interface.init(config->data); + } +} + void loadServerConfigFromString(char *config) { - char *err = NULL; + const char *err = NULL; int linenum = 0, totlines, i; int slaveof_linenum = 0; sds *lines; + int save_loaded = 0; lines = sdssplitlen(config,strlen(config),"\n",1,&totlines); @@ -201,42 +420,44 @@ void loadServerConfigFromString(char *config) { } sdstolower(argv[0]); - /* Execute config directives */ - if (!strcasecmp(argv[0],"timeout") && argc == 2) { - server.maxidletime = atoi(argv[1]); - if (server.maxidletime < 0) { - err = "Invalid timeout value"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"tcp-keepalive") && argc == 2) { - server.tcpkeepalive = atoi(argv[1]); - if (server.tcpkeepalive < 0) { - err = "Invalid tcp-keepalive value"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"protected-mode") && argc == 2) { - if ((server.protected_mode = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"port") && argc == 2) { - server.port = atoi(argv[1]); - if (server.port < 0 || server.port > 65535) { - err = "Invalid port"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"tcp-backlog") && argc == 2) { - server.tcp_backlog = atoi(argv[1]); - if (server.tcp_backlog < 0) { - err = "Invalid backlog value"; goto loaderr; + /* Iterate the configs that are standard */ + int match = 0; + for (standardConfig *config = configs; config->name != NULL; config++) { + if ((!strcasecmp(argv[0],config->name) || + (config->alias && !strcasecmp(argv[0],config->alias)))) + { + if (argc != 2) { + err = "wrong number of arguments"; + goto loaderr; + } + if (!config->interface.set(config->data, argv[1], 0, &err)) { + goto loaderr; + } + + match = 1; + break; } - } else if (!strcasecmp(argv[0],"bind") && argc >= 2) { + } + + if (match) { + sdsfreesplitres(argv,argc); + continue; + } + + /* Execute config directives */ + if (!strcasecmp(argv[0],"bind") && argc >= 2) { int j, addresses = argc-1; if (addresses > CONFIG_BINDADDR_MAX) { err = "Too many bind addresses specified"; goto loaderr; } + /* Free old bind addresses */ + for (j = 0; j < server.bindaddr_count; j++) { + zfree(server.bindaddr[j]); + } for (j = 0; j < addresses; j++) server.bindaddr[j] = zstrdup(argv[j+1]); server.bindaddr_count = addresses; - } else if (!strcasecmp(argv[0],"unixsocket") && argc == 2) { - server.unixsocket = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"unixsocketperm") && argc == 2) { errno = 0; server.unixsocketperm = (mode_t)strtol(argv[1], NULL, 8); @@ -244,6 +465,14 @@ void loadServerConfigFromString(char *config) { err = "Invalid socket file permissions"; goto loaderr; } } else if (!strcasecmp(argv[0],"save")) { + /* We don't reset save params before loading, because if they're not part + * of the file the defaults should be used. + */ + if (!save_loaded) { + save_loaded = 1; + resetServerSaveParams(); + } + if (argc == 3) { int seconds = atoi(argv[1]); int changes = atoi(argv[2]); @@ -260,13 +489,6 @@ void loadServerConfigFromString(char *config) { argv[1], strerror(errno)); exit(1); } - } else if (!strcasecmp(argv[0],"loglevel") && argc == 2) { - server.verbosity = configEnumGetValue(loglevel_enum,argv[1]); - if (server.verbosity == INT_MIN) { - err = "Invalid log level. " - "Must be one of debug, verbose, notice, warning"; - goto loaderr; - } } else if (!strcasecmp(argv[0],"logfile") && argc == 2) { FILE *logfp; @@ -283,327 +505,27 @@ void loadServerConfigFromString(char *config) { } fclose(logfp); } - } else if (!strcasecmp(argv[0],"always-show-logo") && argc == 2) { - if ((server.always_show_logo = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"syslog-enabled") && argc == 2) { - if ((server.syslog_enabled = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"syslog-ident") && argc == 2) { - if (server.syslog_ident) zfree(server.syslog_ident); - server.syslog_ident = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"syslog-facility") && argc == 2) { - server.syslog_facility = - configEnumGetValue(syslog_facility_enum,argv[1]); - if (server.syslog_facility == INT_MIN) { - err = "Invalid log facility. Must be one of USER or between LOCAL0-LOCAL7"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"databases") && argc == 2) { - server.dbnum = atoi(argv[1]); - if (server.dbnum < 1) { - err = "Invalid number of databases"; goto loaderr; - } } else if (!strcasecmp(argv[0],"include") && argc == 2) { - loadServerConfig(argv[1],NULL); - } else if (!strcasecmp(argv[0],"maxclients") && argc == 2) { - server.maxclients = atoi(argv[1]); - if (server.maxclients < 1) { - err = "Invalid max clients limit"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"maxmemory") && argc == 2) { - server.maxmemory = memtoll(argv[1],NULL); - } else if (!strcasecmp(argv[0],"maxmemory-policy") && argc == 2) { - server.maxmemory_policy = - configEnumGetValue(maxmemory_policy_enum,argv[1]); - if (server.maxmemory_policy == INT_MIN) { - err = "Invalid maxmemory policy"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"maxmemory-samples") && argc == 2) { - server.maxmemory_samples = atoi(argv[1]); - if (server.maxmemory_samples <= 0) { - err = "maxmemory-samples must be 1 or greater"; - goto loaderr; - } - } else if ((!strcasecmp(argv[0],"proto-max-bulk-len")) && argc == 2) { - server.proto_max_bulk_len = memtoll(argv[1],NULL); - } else if ((!strcasecmp(argv[0],"client-query-buffer-limit")) && argc == 2) { - server.client_max_querybuf_len = memtoll(argv[1],NULL); - } else if (!strcasecmp(argv[0],"lfu-log-factor") && argc == 2) { - server.lfu_log_factor = atoi(argv[1]); - if (server.lfu_log_factor < 0) { - err = "lfu-log-factor must be 0 or greater"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"lfu-decay-time") && argc == 2) { - server.lfu_decay_time = atoi(argv[1]); - if (server.lfu_decay_time < 0) { - err = "lfu-decay-time must be 0 or greater"; - goto loaderr; - } + loadServerConfig(argv[1], 0, NULL); } else if ((!strcasecmp(argv[0],"slaveof") || !strcasecmp(argv[0],"replicaof")) && argc == 3) { slaveof_linenum = linenum; - server.masterhost = sdsnew(argv[1]); - server.masterport = atoi(argv[2]); - server.repl_state = REPL_STATE_CONNECT; - } else if ((!strcasecmp(argv[0],"repl-ping-slave-period") || - !strcasecmp(argv[0],"repl-ping-replica-period")) && - argc == 2) - { - server.repl_ping_slave_period = atoi(argv[1]); - if (server.repl_ping_slave_period <= 0) { - err = "repl-ping-replica-period must be 1 or greater"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"repl-timeout") && argc == 2) { - server.repl_timeout = atoi(argv[1]); - if (server.repl_timeout <= 0) { - err = "repl-timeout must be 1 or greater"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"repl-disable-tcp-nodelay") && argc==2) { - if ((server.repl_disable_tcp_nodelay = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"repl-diskless-sync") && argc==2) { - if ((server.repl_diskless_sync = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"repl-diskless-sync-delay") && argc==2) { - server.repl_diskless_sync_delay = atoi(argv[1]); - if (server.repl_diskless_sync_delay < 0) { - err = "repl-diskless-sync-delay can't be negative"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"repl-backlog-size") && argc == 2) { - long long size = memtoll(argv[1],NULL); - if (size <= 0) { - err = "repl-backlog-size must be 1 or greater."; - goto loaderr; - } - resizeReplicationBacklog(size); - } else if (!strcasecmp(argv[0],"repl-backlog-ttl") && argc == 2) { - server.repl_backlog_time_limit = atoi(argv[1]); - if (server.repl_backlog_time_limit < 0) { - err = "repl-backlog-ttl can't be negative "; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"masterauth") && argc == 2) { - zfree(server.masterauth); - server.masterauth = argv[1][0] ? zstrdup(argv[1]) : NULL; - } else if ((!strcasecmp(argv[0],"slave-serve-stale-data") || - !strcasecmp(argv[0],"replica-serve-stale-data")) - && argc == 2) - { - if ((server.repl_serve_stale_data = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if ((!strcasecmp(argv[0],"slave-read-only") || - !strcasecmp(argv[0],"replica-read-only")) - && argc == 2) - { - if ((server.repl_slave_ro = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if ((!strcasecmp(argv[0],"slave-ignore-maxmemory") || - !strcasecmp(argv[0],"replica-ignore-maxmemory")) - && argc == 2) - { - if ((server.repl_slave_ignore_maxmemory = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"rdbcompression") && argc == 2) { - if ((server.rdb_compression = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"rdbchecksum") && argc == 2) { - if ((server.rdb_checksum = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"activerehashing") && argc == 2) { - if ((server.activerehashing = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"lazyfree-lazy-eviction") && argc == 2) { - if ((server.lazyfree_lazy_eviction = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"lazyfree-lazy-expire") && argc == 2) { - if ((server.lazyfree_lazy_expire = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"lazyfree-lazy-server-del") && argc == 2){ - if ((server.lazyfree_lazy_server_del = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if ((!strcasecmp(argv[0],"slave-lazy-flush") || - !strcasecmp(argv[0],"replica-lazy-flush")) && argc == 2) - { - if ((server.repl_slave_lazy_flush = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"activedefrag") && argc == 2) { - if ((server.active_defrag_enabled = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - if (server.active_defrag_enabled) { -#ifndef HAVE_DEFRAG - err = "active defrag can't be enabled without proper jemalloc support"; goto loaderr; -#endif - } - } else if (!strcasecmp(argv[0],"daemonize") && argc == 2) { - if ((server.daemonize = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"dynamic-hz") && argc == 2) { - if ((server.dynamic_hz = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"hz") && argc == 2) { - server.config_hz = atoi(argv[1]); - if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; - if (server.config_hz > CONFIG_MAX_HZ) server.config_hz = CONFIG_MAX_HZ; - } else if (!strcasecmp(argv[0],"appendonly") && argc == 2) { - int yes; - - if ((yes = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - server.aof_state = yes ? AOF_ON : AOF_OFF; - } else if (!strcasecmp(argv[0],"appendfilename") && argc == 2) { - if (!pathIsBaseName(argv[1])) { - err = "appendfilename can't be a path, just a filename"; - goto loaderr; - } - zfree(server.aof_filename); - server.aof_filename = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"no-appendfsync-on-rewrite") - && argc == 2) { - if ((server.aof_no_fsync_on_rewrite= yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"appendfsync") && argc == 2) { - server.aof_fsync = configEnumGetValue(aof_fsync_enum,argv[1]); - if (server.aof_fsync == INT_MIN) { - err = "argument must be 'no', 'always' or 'everysec'"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"auto-aof-rewrite-percentage") && - argc == 2) - { - server.aof_rewrite_perc = atoi(argv[1]); - if (server.aof_rewrite_perc < 0) { - err = "Invalid negative percentage for AOF auto rewrite"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"auto-aof-rewrite-min-size") && - argc == 2) - { - server.aof_rewrite_min_size = memtoll(argv[1],NULL); - } else if (!strcasecmp(argv[0],"aof-rewrite-incremental-fsync") && - argc == 2) - { - if ((server.aof_rewrite_incremental_fsync = - yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"rdb-save-incremental-fsync") && - argc == 2) - { - if ((server.rdb_save_incremental_fsync = - yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"aof-load-truncated") && argc == 2) { - if ((server.aof_load_truncated = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; + sdsfree(server.masterhost); + if (!strcasecmp(argv[1], "no") && !strcasecmp(argv[2], "one")) { + server.masterhost = NULL; + continue; } - } else if (!strcasecmp(argv[0],"aof-use-rdb-preamble") && argc == 2) { - if ((server.aof_use_rdb_preamble = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"requirepass") && argc == 2) { - if (strlen(argv[1]) > CONFIG_AUTHPASS_MAX_LEN) { - err = "Password is longer than CONFIG_AUTHPASS_MAX_LEN"; - goto loaderr; - } - server.requirepass = argv[1][0] ? zstrdup(argv[1]) : NULL; - } else if (!strcasecmp(argv[0],"pidfile") && argc == 2) { - zfree(server.pidfile); - server.pidfile = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"dbfilename") && argc == 2) { - if (!pathIsBaseName(argv[1])) { - err = "dbfilename can't be a path, just a filename"; - goto loaderr; - } - zfree(server.rdb_filename); - server.rdb_filename = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"active-defrag-threshold-lower") && argc == 2) { - server.active_defrag_threshold_lower = atoi(argv[1]); - if (server.active_defrag_threshold_lower < 0 || - server.active_defrag_threshold_lower > 1000) { - err = "active-defrag-threshold-lower must be between 0 and 1000"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-threshold-upper") && argc == 2) { - server.active_defrag_threshold_upper = atoi(argv[1]); - if (server.active_defrag_threshold_upper < 0 || - server.active_defrag_threshold_upper > 1000) { - err = "active-defrag-threshold-upper must be between 0 and 1000"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-ignore-bytes") && argc == 2) { - server.active_defrag_ignore_bytes = memtoll(argv[1], NULL); - if (server.active_defrag_ignore_bytes <= 0) { - err = "active-defrag-ignore-bytes must above 0"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-cycle-min") && argc == 2) { - server.active_defrag_cycle_min = atoi(argv[1]); - if (server.active_defrag_cycle_min < 1 || server.active_defrag_cycle_min > 99) { - err = "active-defrag-cycle-min must be between 1 and 99"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-cycle-max") && argc == 2) { - server.active_defrag_cycle_max = atoi(argv[1]); - if (server.active_defrag_cycle_max < 1 || server.active_defrag_cycle_max > 99) { - err = "active-defrag-cycle-max must be between 1 and 99"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-max-scan-fields") && argc == 2) { - server.active_defrag_max_scan_fields = strtoll(argv[1],NULL,10); - if (server.active_defrag_max_scan_fields < 1) { - err = "active-defrag-max-scan-fields must be positive"; - goto loaderr; + server.masterhost = sdsnew(argv[1]); + char *ptr; + server.masterport = strtol(argv[2], &ptr, 10); + if (server.masterport < 0 || server.masterport > 65535 || *ptr != '\0') { + err = "Invalid master port"; goto loaderr; } - } else if (!strcasecmp(argv[0],"hash-max-ziplist-entries") && argc == 2) { - server.hash_max_ziplist_entries = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"hash-max-ziplist-value") && argc == 2) { - server.hash_max_ziplist_value = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"stream-node-max-bytes") && argc == 2) { - server.stream_node_max_bytes = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"stream-node-max-entries") && argc == 2) { - server.stream_node_max_entries = atoi(argv[1]); + server.repl_state = REPL_STATE_CONNECT; } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){ /* DEAD OPTION */ } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2) { /* DEAD OPTION */ - } else if (!strcasecmp(argv[0],"list-max-ziplist-size") && argc == 2) { - server.list_max_ziplist_size = atoi(argv[1]); - } else if (!strcasecmp(argv[0],"list-compress-depth") && argc == 2) { - server.list_compress_depth = atoi(argv[1]); - } else if (!strcasecmp(argv[0],"set-max-intset-entries") && argc == 2) { - server.set_max_intset_entries = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"zset-max-ziplist-entries") && argc == 2) { - server.zset_max_ziplist_entries = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"zset-max-ziplist-value") && argc == 2) { - server.zset_max_ziplist_value = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"hll-sparse-max-bytes") && argc == 2) { - server.hll_sparse_max_bytes = memtoll(argv[1], NULL); } else if (!strcasecmp(argv[0],"rename-command") && argc == 3) { struct redisCommand *cmd = lookupCommand(argv[1]); int retval; @@ -628,88 +550,9 @@ void loadServerConfigFromString(char *config) { err = "Target command name already exists"; goto loaderr; } } - } else if (!strcasecmp(argv[0],"cluster-enabled") && argc == 2) { - if ((server.cluster_enabled = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } } else if (!strcasecmp(argv[0],"cluster-config-file") && argc == 2) { zfree(server.cluster_configfile); server.cluster_configfile = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"cluster-announce-ip") && argc == 2) { - zfree(server.cluster_announce_ip); - server.cluster_announce_ip = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"cluster-announce-port") && argc == 2) { - server.cluster_announce_port = atoi(argv[1]); - if (server.cluster_announce_port < 0 || - server.cluster_announce_port > 65535) - { - err = "Invalid port"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"cluster-announce-bus-port") && - argc == 2) - { - server.cluster_announce_bus_port = atoi(argv[1]); - if (server.cluster_announce_bus_port < 0 || - server.cluster_announce_bus_port > 65535) - { - err = "Invalid port"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"cluster-require-full-coverage") && - argc == 2) - { - if ((server.cluster_require_full_coverage = yesnotoi(argv[1])) == -1) - { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"cluster-node-timeout") && argc == 2) { - server.cluster_node_timeout = strtoll(argv[1],NULL,10); - if (server.cluster_node_timeout <= 0) { - err = "cluster node timeout must be 1 or greater"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"cluster-migration-barrier") - && argc == 2) - { - server.cluster_migration_barrier = atoi(argv[1]); - if (server.cluster_migration_barrier < 0) { - err = "cluster migration barrier must zero or positive"; - goto loaderr; - } - } else if ((!strcasecmp(argv[0],"cluster-slave-validity-factor") || - !strcasecmp(argv[0],"cluster-replica-validity-factor")) - && argc == 2) - { - server.cluster_slave_validity_factor = atoi(argv[1]); - if (server.cluster_slave_validity_factor < 0) { - err = "cluster replica validity factor must be zero or positive"; - goto loaderr; - } - } else if ((!strcasecmp(argv[0],"cluster-slave-no-failover") || - !strcasecmp(argv[0],"cluster-replica-no-failover")) && - argc == 2) - { - server.cluster_slave_no_failover = yesnotoi(argv[1]); - if (server.cluster_slave_no_failover == -1) { - err = "argument must be 'yes' or 'no'"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { - server.lua_time_limit = strtoll(argv[1],NULL,10); - } else if (!strcasecmp(argv[0],"lua-replicate-commands") && argc == 2) { - server.lua_always_replicate_commands = yesnotoi(argv[1]); - } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && - argc == 2) - { - server.slowlog_log_slower_than = strtoll(argv[1],NULL,10); - } else if (!strcasecmp(argv[0],"latency-monitor-threshold") && - argc == 2) - { - server.latency_monitor_threshold = strtoll(argv[1],NULL,10); - if (server.latency_monitor_threshold < 0) { - err = "The latency threshold can't be negative"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"slowlog-max-len") && argc == 2) { - server.slowlog_max_len = strtoll(argv[1],NULL,10); } else if (!strcasecmp(argv[0],"client-output-buffer-limit") && argc == 5) { @@ -732,43 +575,8 @@ void loadServerConfigFromString(char *config) { server.client_obuf_limits[class].hard_limit_bytes = hard; server.client_obuf_limits[class].soft_limit_bytes = soft; server.client_obuf_limits[class].soft_limit_seconds = soft_seconds; - } else if (!strcasecmp(argv[0],"stop-writes-on-bgsave-error") && - argc == 2) { - if ((server.stop_writes_on_bgsave_err = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if ((!strcasecmp(argv[0],"slave-priority") || - !strcasecmp(argv[0],"replica-priority")) && argc == 2) - { - server.slave_priority = atoi(argv[1]); - } else if ((!strcasecmp(argv[0],"slave-announce-ip") || - !strcasecmp(argv[0],"replica-announce-ip")) && argc == 2) - { - zfree(server.slave_announce_ip); - server.slave_announce_ip = zstrdup(argv[1]); - } else if ((!strcasecmp(argv[0],"slave-announce-port") || - !strcasecmp(argv[0],"replica-announce-port")) && argc == 2) - { - server.slave_announce_port = atoi(argv[1]); - if (server.slave_announce_port < 0 || - server.slave_announce_port > 65535) - { - err = "Invalid port"; goto loaderr; - } - } else if ((!strcasecmp(argv[0],"min-slaves-to-write") || - !strcasecmp(argv[0],"min-replicas-to-write")) && argc == 2) - { - server.repl_min_slaves_to_write = atoi(argv[1]); - if (server.repl_min_slaves_to_write < 0) { - err = "Invalid value for min-replicas-to-write."; goto loaderr; - } - } else if ((!strcasecmp(argv[0],"min-slaves-max-lag") || - !strcasecmp(argv[0],"min-replicas-max-lag")) && argc == 2) - { - server.repl_min_slaves_max_lag = atoi(argv[1]); - if (server.repl_min_slaves_max_lag < 0) { - err = "Invalid value for min-replicas-max-lag."; goto loaderr; - } + } else if (!strcasecmp(argv[0],"oom-score-adj-values") && argc == 1 + CONFIG_OOM_COUNT) { + if (updateOOMScoreAdjValues(&argv[1], &err, 0) == C_ERR) goto loaderr; } else if (!strcasecmp(argv[0],"notify-keyspace-events") && argc == 2) { int flags = keyspaceEventsStringToFlags(argv[1]); @@ -777,13 +585,14 @@ void loadServerConfigFromString(char *config) { goto loaderr; } server.notify_keyspace_events = flags; - } else if (!strcasecmp(argv[0],"supervised") && argc == 2) { - server.supervised_mode = - configEnumGetValue(supervised_mode_enum,argv[1]); - - if (server.supervised_mode == INT_MIN) { - err = "Invalid option for 'supervised'. " - "Allowed values: 'upstart', 'systemd', 'auto', or 'no'"; + } else if (!strcasecmp(argv[0],"user") && argc >= 2) { + int argc_err; + if (ACLAppendUserForLoading(argv,argc,&argc_err) == C_ERR) { + char buf[1024]; + const char *errmsg = ACLSetUserStringError(); + snprintf(buf,sizeof(buf),"Error in user declaration '%s': %s", + argv[argc_err],errmsg); + err = buf; goto loaderr; } } else if (!strcasecmp(argv[0],"loadmodule") && argc >= 2) { @@ -796,8 +605,7 @@ void loadServerConfigFromString(char *config) { err = "sentinel directive while not in sentinel mode"; goto loaderr; } - err = sentinelHandleConfiguration(argv+1,argc-1); - if (err) goto loaderr; + queueSentinelConfig(argv+1,argc-1,linenum,lines[i]); } } else { err = "Bad directive or wrong number of arguments"; goto loaderr; @@ -813,11 +621,16 @@ void loadServerConfigFromString(char *config) { goto loaderr; } + /* To ensure backward compatibility and work while hz is out of range */ + if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; + if (server.config_hz > CONFIG_MAX_HZ) server.config_hz = CONFIG_MAX_HZ; + sdsfreesplitres(lines,totlines); return; loaderr: - fprintf(stderr, "\n*** FATAL CONFIG FILE ERROR ***\n"); + fprintf(stderr, "\n*** FATAL CONFIG FILE ERROR (Redis %s) ***\n", + REDIS_VERSION); fprintf(stderr, "Reading the configuration file, at line %d\n", linenum); fprintf(stderr, ">>> '%s'\n", lines[i]); fprintf(stderr, "%s\n", err); @@ -831,27 +644,31 @@ void loadServerConfigFromString(char *config) { * Both filename and options can be NULL, in such a case are considered * empty. This way loadServerConfig can be used to just load a file or * just load a string. */ -void loadServerConfig(char *filename, char *options) { +void loadServerConfig(char *filename, char config_from_stdin, char *options) { sds config = sdsempty(); char buf[CONFIG_MAX_LINE+1]; + FILE *fp; /* Load the file content */ if (filename) { - FILE *fp; - - if (filename[0] == '-' && filename[1] == '\0') { - fp = stdin; - } else { - if ((fp = fopen(filename,"r")) == NULL) { - serverLog(LL_WARNING, - "Fatal error, can't open config file '%s'", filename); - exit(1); - } + if ((fp = fopen(filename,"r")) == NULL) { + serverLog(LL_WARNING, + "Fatal error, can't open config file '%s': %s", + filename, strerror(errno)); + exit(1); } while(fgets(buf,CONFIG_MAX_LINE+1,fp) != NULL) config = sdscat(config,buf); - if (fp != stdin) fclose(fp); + fclose(fp); + } + /* Append content from stdin */ + if (config_from_stdin) { + serverLog(LL_WARNING,"Reading config from stdin"); + fp = stdin; + while(fgets(buf,CONFIG_MAX_LINE+1,fp) != NULL) + config = sdscat(config,buf); } + /* Append the additional options */ if (options) { config = sdscat(config,"\n"); @@ -884,12 +701,6 @@ void loadServerConfig(char *filename, char *options) { if (err || ll < 0) goto badfmt; \ _var = ll; -#define config_set_enum_field(_name,_var,_enumvar) \ - } else if (!strcasecmp(c->argv[2]->ptr,_name)) { \ - int enumval = configEnumGetValue(_enumvar,o->ptr); \ - if (enumval == INT_MIN) goto badfmt; \ - _var = enumval; - #define config_set_special_field(_name) \ } else if (!strcasecmp(c->argv[2]->ptr,_name)) { @@ -903,69 +714,47 @@ void configSetCommand(client *c) { robj *o; long long ll; int err; + const char *errstr = NULL; serverAssertWithInfo(c,c->argv[2],sdsEncodedObject(c->argv[2])); serverAssertWithInfo(c,c->argv[3],sdsEncodedObject(c->argv[3])); o = c->argv[3]; + /* Iterate the configs that are standard */ + for (standardConfig *config = configs; config->name != NULL; config++) { + if (!(config->flags & IMMUTABLE_CONFIG) && + (!strcasecmp(c->argv[2]->ptr,config->name) || + (config->alias && !strcasecmp(c->argv[2]->ptr,config->alias)))) + { + if (config->flags & SENSITIVE_CONFIG) { + redactClientCommandArgument(c,3); + } + if (!config->interface.set(config->data,o->ptr,1,&errstr)) { + goto badfmt; + } + addReply(c,shared.ok); + return; + } + } + if (0) { /* this starts the config_set macros else-if chain. */ /* Special fields that can't be handled with general macros. */ - config_set_special_field("dbfilename") { - if (!pathIsBaseName(o->ptr)) { - addReplyError(c, "dbfilename can't be a path, just a filename"); + config_set_special_field("bind") { + int vlen; + sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen); + + if (vlen < 1 || vlen > CONFIG_BINDADDR_MAX) { + addReplyError(c, "Too many bind addresses specified."); + sdsfreesplitres(v, vlen); return; } - zfree(server.rdb_filename); - server.rdb_filename = zstrdup(o->ptr); - } config_set_special_field("requirepass") { - if (sdslen(o->ptr) > CONFIG_AUTHPASS_MAX_LEN) goto badfmt; - zfree(server.requirepass); - server.requirepass = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; - } config_set_special_field("masterauth") { - zfree(server.masterauth); - server.masterauth = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; - } config_set_special_field("cluster-announce-ip") { - zfree(server.cluster_announce_ip); - server.cluster_announce_ip = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; - } config_set_special_field("maxclients") { - int orig_value = server.maxclients; - - if (getLongLongFromObject(o,&ll) == C_ERR || ll < 1) goto badfmt; - - /* Try to check if the OS is capable of supporting so many FDs. */ - server.maxclients = ll; - if (ll > orig_value) { - adjustOpenFilesLimit(); - if (server.maxclients != ll) { - addReplyErrorFormat(c,"The operating system is not able to handle the specified number of clients, try with %d", server.maxclients); - server.maxclients = orig_value; - return; - } - if ((unsigned int) aeGetSetSize(server.el) < - server.maxclients + CONFIG_FDSET_INCR) - { - if (aeResizeSetSize(server.el, - server.maxclients + CONFIG_FDSET_INCR) == AE_ERR) - { - addReplyError(c,"The event loop API used by Redis is not able to handle the specified number of clients"); - server.maxclients = orig_value; - return; - } - } - } - } config_set_special_field("appendonly") { - int enable = yesnotoi(o->ptr); - - if (enable == -1) goto badfmt; - if (enable == 0 && server.aof_state != AOF_OFF) { - stopAppendOnly(); - } else if (enable && server.aof_state == AOF_OFF) { - if (startAppendOnly() == C_ERR) { - addReplyError(c, - "Unable to turn on AOF. Check server logs."); - return; - } + + if (changeBindAddr(v, vlen) == C_ERR) { + addReplyError(c, "Failed to bind to specified addresses."); + sdsfreesplitres(v, vlen); + return; } + sdsfreesplitres(v, vlen); } config_set_special_field("save") { int vlen, j; sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen); @@ -1051,224 +840,31 @@ void configSetCommand(client *c) { server.client_obuf_limits[class].soft_limit_seconds = soft_seconds; } sdsfreesplitres(v,vlen); + } config_set_special_field("oom-score-adj-values") { + int vlen; + int success = 1; + + sds *v = sdssplitlen(o->ptr, sdslen(o->ptr), " ", 1, &vlen); + if (vlen != CONFIG_OOM_COUNT || updateOOMScoreAdjValues(v, &errstr, 1) == C_ERR) + success = 0; + + sdsfreesplitres(v, vlen); + if (!success) + goto badfmt; } config_set_special_field("notify-keyspace-events") { int flags = keyspaceEventsStringToFlags(o->ptr); if (flags == -1) goto badfmt; server.notify_keyspace_events = flags; - } config_set_special_field_with_alias("slave-announce-ip", - "replica-announce-ip") - { - zfree(server.slave_announce_ip); - server.slave_announce_ip = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; - - /* Boolean fields. - * config_set_bool_field(name,var). */ - } config_set_bool_field( - "rdbcompression", server.rdb_compression) { - } config_set_bool_field( - "repl-disable-tcp-nodelay",server.repl_disable_tcp_nodelay) { - } config_set_bool_field( - "repl-diskless-sync",server.repl_diskless_sync) { - } config_set_bool_field( - "cluster-require-full-coverage",server.cluster_require_full_coverage) { - } config_set_bool_field( - "cluster-slave-no-failover",server.cluster_slave_no_failover) { - } config_set_bool_field( - "cluster-replica-no-failover",server.cluster_slave_no_failover) { - } config_set_bool_field( - "aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync) { - } config_set_bool_field( - "rdb-save-incremental-fsync",server.rdb_save_incremental_fsync) { - } config_set_bool_field( - "aof-load-truncated",server.aof_load_truncated) { - } config_set_bool_field( - "aof-use-rdb-preamble",server.aof_use_rdb_preamble) { - } config_set_bool_field( - "slave-serve-stale-data",server.repl_serve_stale_data) { - } config_set_bool_field( - "replica-serve-stale-data",server.repl_serve_stale_data) { - } config_set_bool_field( - "slave-read-only",server.repl_slave_ro) { - } config_set_bool_field( - "replica-read-only",server.repl_slave_ro) { - } config_set_bool_field( - "slave-ignore-maxmemory",server.repl_slave_ignore_maxmemory) { - } config_set_bool_field( - "replica-ignore-maxmemory",server.repl_slave_ignore_maxmemory) { - } config_set_bool_field( - "activerehashing",server.activerehashing) { - } config_set_bool_field( - "activedefrag",server.active_defrag_enabled) { -#ifndef HAVE_DEFRAG - if (server.active_defrag_enabled) { - server.active_defrag_enabled = 0; - addReplyError(c, - "-DISABLED Active defragmentation cannot be enabled: it " - "requires a Redis server compiled with a modified Jemalloc " - "like the one shipped by default with the Redis source " - "distribution"); - return; - } -#endif - } config_set_bool_field( - "protected-mode",server.protected_mode) { - } config_set_bool_field( - "stop-writes-on-bgsave-error",server.stop_writes_on_bgsave_err) { - } config_set_bool_field( - "lazyfree-lazy-eviction",server.lazyfree_lazy_eviction) { - } config_set_bool_field( - "lazyfree-lazy-expire",server.lazyfree_lazy_expire) { - } config_set_bool_field( - "lazyfree-lazy-server-del",server.lazyfree_lazy_server_del) { - } config_set_bool_field( - "slave-lazy-flush",server.repl_slave_lazy_flush) { - } config_set_bool_field( - "replica-lazy-flush",server.repl_slave_lazy_flush) { - } config_set_bool_field( - "no-appendfsync-on-rewrite",server.aof_no_fsync_on_rewrite) { - } config_set_bool_field( - "dynamic-hz",server.dynamic_hz) { - /* Numerical fields. * config_set_numerical_field(name,var,min,max) */ - } config_set_numerical_field( - "tcp-keepalive",server.tcpkeepalive,0,INT_MAX) { - } config_set_numerical_field( - "maxmemory-samples",server.maxmemory_samples,1,INT_MAX) { - } config_set_numerical_field( - "lfu-log-factor",server.lfu_log_factor,0,INT_MAX) { - } config_set_numerical_field( - "lfu-decay-time",server.lfu_decay_time,0,INT_MAX) { - } config_set_numerical_field( - "timeout",server.maxidletime,0,INT_MAX) { - } config_set_numerical_field( - "active-defrag-threshold-lower",server.active_defrag_threshold_lower,0,1000) { - } config_set_numerical_field( - "active-defrag-threshold-upper",server.active_defrag_threshold_upper,0,1000) { - } config_set_memory_field( - "active-defrag-ignore-bytes",server.active_defrag_ignore_bytes) { - } config_set_numerical_field( - "active-defrag-cycle-min",server.active_defrag_cycle_min,1,99) { - } config_set_numerical_field( - "active-defrag-cycle-max",server.active_defrag_cycle_max,1,99) { - } config_set_numerical_field( - "active-defrag-max-scan-fields",server.active_defrag_max_scan_fields,1,LONG_MAX) { - } config_set_numerical_field( - "auto-aof-rewrite-percentage",server.aof_rewrite_perc,0,INT_MAX){ - } config_set_numerical_field( - "hash-max-ziplist-entries",server.hash_max_ziplist_entries,0,LONG_MAX) { - } config_set_numerical_field( - "hash-max-ziplist-value",server.hash_max_ziplist_value,0,LONG_MAX) { - } config_set_numerical_field( - "stream-node-max-bytes",server.stream_node_max_bytes,0,LONG_MAX) { - } config_set_numerical_field( - "stream-node-max-entries",server.stream_node_max_entries,0,LLONG_MAX) { - } config_set_numerical_field( - "list-max-ziplist-size",server.list_max_ziplist_size,INT_MIN,INT_MAX) { - } config_set_numerical_field( - "list-compress-depth",server.list_compress_depth,0,INT_MAX) { - } config_set_numerical_field( - "set-max-intset-entries",server.set_max_intset_entries,0,LONG_MAX) { - } config_set_numerical_field( - "zset-max-ziplist-entries",server.zset_max_ziplist_entries,0,LONG_MAX) { - } config_set_numerical_field( - "zset-max-ziplist-value",server.zset_max_ziplist_value,0,LONG_MAX) { - } config_set_numerical_field( - "hll-sparse-max-bytes",server.hll_sparse_max_bytes,0,LONG_MAX) { - } config_set_numerical_field( - "lua-time-limit",server.lua_time_limit,0,LONG_MAX) { - } config_set_numerical_field( - "slowlog-log-slower-than",server.slowlog_log_slower_than,-1,LLONG_MAX) { - } config_set_numerical_field( - "slowlog-max-len",ll,0,LONG_MAX) { - /* Cast to unsigned. */ - server.slowlog_max_len = (unsigned long)ll; - } config_set_numerical_field( - "latency-monitor-threshold",server.latency_monitor_threshold,0,LLONG_MAX){ - } config_set_numerical_field( - "repl-ping-slave-period",server.repl_ping_slave_period,1,INT_MAX) { - } config_set_numerical_field( - "repl-ping-replica-period",server.repl_ping_slave_period,1,INT_MAX) { - } config_set_numerical_field( - "repl-timeout",server.repl_timeout,1,INT_MAX) { - } config_set_numerical_field( - "repl-backlog-ttl",server.repl_backlog_time_limit,0,LONG_MAX) { - } config_set_numerical_field( - "repl-diskless-sync-delay",server.repl_diskless_sync_delay,0,INT_MAX) { - } config_set_numerical_field( - "slave-priority",server.slave_priority,0,INT_MAX) { - } config_set_numerical_field( - "replica-priority",server.slave_priority,0,INT_MAX) { - } config_set_numerical_field( - "slave-announce-port",server.slave_announce_port,0,65535) { - } config_set_numerical_field( - "replica-announce-port",server.slave_announce_port,0,65535) { - } config_set_numerical_field( - "min-slaves-to-write",server.repl_min_slaves_to_write,0,INT_MAX) { - refreshGoodSlavesCount(); - } config_set_numerical_field( - "min-replicas-to-write",server.repl_min_slaves_to_write,0,INT_MAX) { - refreshGoodSlavesCount(); - } config_set_numerical_field( - "min-slaves-max-lag",server.repl_min_slaves_max_lag,0,INT_MAX) { - refreshGoodSlavesCount(); - } config_set_numerical_field( - "min-replicas-max-lag",server.repl_min_slaves_max_lag,0,INT_MAX) { - refreshGoodSlavesCount(); - } config_set_numerical_field( - "cluster-node-timeout",server.cluster_node_timeout,0,LLONG_MAX) { - } config_set_numerical_field( - "cluster-announce-port",server.cluster_announce_port,0,65535) { - } config_set_numerical_field( - "cluster-announce-bus-port",server.cluster_announce_bus_port,0,65535) { - } config_set_numerical_field( - "cluster-migration-barrier",server.cluster_migration_barrier,0,INT_MAX){ - } config_set_numerical_field( - "cluster-slave-validity-factor",server.cluster_slave_validity_factor,0,INT_MAX) { - } config_set_numerical_field( - "cluster-replica-validity-factor",server.cluster_slave_validity_factor,0,INT_MAX) { - } config_set_numerical_field( - "hz",server.config_hz,0,INT_MAX) { - /* Hz is more an hint from the user, so we accept values out of range - * but cap them to reasonable values. */ - if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; - if (server.config_hz > CONFIG_MAX_HZ) server.config_hz = CONFIG_MAX_HZ; } config_set_numerical_field( "watchdog-period",ll,0,INT_MAX) { if (ll) enableWatchdog(ll); else disableWatchdog(); - - /* Memory fields. - * config_set_memory_field(name,var) */ - } config_set_memory_field("maxmemory",server.maxmemory) { - if (server.maxmemory) { - if (server.maxmemory < zmalloc_used_memory()) { - serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy."); - } - freeMemoryIfNeededAndSafe(); - } - } config_set_memory_field( - "proto-max-bulk-len",server.proto_max_bulk_len) { - } config_set_memory_field( - "client-query-buffer-limit",server.client_max_querybuf_len) { - } config_set_memory_field("repl-backlog-size",ll) { - resizeReplicationBacklog(ll); - } config_set_memory_field("auto-aof-rewrite-min-size",ll) { - server.aof_rewrite_min_size = ll; - - /* Enumeration fields. - * config_set_enum_field(name,var,enum_var) */ - } config_set_enum_field( - "loglevel",server.verbosity,loglevel_enum) { - } config_set_enum_field( - "maxmemory-policy",server.maxmemory_policy,maxmemory_policy_enum) { - } config_set_enum_field( - "appendfsync",server.aof_fsync,aof_fsync_enum) { - - /* Everyhing else is an error... */ + /* Everything else is an error... */ } config_set_else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", (char*)c->argv[2]->ptr); @@ -1280,9 +876,16 @@ void configSetCommand(client *c) { return; badfmt: /* Bad format errors */ - addReplyErrorFormat(c,"Invalid argument '%s' for CONFIG SET '%s'", - (char*)o->ptr, - (char*)c->argv[2]->ptr); + if (errstr) { + addReplyErrorFormat(c,"Invalid argument '%s' for CONFIG SET '%s' - %s", + (char*)o->ptr, + (char*)c->argv[2]->ptr, + errstr); + } else { + addReplyErrorFormat(c,"Invalid argument '%s' for CONFIG SET '%s'", + (char*)o->ptr, + (char*)c->argv[2]->ptr); + } } /*----------------------------------------------------------------------------- @@ -1295,7 +898,7 @@ void configSetCommand(client *c) { addReplyBulkCString(c,_var ? _var : ""); \ matches++; \ } \ -} while(0); +} while(0) #define config_get_bool_field(_name,_var) do { \ if (stringmatch(pattern,_name,1)) { \ @@ -1303,7 +906,7 @@ void configSetCommand(client *c) { addReplyBulkCString(c,_var ? "yes" : "no"); \ matches++; \ } \ -} while(0); +} while(0) #define config_get_numerical_field(_name,_var) do { \ if (stringmatch(pattern,_name,1)) { \ @@ -1312,181 +915,38 @@ void configSetCommand(client *c) { addReplyBulkCString(c,buf); \ matches++; \ } \ -} while(0); - -#define config_get_enum_field(_name,_var,_enumvar) do { \ - if (stringmatch(pattern,_name,1)) { \ - addReplyBulkCString(c,_name); \ - addReplyBulkCString(c,configEnumGetNameOrUnknown(_enumvar,_var)); \ - matches++; \ - } \ -} while(0); +} while(0) void configGetCommand(client *c) { robj *o = c->argv[2]; - void *replylen = addDeferredMultiBulkLength(c); + void *replylen = addReplyDeferredLen(c); char *pattern = o->ptr; char buf[128]; int matches = 0; serverAssertWithInfo(c,o,sdsEncodedObject(o)); + /* Iterate the configs that are standard */ + for (standardConfig *config = configs; config->name != NULL; config++) { + if (stringmatch(pattern,config->name,1)) { + addReplyBulkCString(c,config->name); + config->interface.get(c,config->data); + matches++; + } + if (config->alias && stringmatch(pattern,config->alias,1)) { + addReplyBulkCString(c,config->alias); + config->interface.get(c,config->data); + matches++; + } + } + /* String values */ - config_get_string_field("dbfilename",server.rdb_filename); - config_get_string_field("requirepass",server.requirepass); - config_get_string_field("masterauth",server.masterauth); - config_get_string_field("cluster-announce-ip",server.cluster_announce_ip); - config_get_string_field("unixsocket",server.unixsocket); config_get_string_field("logfile",server.logfile); - config_get_string_field("pidfile",server.pidfile); - config_get_string_field("slave-announce-ip",server.slave_announce_ip); - config_get_string_field("replica-announce-ip",server.slave_announce_ip); /* Numerical values */ - config_get_numerical_field("maxmemory",server.maxmemory); - config_get_numerical_field("proto-max-bulk-len",server.proto_max_bulk_len); - config_get_numerical_field("client-query-buffer-limit",server.client_max_querybuf_len); - config_get_numerical_field("maxmemory-samples",server.maxmemory_samples); - config_get_numerical_field("lfu-log-factor",server.lfu_log_factor); - config_get_numerical_field("lfu-decay-time",server.lfu_decay_time); - config_get_numerical_field("timeout",server.maxidletime); - config_get_numerical_field("active-defrag-threshold-lower",server.active_defrag_threshold_lower); - config_get_numerical_field("active-defrag-threshold-upper",server.active_defrag_threshold_upper); - config_get_numerical_field("active-defrag-ignore-bytes",server.active_defrag_ignore_bytes); - config_get_numerical_field("active-defrag-cycle-min",server.active_defrag_cycle_min); - config_get_numerical_field("active-defrag-cycle-max",server.active_defrag_cycle_max); - config_get_numerical_field("active-defrag-max-scan-fields",server.active_defrag_max_scan_fields); - config_get_numerical_field("auto-aof-rewrite-percentage", - server.aof_rewrite_perc); - config_get_numerical_field("auto-aof-rewrite-min-size", - server.aof_rewrite_min_size); - config_get_numerical_field("hash-max-ziplist-entries", - server.hash_max_ziplist_entries); - config_get_numerical_field("hash-max-ziplist-value", - server.hash_max_ziplist_value); - config_get_numerical_field("stream-node-max-bytes", - server.stream_node_max_bytes); - config_get_numerical_field("stream-node-max-entries", - server.stream_node_max_entries); - config_get_numerical_field("list-max-ziplist-size", - server.list_max_ziplist_size); - config_get_numerical_field("list-compress-depth", - server.list_compress_depth); - config_get_numerical_field("set-max-intset-entries", - server.set_max_intset_entries); - config_get_numerical_field("zset-max-ziplist-entries", - server.zset_max_ziplist_entries); - config_get_numerical_field("zset-max-ziplist-value", - server.zset_max_ziplist_value); - config_get_numerical_field("hll-sparse-max-bytes", - server.hll_sparse_max_bytes); - config_get_numerical_field("lua-time-limit",server.lua_time_limit); - config_get_numerical_field("slowlog-log-slower-than", - server.slowlog_log_slower_than); - config_get_numerical_field("latency-monitor-threshold", - server.latency_monitor_threshold); - config_get_numerical_field("slowlog-max-len", - server.slowlog_max_len); - config_get_numerical_field("port",server.port); - config_get_numerical_field("cluster-announce-port",server.cluster_announce_port); - config_get_numerical_field("cluster-announce-bus-port",server.cluster_announce_bus_port); - config_get_numerical_field("tcp-backlog",server.tcp_backlog); - config_get_numerical_field("databases",server.dbnum); - config_get_numerical_field("repl-ping-slave-period",server.repl_ping_slave_period); - config_get_numerical_field("repl-ping-replica-period",server.repl_ping_slave_period); - config_get_numerical_field("repl-timeout",server.repl_timeout); - config_get_numerical_field("repl-backlog-size",server.repl_backlog_size); - config_get_numerical_field("repl-backlog-ttl",server.repl_backlog_time_limit); - config_get_numerical_field("maxclients",server.maxclients); config_get_numerical_field("watchdog-period",server.watchdog_period); - config_get_numerical_field("slave-priority",server.slave_priority); - config_get_numerical_field("replica-priority",server.slave_priority); - config_get_numerical_field("slave-announce-port",server.slave_announce_port); - config_get_numerical_field("replica-announce-port",server.slave_announce_port); - config_get_numerical_field("min-slaves-to-write",server.repl_min_slaves_to_write); - config_get_numerical_field("min-replicas-to-write",server.repl_min_slaves_to_write); - config_get_numerical_field("min-slaves-max-lag",server.repl_min_slaves_max_lag); - config_get_numerical_field("min-replicas-max-lag",server.repl_min_slaves_max_lag); - config_get_numerical_field("hz",server.config_hz); - config_get_numerical_field("cluster-node-timeout",server.cluster_node_timeout); - config_get_numerical_field("cluster-migration-barrier",server.cluster_migration_barrier); - config_get_numerical_field("cluster-slave-validity-factor",server.cluster_slave_validity_factor); - config_get_numerical_field("cluster-replica-validity-factor",server.cluster_slave_validity_factor); - config_get_numerical_field("repl-diskless-sync-delay",server.repl_diskless_sync_delay); - config_get_numerical_field("tcp-keepalive",server.tcpkeepalive); - - /* Bool (yes/no) values */ - config_get_bool_field("cluster-require-full-coverage", - server.cluster_require_full_coverage); - config_get_bool_field("cluster-slave-no-failover", - server.cluster_slave_no_failover); - config_get_bool_field("cluster-replica-no-failover", - server.cluster_slave_no_failover); - config_get_bool_field("no-appendfsync-on-rewrite", - server.aof_no_fsync_on_rewrite); - config_get_bool_field("slave-serve-stale-data", - server.repl_serve_stale_data); - config_get_bool_field("replica-serve-stale-data", - server.repl_serve_stale_data); - config_get_bool_field("slave-read-only", - server.repl_slave_ro); - config_get_bool_field("replica-read-only", - server.repl_slave_ro); - config_get_bool_field("slave-ignore-maxmemory", - server.repl_slave_ignore_maxmemory); - config_get_bool_field("replica-ignore-maxmemory", - server.repl_slave_ignore_maxmemory); - config_get_bool_field("stop-writes-on-bgsave-error", - server.stop_writes_on_bgsave_err); - config_get_bool_field("daemonize", server.daemonize); - config_get_bool_field("rdbcompression", server.rdb_compression); - config_get_bool_field("rdbchecksum", server.rdb_checksum); - config_get_bool_field("activerehashing", server.activerehashing); - config_get_bool_field("activedefrag", server.active_defrag_enabled); - config_get_bool_field("protected-mode", server.protected_mode); - config_get_bool_field("repl-disable-tcp-nodelay", - server.repl_disable_tcp_nodelay); - config_get_bool_field("repl-diskless-sync", - server.repl_diskless_sync); - config_get_bool_field("aof-rewrite-incremental-fsync", - server.aof_rewrite_incremental_fsync); - config_get_bool_field("rdb-save-incremental-fsync", - server.rdb_save_incremental_fsync); - config_get_bool_field("aof-load-truncated", - server.aof_load_truncated); - config_get_bool_field("aof-use-rdb-preamble", - server.aof_use_rdb_preamble); - config_get_bool_field("lazyfree-lazy-eviction", - server.lazyfree_lazy_eviction); - config_get_bool_field("lazyfree-lazy-expire", - server.lazyfree_lazy_expire); - config_get_bool_field("lazyfree-lazy-server-del", - server.lazyfree_lazy_server_del); - config_get_bool_field("slave-lazy-flush", - server.repl_slave_lazy_flush); - config_get_bool_field("replica-lazy-flush", - server.repl_slave_lazy_flush); - config_get_bool_field("dynamic-hz", - server.dynamic_hz); - - /* Enum values */ - config_get_enum_field("maxmemory-policy", - server.maxmemory_policy,maxmemory_policy_enum); - config_get_enum_field("loglevel", - server.verbosity,loglevel_enum); - config_get_enum_field("supervised", - server.supervised_mode,supervised_mode_enum); - config_get_enum_field("appendfsync", - server.aof_fsync,aof_fsync_enum); - config_get_enum_field("syslog-facility", - server.syslog_facility,syslog_facility_enum); /* Everything we can't handle with macros follows. */ - if (stringmatch(pattern,"appendonly",1)) { - addReplyBulkCString(c,"appendonly"); - addReplyBulkCString(c,server.aof_state == AOF_OFF ? "no" : "yes"); - matches++; - } if (stringmatch(pattern,"dir",1)) { char buf[1024]; @@ -1533,7 +993,7 @@ void configGetCommand(client *c) { } if (stringmatch(pattern,"unixsocketperm",1)) { char buf[32]; - snprintf(buf,sizeof(buf),"%o",server.unixsocketperm); + snprintf(buf,sizeof(buf),"%lo",(unsigned long) server.unixsocketperm); addReplyBulkCString(c,"unixsocketperm"); addReplyBulkCString(c,buf); matches++; @@ -1555,12 +1015,10 @@ void configGetCommand(client *c) { matches++; } if (stringmatch(pattern,"notify-keyspace-events",1)) { - robj *flagsobj = createObject(OBJ_STRING, - keyspaceEventsFlagsToString(server.notify_keyspace_events)); + sds flags = keyspaceEventsFlagsToString(server.notify_keyspace_events); addReplyBulkCString(c,"notify-keyspace-events"); - addReplyBulk(c,flagsobj); - decrRefCount(flagsobj); + addReplyBulkSds(c,flags); matches++; } if (stringmatch(pattern,"bind",1)) { @@ -1571,7 +1029,24 @@ void configGetCommand(client *c) { sdsfree(aux); matches++; } - setDeferredMultiBulkLength(c,replylen,matches*2); + + if (stringmatch(pattern,"oom-score-adj-values",0)) { + sds buf = sdsempty(); + int j; + + for (j = 0; j < CONFIG_OOM_COUNT; j++) { + buf = sdscatprintf(buf,"%d", server.oom_score_adj_values[j]); + if (j != CONFIG_OOM_COUNT-1) + buf = sdscatlen(buf," ",1); + } + + addReplyBulkCString(c,"oom-score-adj-values"); + addReplyBulkCString(c,buf); + sdsfree(buf); + matches++; + } + + setDeferredMapLen(c,replylen,matches); } /*----------------------------------------------------------------------------- @@ -1598,7 +1073,8 @@ dictType optionToLineDictType = { NULL, /* val dup */ dictSdsKeyCaseCompare, /* key compare */ dictSdsDestructor, /* key destructor */ - dictListDestructor /* val destructor */ + dictListDestructor, /* val destructor */ + NULL /* allow to expand */ }; dictType optionSetDictType = { @@ -1607,7 +1083,8 @@ dictType optionSetDictType = { NULL, /* val dup */ dictSdsKeyCaseCompare, /* key compare */ dictSdsDestructor, /* key destructor */ - NULL /* val destructor */ + NULL, /* val destructor */ + NULL /* allow to expand */ }; /* The config rewrite state. */ @@ -1618,6 +1095,8 @@ struct rewriteConfigState { sds *lines; /* Current lines as an array of sds strings */ int has_tail; /* True if we already added directives that were not present in the original config file. */ + int force_all; /* True if we want all keywords to be force + written. Currently only used for testing. */ }; /* Append the new line to the current configuration state. */ @@ -1664,6 +1143,7 @@ struct rewriteConfigState *rewriteConfigReadOldFile(char *path) { state->numlines = 0; state->lines = NULL; state->has_tail = 0; + state->force_all = 0; if (fp == NULL) return state; /* Read the old file line by line, populate the state. */ @@ -1707,13 +1187,22 @@ struct rewriteConfigState *rewriteConfigReadOldFile(char *path) { char *p = strstr(argv[0],"slave"); if (p) { sds alt = sdsempty(); - alt = sdscatlen(alt,argv[0],p-argv[0]);; + alt = sdscatlen(alt,argv[0],p-argv[0]); alt = sdscatlen(alt,"replica",7); alt = sdscatlen(alt,p+5,strlen(p+5)); sdsfree(argv[0]); argv[0] = alt; } - rewriteConfigAddLineNumberToOption(state,argv[0],linenum); + /* If this is sentinel config, we use sentinel "sentinel " as option + to avoid messing up the sequence. */ + if (server.sentinel_mode && argc > 1 && !strcasecmp(argv[0],"sentinel")) { + sds sentinelOption = sdsempty(); + sentinelOption = sdscatfmt(sentinelOption,"%S %S",argv[0],argv[1]); + rewriteConfigAddLineNumberToOption(state,sentinelOption,linenum); + sdsfree(sentinelOption); + } else { + rewriteConfigAddLineNumberToOption(state,argv[0],linenum); + } sdsfreesplitres(argv,argc); } fclose(fp); @@ -1742,7 +1231,7 @@ void rewriteConfigRewriteLine(struct rewriteConfigState *state, const char *opti rewriteConfigMarkAsProcessed(state,option); - if (!l && !force) { + if (!l && !force && !state->force_all) { /* Option not used previously, and we are not forced to use it. */ sdsfree(line); sdsfree(o); @@ -1790,7 +1279,7 @@ int rewriteConfigFormatMemory(char *buf, size_t len, long long bytes) { } /* Rewrite a simple "option-name " configuration option. */ -void rewriteConfigBytesOption(struct rewriteConfigState *state, char *option, long long value, long long defvalue) { +void rewriteConfigBytesOption(struct rewriteConfigState *state, const char *option, long long value, long long defvalue) { char buf[64]; int force = value != defvalue; sds line; @@ -1801,7 +1290,7 @@ void rewriteConfigBytesOption(struct rewriteConfigState *state, char *option, lo } /* Rewrite a yes/no option. */ -void rewriteConfigYesNoOption(struct rewriteConfigState *state, char *option, int value, int defvalue) { +void rewriteConfigYesNoOption(struct rewriteConfigState *state, const char *option, int value, int defvalue) { int force = value != defvalue; sds line = sdscatprintf(sdsempty(),"%s %s",option, value ? "yes" : "no"); @@ -1810,7 +1299,7 @@ void rewriteConfigYesNoOption(struct rewriteConfigState *state, char *option, in } /* Rewrite a string option. */ -void rewriteConfigStringOption(struct rewriteConfigState *state, char *option, char *value, char *defvalue) { +void rewriteConfigStringOption(struct rewriteConfigState *state, const char *option, char *value, const char *defvalue) { int force = 1; sds line; @@ -1831,15 +1320,37 @@ void rewriteConfigStringOption(struct rewriteConfigState *state, char *option, c rewriteConfigRewriteLine(state,option,line,force); } +/* Rewrite a SDS string option. */ +void rewriteConfigSdsOption(struct rewriteConfigState *state, const char *option, sds value, const sds defvalue) { + int force = 1; + sds line; + + /* If there is no value set, we don't want the SDS option + * to be present in the configuration at all. */ + if (value == NULL) { + rewriteConfigMarkAsProcessed(state, option); + return; + } + + /* Set force to zero if the value is set to its default. */ + if (defvalue && sdscmp(value, defvalue) == 0) force = 0; + + line = sdsnew(option); + line = sdscatlen(line, " ", 1); + line = sdscatrepr(line, value, sdslen(value)); + + rewriteConfigRewriteLine(state, option, line, force); +} + /* Rewrite a numerical (long long range) option. */ -void rewriteConfigNumericalOption(struct rewriteConfigState *state, char *option, long long value, long long defvalue) { +void rewriteConfigNumericalOption(struct rewriteConfigState *state, const char *option, long long value, long long defvalue) { int force = value != defvalue; sds line = sdscatprintf(sdsempty(),"%s %lld",option,value); rewriteConfigRewriteLine(state,option,line,force); } -/* Rewrite a octal option. */ +/* Rewrite an octal option. */ void rewriteConfigOctalOption(struct rewriteConfigState *state, char *option, int value, int defvalue) { int force = value != defvalue; sds line = sdscatprintf(sdsempty(),"%s %o",option,value); @@ -1850,7 +1361,7 @@ void rewriteConfigOctalOption(struct rewriteConfigState *state, char *option, in /* Rewrite an enumeration option. It takes as usually state and option name, * and in addition the enumeration array and the default value for the * option. */ -void rewriteConfigEnumOption(struct rewriteConfigState *state, char *option, int value, configEnum *ce, int defval) { +void rewriteConfigEnumOption(struct rewriteConfigState *state, const char *option, int value, configEnum *ce, int defval) { sds line; const char *name = configEnumGetNameOrUnknown(ce,value); int force = value != defval; @@ -1859,35 +1370,66 @@ void rewriteConfigEnumOption(struct rewriteConfigState *state, char *option, int rewriteConfigRewriteLine(state,option,line,force); } -/* Rewrite the syslog-facility option. */ -void rewriteConfigSyslogfacilityOption(struct rewriteConfigState *state) { - int value = server.syslog_facility; - int force = value != LOG_LOCAL0; - const char *name = NULL, *option = "syslog-facility"; - sds line; - - name = configEnumGetNameOrUnknown(syslog_facility_enum,value); - line = sdscatprintf(sdsempty(),"%s %s",option,name); - rewriteConfigRewriteLine(state,option,line,force); -} - /* Rewrite the save option. */ void rewriteConfigSaveOption(struct rewriteConfigState *state) { int j; sds line; - /* Note that if there are no save parameters at all, all the current - * config line with "save" will be detected as orphaned and deleted, - * resulting into no RDB persistence as expected. */ - for (j = 0; j < server.saveparamslen; j++) { - line = sdscatprintf(sdsempty(),"save %ld %d", - (long) server.saveparams[j].seconds, server.saveparams[j].changes); - rewriteConfigRewriteLine(state,"save",line,1); + /* In Sentinel mode we don't need to rewrite the save parameters */ + if (server.sentinel_mode) { + rewriteConfigMarkAsProcessed(state,"save"); + return; } + + /* Rewrite save parameters, or an empty 'save ""' line to avoid the + * defaults from being used. + */ + if (!server.saveparamslen) { + rewriteConfigRewriteLine(state,"save",sdsnew("save \"\""),1); + } else { + for (j = 0; j < server.saveparamslen; j++) { + line = sdscatprintf(sdsempty(),"save %ld %d", + (long) server.saveparams[j].seconds, server.saveparams[j].changes); + rewriteConfigRewriteLine(state,"save",line,1); + } + } + /* Mark "save" as processed in case server.saveparamslen is zero. */ rewriteConfigMarkAsProcessed(state,"save"); } +/* Rewrite the user option. */ +void rewriteConfigUserOption(struct rewriteConfigState *state) { + /* If there is a user file defined we just mark this configuration + * directive as processed, so that all the lines containing users + * inside the config file gets discarded. */ + if (server.acl_filename[0] != '\0') { + rewriteConfigMarkAsProcessed(state,"user"); + return; + } + + /* Otherwise scan the list of users and rewrite every line. Note that + * in case the list here is empty, the effect will just be to comment + * all the users directive inside the config file. */ + raxIterator ri; + raxStart(&ri,Users); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + user *u = ri.data; + sds line = sdsnew("user "); + line = sdscatsds(line,u->name); + line = sdscatlen(line," ",1); + sds descr = ACLDescribeUser(u); + line = sdscatsds(line,descr); + sdsfree(descr); + rewriteConfigRewriteLine(state,"user",line,1); + } + raxStop(&ri); + + /* Mark "user" as processed in case there are no defined users. */ + rewriteConfigMarkAsProcessed(state,"user"); +} + /* Rewrite the dir option, always using absolute paths.*/ void rewriteConfigDirOption(struct rewriteConfigState *state) { char cwd[1024]; @@ -1958,6 +1500,26 @@ void rewriteConfigClientoutputbufferlimitOption(struct rewriteConfigState *state } } +/* Rewrite the oom-score-adj-values option. */ +void rewriteConfigOOMScoreAdjValuesOption(struct rewriteConfigState *state) { + int force = 0; + int j; + char *option = "oom-score-adj-values"; + sds line; + + line = sdsnew(option); + line = sdscatlen(line, " ", 1); + for (j = 0; j < CONFIG_OOM_COUNT; j++) { + if (server.oom_score_adj_values[j] != configOOMScoreAdjValuesDefaults[j]) + force = 1; + + line = sdscatprintf(line, "%d", server.oom_score_adj_values[j]); + if (j+1 != CONFIG_OOM_COUNT) + line = sdscatlen(line, " ", 1); + } + rewriteConfigRewriteLine(state,option,line,force); +} + /* Rewrite the bind option. */ void rewriteConfigBindOption(struct rewriteConfigState *state) { int force = 1; @@ -2043,60 +1605,62 @@ void rewriteConfigRemoveOrphaned(struct rewriteConfigState *state) { dictReleaseIterator(di); } -/* This function overwrites the old configuration file with the new content. - * - * 1) The old file length is obtained. - * 2) If the new content is smaller, padding is added. - * 3) A single write(2) call is used to replace the content of the file. - * 4) Later the file is truncated to the length of the new content. - * - * This way we are sure the file is left in a consistent state even if the - * process is stopped between any of the four operations. +/* This function replaces the old configuration file with the new content + * in an atomic manner. * * The function returns 0 on success, otherwise -1 is returned and errno - * set accordingly. */ + * is set accordingly. */ int rewriteConfigOverwriteFile(char *configfile, sds content) { - int retval = 0; - int fd = open(configfile,O_RDWR|O_CREAT,0644); - int content_size = sdslen(content), padding = 0; - struct stat sb; - sds content_padded; - - /* 1) Open the old file (or create a new one if it does not - * exist), get the size. */ - if (fd == -1) return -1; /* errno set by open(). */ - if (fstat(fd,&sb) == -1) { - close(fd); - return -1; /* errno set by fstat(). */ - } - - /* 2) Pad the content at least match the old file size. */ - content_padded = sdsdup(content); - if (content_size < sb.st_size) { - /* If the old file was bigger, pad the content with - * a newline plus as many "#" chars as required. */ - padding = sb.st_size - content_size; - content_padded = sdsgrowzero(content_padded,sb.st_size); - content_padded[content_size] = '\n'; - memset(content_padded+content_size+1,'#',padding-1); - } - - /* 3) Write the new content using a single write(2). */ - if (write(fd,content_padded,strlen(content_padded)) == -1) { - retval = -1; - goto cleanup; - } - - /* 4) Truncate the file to the right length if we used padding. */ - if (padding) { - if (ftruncate(fd,content_size) == -1) { - /* Non critical error... */ - } + int fd = -1; + int retval = -1; + char tmp_conffile[PATH_MAX]; + const char *tmp_suffix = ".XXXXXX"; + size_t offset = 0; + ssize_t written_bytes = 0; + + int tmp_path_len = snprintf(tmp_conffile, sizeof(tmp_conffile), "%s%s", configfile, tmp_suffix); + if (tmp_path_len <= 0 || (unsigned int)tmp_path_len >= sizeof(tmp_conffile)) { + serverLog(LL_WARNING, "Config file full path is too long"); + errno = ENAMETOOLONG; + return retval; + } + +#ifdef _GNU_SOURCE + fd = mkostemp(tmp_conffile, O_CLOEXEC); +#else + /* There's a theoretical chance here to leak the FD if a module thread forks & execv in the middle */ + fd = mkstemp(tmp_conffile); +#endif + + if (fd == -1) { + serverLog(LL_WARNING, "Could not create tmp config file (%s)", strerror(errno)); + return retval; + } + + while (offset < sdslen(content)) { + written_bytes = write(fd, content + offset, sdslen(content) - offset); + if (written_bytes <= 0) { + if (errno == EINTR) continue; /* FD is blocking, no other retryable errors */ + serverLog(LL_WARNING, "Failed after writing (%zd) bytes to tmp config file (%s)", offset, strerror(errno)); + goto cleanup; + } + offset+=written_bytes; + } + + if (fsync(fd)) + serverLog(LL_WARNING, "Could not sync tmp config file to disk (%s)", strerror(errno)); + else if (fchmod(fd, 0644 & ~server.umask) == -1) + serverLog(LL_WARNING, "Could not chmod config file (%s)", strerror(errno)); + else if (rename(tmp_conffile, configfile) == -1) + serverLog(LL_WARNING, "Could not rename tmp config file (%s)", strerror(errno)); + else { + retval = 0; + serverLog(LL_DEBUG, "Rewritten config file (%s) successfully", configfile); } cleanup: - sdsfree(content_padded); close(fd); + if (retval) unlink(tmp_conffile); return retval; } @@ -2106,118 +1670,38 @@ int rewriteConfigOverwriteFile(char *configfile, sds content) { * * Configuration parameters that are at their default value, unless already * explicitly included in the old configuration file, are not rewritten. + * The force_all flag overrides this behavior and forces everything to be + * written. This is currently only used for testing purposes. * * On error -1 is returned and errno is set accordingly, otherwise 0. */ -int rewriteConfig(char *path) { +int rewriteConfig(char *path, int force_all) { struct rewriteConfigState *state; sds newcontent; int retval; /* Step 1: read the old config into our rewrite state. */ if ((state = rewriteConfigReadOldFile(path)) == NULL) return -1; + if (force_all) state->force_all = 1; /* Step 2: rewrite every single option, replacing or appending it inside * the rewrite state. */ - rewriteConfigYesNoOption(state,"daemonize",server.daemonize,0); - rewriteConfigStringOption(state,"pidfile",server.pidfile,CONFIG_DEFAULT_PID_FILE); - rewriteConfigNumericalOption(state,"port",server.port,CONFIG_DEFAULT_SERVER_PORT); - rewriteConfigNumericalOption(state,"cluster-announce-port",server.cluster_announce_port,CONFIG_DEFAULT_CLUSTER_ANNOUNCE_PORT); - rewriteConfigNumericalOption(state,"cluster-announce-bus-port",server.cluster_announce_bus_port,CONFIG_DEFAULT_CLUSTER_ANNOUNCE_BUS_PORT); - rewriteConfigNumericalOption(state,"tcp-backlog",server.tcp_backlog,CONFIG_DEFAULT_TCP_BACKLOG); + /* Iterate the configs that are standard */ + for (standardConfig *config = configs; config->name != NULL; config++) { + config->interface.rewrite(config->data, config->name, state); + } + rewriteConfigBindOption(state); - rewriteConfigStringOption(state,"unixsocket",server.unixsocket,NULL); rewriteConfigOctalOption(state,"unixsocketperm",server.unixsocketperm,CONFIG_DEFAULT_UNIX_SOCKET_PERM); - rewriteConfigNumericalOption(state,"timeout",server.maxidletime,CONFIG_DEFAULT_CLIENT_TIMEOUT); - rewriteConfigNumericalOption(state,"tcp-keepalive",server.tcpkeepalive,CONFIG_DEFAULT_TCP_KEEPALIVE); - rewriteConfigNumericalOption(state,"replica-announce-port",server.slave_announce_port,CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT); - rewriteConfigEnumOption(state,"loglevel",server.verbosity,loglevel_enum,CONFIG_DEFAULT_VERBOSITY); rewriteConfigStringOption(state,"logfile",server.logfile,CONFIG_DEFAULT_LOGFILE); - rewriteConfigYesNoOption(state,"syslog-enabled",server.syslog_enabled,CONFIG_DEFAULT_SYSLOG_ENABLED); - rewriteConfigStringOption(state,"syslog-ident",server.syslog_ident,CONFIG_DEFAULT_SYSLOG_IDENT); - rewriteConfigSyslogfacilityOption(state); rewriteConfigSaveOption(state); - rewriteConfigNumericalOption(state,"databases",server.dbnum,CONFIG_DEFAULT_DBNUM); - rewriteConfigYesNoOption(state,"stop-writes-on-bgsave-error",server.stop_writes_on_bgsave_err,CONFIG_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR); - rewriteConfigYesNoOption(state,"rdbcompression",server.rdb_compression,CONFIG_DEFAULT_RDB_COMPRESSION); - rewriteConfigYesNoOption(state,"rdbchecksum",server.rdb_checksum,CONFIG_DEFAULT_RDB_CHECKSUM); - rewriteConfigStringOption(state,"dbfilename",server.rdb_filename,CONFIG_DEFAULT_RDB_FILENAME); + rewriteConfigUserOption(state); rewriteConfigDirOption(state); rewriteConfigSlaveofOption(state,"replicaof"); - rewriteConfigStringOption(state,"replica-announce-ip",server.slave_announce_ip,CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP); - rewriteConfigStringOption(state,"masterauth",server.masterauth,NULL); - rewriteConfigStringOption(state,"cluster-announce-ip",server.cluster_announce_ip,NULL); - rewriteConfigYesNoOption(state,"replica-serve-stale-data",server.repl_serve_stale_data,CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA); - rewriteConfigYesNoOption(state,"replica-read-only",server.repl_slave_ro,CONFIG_DEFAULT_SLAVE_READ_ONLY); - rewriteConfigYesNoOption(state,"replica-ignore-maxmemory",server.repl_slave_ignore_maxmemory,CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY); - rewriteConfigNumericalOption(state,"repl-ping-replica-period",server.repl_ping_slave_period,CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD); - rewriteConfigNumericalOption(state,"repl-timeout",server.repl_timeout,CONFIG_DEFAULT_REPL_TIMEOUT); - rewriteConfigBytesOption(state,"repl-backlog-size",server.repl_backlog_size,CONFIG_DEFAULT_REPL_BACKLOG_SIZE); - rewriteConfigBytesOption(state,"repl-backlog-ttl",server.repl_backlog_time_limit,CONFIG_DEFAULT_REPL_BACKLOG_TIME_LIMIT); - rewriteConfigYesNoOption(state,"repl-disable-tcp-nodelay",server.repl_disable_tcp_nodelay,CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY); - rewriteConfigYesNoOption(state,"repl-diskless-sync",server.repl_diskless_sync,CONFIG_DEFAULT_REPL_DISKLESS_SYNC); - rewriteConfigNumericalOption(state,"repl-diskless-sync-delay",server.repl_diskless_sync_delay,CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY); - rewriteConfigNumericalOption(state,"replica-priority",server.slave_priority,CONFIG_DEFAULT_SLAVE_PRIORITY); - rewriteConfigNumericalOption(state,"min-replicas-to-write",server.repl_min_slaves_to_write,CONFIG_DEFAULT_MIN_SLAVES_TO_WRITE); - rewriteConfigNumericalOption(state,"min-replicas-max-lag",server.repl_min_slaves_max_lag,CONFIG_DEFAULT_MIN_SLAVES_MAX_LAG); - rewriteConfigStringOption(state,"requirepass",server.requirepass,NULL); - rewriteConfigNumericalOption(state,"maxclients",server.maxclients,CONFIG_DEFAULT_MAX_CLIENTS); - rewriteConfigBytesOption(state,"maxmemory",server.maxmemory,CONFIG_DEFAULT_MAXMEMORY); - rewriteConfigBytesOption(state,"proto-max-bulk-len",server.proto_max_bulk_len,CONFIG_DEFAULT_PROTO_MAX_BULK_LEN); - rewriteConfigBytesOption(state,"client-query-buffer-limit",server.client_max_querybuf_len,PROTO_MAX_QUERYBUF_LEN); - rewriteConfigEnumOption(state,"maxmemory-policy",server.maxmemory_policy,maxmemory_policy_enum,CONFIG_DEFAULT_MAXMEMORY_POLICY); - rewriteConfigNumericalOption(state,"maxmemory-samples",server.maxmemory_samples,CONFIG_DEFAULT_MAXMEMORY_SAMPLES); - rewriteConfigNumericalOption(state,"lfu-log-factor",server.lfu_log_factor,CONFIG_DEFAULT_LFU_LOG_FACTOR); - rewriteConfigNumericalOption(state,"lfu-decay-time",server.lfu_decay_time,CONFIG_DEFAULT_LFU_DECAY_TIME); - rewriteConfigNumericalOption(state,"active-defrag-threshold-lower",server.active_defrag_threshold_lower,CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER); - rewriteConfigNumericalOption(state,"active-defrag-threshold-upper",server.active_defrag_threshold_upper,CONFIG_DEFAULT_DEFRAG_THRESHOLD_UPPER); - rewriteConfigBytesOption(state,"active-defrag-ignore-bytes",server.active_defrag_ignore_bytes,CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES); - rewriteConfigNumericalOption(state,"active-defrag-cycle-min",server.active_defrag_cycle_min,CONFIG_DEFAULT_DEFRAG_CYCLE_MIN); - rewriteConfigNumericalOption(state,"active-defrag-cycle-max",server.active_defrag_cycle_max,CONFIG_DEFAULT_DEFRAG_CYCLE_MAX); - rewriteConfigNumericalOption(state,"active-defrag-max-scan-fields",server.active_defrag_max_scan_fields,CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS); - rewriteConfigYesNoOption(state,"appendonly",server.aof_state != AOF_OFF,0); - rewriteConfigStringOption(state,"appendfilename",server.aof_filename,CONFIG_DEFAULT_AOF_FILENAME); - rewriteConfigEnumOption(state,"appendfsync",server.aof_fsync,aof_fsync_enum,CONFIG_DEFAULT_AOF_FSYNC); - rewriteConfigYesNoOption(state,"no-appendfsync-on-rewrite",server.aof_no_fsync_on_rewrite,CONFIG_DEFAULT_AOF_NO_FSYNC_ON_REWRITE); - rewriteConfigNumericalOption(state,"auto-aof-rewrite-percentage",server.aof_rewrite_perc,AOF_REWRITE_PERC); - rewriteConfigBytesOption(state,"auto-aof-rewrite-min-size",server.aof_rewrite_min_size,AOF_REWRITE_MIN_SIZE); - rewriteConfigNumericalOption(state,"lua-time-limit",server.lua_time_limit,LUA_SCRIPT_TIME_LIMIT); - rewriteConfigYesNoOption(state,"cluster-enabled",server.cluster_enabled,0); rewriteConfigStringOption(state,"cluster-config-file",server.cluster_configfile,CONFIG_DEFAULT_CLUSTER_CONFIG_FILE); - rewriteConfigYesNoOption(state,"cluster-require-full-coverage",server.cluster_require_full_coverage,CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE); - rewriteConfigYesNoOption(state,"cluster-replica-no-failover",server.cluster_slave_no_failover,CLUSTER_DEFAULT_SLAVE_NO_FAILOVER); - rewriteConfigNumericalOption(state,"cluster-node-timeout",server.cluster_node_timeout,CLUSTER_DEFAULT_NODE_TIMEOUT); - rewriteConfigNumericalOption(state,"cluster-migration-barrier",server.cluster_migration_barrier,CLUSTER_DEFAULT_MIGRATION_BARRIER); - rewriteConfigNumericalOption(state,"cluster-replica-validity-factor",server.cluster_slave_validity_factor,CLUSTER_DEFAULT_SLAVE_VALIDITY); - rewriteConfigNumericalOption(state,"slowlog-log-slower-than",server.slowlog_log_slower_than,CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN); - rewriteConfigNumericalOption(state,"latency-monitor-threshold",server.latency_monitor_threshold,CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD); - rewriteConfigNumericalOption(state,"slowlog-max-len",server.slowlog_max_len,CONFIG_DEFAULT_SLOWLOG_MAX_LEN); rewriteConfigNotifykeyspaceeventsOption(state); - rewriteConfigNumericalOption(state,"hash-max-ziplist-entries",server.hash_max_ziplist_entries,OBJ_HASH_MAX_ZIPLIST_ENTRIES); - rewriteConfigNumericalOption(state,"hash-max-ziplist-value",server.hash_max_ziplist_value,OBJ_HASH_MAX_ZIPLIST_VALUE); - rewriteConfigNumericalOption(state,"stream-node-max-bytes",server.stream_node_max_bytes,OBJ_STREAM_NODE_MAX_BYTES); - rewriteConfigNumericalOption(state,"stream-node-max-entries",server.stream_node_max_entries,OBJ_STREAM_NODE_MAX_ENTRIES); - rewriteConfigNumericalOption(state,"list-max-ziplist-size",server.list_max_ziplist_size,OBJ_LIST_MAX_ZIPLIST_SIZE); - rewriteConfigNumericalOption(state,"list-compress-depth",server.list_compress_depth,OBJ_LIST_COMPRESS_DEPTH); - rewriteConfigNumericalOption(state,"set-max-intset-entries",server.set_max_intset_entries,OBJ_SET_MAX_INTSET_ENTRIES); - rewriteConfigNumericalOption(state,"zset-max-ziplist-entries",server.zset_max_ziplist_entries,OBJ_ZSET_MAX_ZIPLIST_ENTRIES); - rewriteConfigNumericalOption(state,"zset-max-ziplist-value",server.zset_max_ziplist_value,OBJ_ZSET_MAX_ZIPLIST_VALUE); - rewriteConfigNumericalOption(state,"hll-sparse-max-bytes",server.hll_sparse_max_bytes,CONFIG_DEFAULT_HLL_SPARSE_MAX_BYTES); - rewriteConfigYesNoOption(state,"activerehashing",server.activerehashing,CONFIG_DEFAULT_ACTIVE_REHASHING); - rewriteConfigYesNoOption(state,"activedefrag",server.active_defrag_enabled,CONFIG_DEFAULT_ACTIVE_DEFRAG); - rewriteConfigYesNoOption(state,"protected-mode",server.protected_mode,CONFIG_DEFAULT_PROTECTED_MODE); rewriteConfigClientoutputbufferlimitOption(state); - rewriteConfigNumericalOption(state,"hz",server.config_hz,CONFIG_DEFAULT_HZ); - rewriteConfigYesNoOption(state,"aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync,CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC); - rewriteConfigYesNoOption(state,"rdb-save-incremental-fsync",server.rdb_save_incremental_fsync,CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC); - rewriteConfigYesNoOption(state,"aof-load-truncated",server.aof_load_truncated,CONFIG_DEFAULT_AOF_LOAD_TRUNCATED); - rewriteConfigYesNoOption(state,"aof-use-rdb-preamble",server.aof_use_rdb_preamble,CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE); - rewriteConfigEnumOption(state,"supervised",server.supervised_mode,supervised_mode_enum,SUPERVISED_NONE); - rewriteConfigYesNoOption(state,"lazyfree-lazy-eviction",server.lazyfree_lazy_eviction,CONFIG_DEFAULT_LAZYFREE_LAZY_EVICTION); - rewriteConfigYesNoOption(state,"lazyfree-lazy-expire",server.lazyfree_lazy_expire,CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE); - rewriteConfigYesNoOption(state,"lazyfree-lazy-server-del",server.lazyfree_lazy_server_del,CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL); - rewriteConfigYesNoOption(state,"replica-lazy-flush",server.repl_slave_lazy_flush,CONFIG_DEFAULT_SLAVE_LAZY_FLUSH); - rewriteConfigYesNoOption(state,"dynamic-hz",server.dynamic_hz,CONFIG_DEFAULT_DYNAMIC_HZ); + rewriteConfigOOMScoreAdjValuesOption(state); /* Rewrite Sentinel config if in Sentinel mode. */ if (server.sentinel_mode) rewriteConfigSentinelOption(state); @@ -2237,6 +1721,856 @@ int rewriteConfig(char *path) { return retval; } +/*----------------------------------------------------------------------------- + * Configs that fit one of the major types and require no special handling + *----------------------------------------------------------------------------*/ +#define LOADBUF_SIZE 256 +static char loadbuf[LOADBUF_SIZE]; + +#define embedCommonConfig(config_name, config_alias, config_flags) \ + .name = (config_name), \ + .alias = (config_alias), \ + .flags = (config_flags), + +#define embedConfigInterface(initfn, setfn, getfn, rewritefn) .interface = { \ + .init = (initfn), \ + .set = (setfn), \ + .get = (getfn), \ + .rewrite = (rewritefn) \ +}, + +/* What follows is the generic config types that are supported. To add a new + * config with one of these types, add it to the standardConfig table with + * the creation macro for each type. + * + * Each type contains the following: + * * A function defining how to load this type on startup. + * * A function defining how to update this type on CONFIG SET. + * * A function defining how to serialize this type on CONFIG SET. + * * A function defining how to rewrite this type on CONFIG REWRITE. + * * A Macro defining how to create this type. + */ + +/* Bool Configs */ +static void boolConfigInit(typeData data) { + *data.yesno.config = data.yesno.default_value; +} + +static int boolConfigSet(typeData data, sds value, int update, const char **err) { + int yn = yesnotoi(value); + if (yn == -1) { + *err = "argument must be 'yes' or 'no'"; + return 0; + } + if (data.yesno.is_valid_fn && !data.yesno.is_valid_fn(yn, err)) + return 0; + int prev = *(data.yesno.config); + *(data.yesno.config) = yn; + if (update && data.yesno.update_fn && !data.yesno.update_fn(yn, prev, err)) { + *(data.yesno.config) = prev; + return 0; + } + return 1; +} + +static void boolConfigGet(client *c, typeData data) { + addReplyBulkCString(c, *data.yesno.config ? "yes" : "no"); +} + +static void boolConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + rewriteConfigYesNoOption(state, name,*(data.yesno.config), data.yesno.default_value); +} + +#define createBoolConfig(name, alias, flags, config_addr, default, is_valid, update) { \ + embedCommonConfig(name, alias, flags) \ + embedConfigInterface(boolConfigInit, boolConfigSet, boolConfigGet, boolConfigRewrite) \ + .data.yesno = { \ + .config = &(config_addr), \ + .default_value = (default), \ + .is_valid_fn = (is_valid), \ + .update_fn = (update), \ + } \ +} + +/* String Configs */ +static void stringConfigInit(typeData data) { + *data.string.config = (data.string.convert_empty_to_null && !data.string.default_value) ? NULL : zstrdup(data.string.default_value); +} + +static int stringConfigSet(typeData data, sds value, int update, const char **err) { + if (data.string.is_valid_fn && !data.string.is_valid_fn(value, err)) + return 0; + char *prev = *data.string.config; + *data.string.config = (data.string.convert_empty_to_null && !value[0]) ? NULL : zstrdup(value); + if (update && data.string.update_fn && !data.string.update_fn(*data.string.config, prev, err)) { + zfree(*data.string.config); + *data.string.config = prev; + return 0; + } + zfree(prev); + return 1; +} + +static void stringConfigGet(client *c, typeData data) { + addReplyBulkCString(c, *data.string.config ? *data.string.config : ""); +} + +static void stringConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + rewriteConfigStringOption(state, name,*(data.string.config), data.string.default_value); +} + +/* SDS Configs */ +static void sdsConfigInit(typeData data) { + *data.sds.config = (data.sds.convert_empty_to_null && !data.sds.default_value) ? NULL: sdsnew(data.sds.default_value); +} + +static int sdsConfigSet(typeData data, sds value, int update, const char **err) { + if (data.sds.is_valid_fn && !data.sds.is_valid_fn(value, err)) + return 0; + sds prev = *data.sds.config; + *data.sds.config = (data.sds.convert_empty_to_null && (sdslen(value) == 0)) ? NULL : sdsdup(value); + if (update && data.sds.update_fn && !data.sds.update_fn(*data.sds.config, prev, err)) { + sdsfree(*data.sds.config); + *data.sds.config = prev; + return 0; + } + sdsfree(prev); + return 1; +} + +static void sdsConfigGet(client *c, typeData data) { + if (*data.sds.config) { + addReplyBulkSds(c, sdsdup(*data.sds.config)); + } else { + addReplyBulkCString(c, ""); + } +} + +static void sdsConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + rewriteConfigSdsOption(state, name, *(data.sds.config), data.sds.default_value ? sdsnew(data.sds.default_value) : NULL); +} + + +#define ALLOW_EMPTY_STRING 0 +#define EMPTY_STRING_IS_NULL 1 + +#define createStringConfig(name, alias, flags, empty_to_null, config_addr, default, is_valid, update) { \ + embedCommonConfig(name, alias, flags) \ + embedConfigInterface(stringConfigInit, stringConfigSet, stringConfigGet, stringConfigRewrite) \ + .data.string = { \ + .config = &(config_addr), \ + .default_value = (default), \ + .is_valid_fn = (is_valid), \ + .update_fn = (update), \ + .convert_empty_to_null = (empty_to_null), \ + } \ +} + +#define createSDSConfig(name, alias, flags, empty_to_null, config_addr, default, is_valid, update) { \ + embedCommonConfig(name, alias, flags) \ + embedConfigInterface(sdsConfigInit, sdsConfigSet, sdsConfigGet, sdsConfigRewrite) \ + .data.sds = { \ + .config = &(config_addr), \ + .default_value = (default), \ + .is_valid_fn = (is_valid), \ + .update_fn = (update), \ + .convert_empty_to_null = (empty_to_null), \ + } \ +} + +/* Enum configs */ +static void enumConfigInit(typeData data) { + *data.enumd.config = data.enumd.default_value; +} + +static int enumConfigSet(typeData data, sds value, int update, const char **err) { + int enumval = configEnumGetValue(data.enumd.enum_value, value); + if (enumval == INT_MIN) { + sds enumerr = sdsnew("argument must be one of the following: "); + configEnum *enumNode = data.enumd.enum_value; + while(enumNode->name != NULL) { + enumerr = sdscatlen(enumerr, enumNode->name, + strlen(enumNode->name)); + enumerr = sdscatlen(enumerr, ", ", 2); + enumNode++; + } + sdsrange(enumerr,0,-3); /* Remove final ", ". */ + + strncpy(loadbuf, enumerr, LOADBUF_SIZE); + loadbuf[LOADBUF_SIZE - 1] = '\0'; + + sdsfree(enumerr); + *err = loadbuf; + return 0; + } + if (data.enumd.is_valid_fn && !data.enumd.is_valid_fn(enumval, err)) + return 0; + int prev = *(data.enumd.config); + *(data.enumd.config) = enumval; + if (update && data.enumd.update_fn && !data.enumd.update_fn(enumval, prev, err)) { + *(data.enumd.config) = prev; + return 0; + } + return 1; +} + +static void enumConfigGet(client *c, typeData data) { + addReplyBulkCString(c, configEnumGetNameOrUnknown(data.enumd.enum_value,*data.enumd.config)); +} + +static void enumConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + rewriteConfigEnumOption(state, name,*(data.enumd.config), data.enumd.enum_value, data.enumd.default_value); +} + +#define createEnumConfig(name, alias, flags, enum, config_addr, default, is_valid, update) { \ + embedCommonConfig(name, alias, flags) \ + embedConfigInterface(enumConfigInit, enumConfigSet, enumConfigGet, enumConfigRewrite) \ + .data.enumd = { \ + .config = &(config_addr), \ + .default_value = (default), \ + .is_valid_fn = (is_valid), \ + .update_fn = (update), \ + .enum_value = (enum), \ + } \ +} + +/* Gets a 'long long val' and sets it into the union, using a macro to get + * compile time type check. */ +#define SET_NUMERIC_TYPE(val) \ + if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { \ + *(data.numeric.config.i) = (int) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_UINT) { \ + *(data.numeric.config.ui) = (unsigned int) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG) { \ + *(data.numeric.config.l) = (long) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG) { \ + *(data.numeric.config.ul) = (unsigned long) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { \ + *(data.numeric.config.ll) = (long long) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG) { \ + *(data.numeric.config.ull) = (unsigned long long) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { \ + *(data.numeric.config.st) = (size_t) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SSIZE_T) { \ + *(data.numeric.config.sst) = (ssize_t) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_OFF_T) { \ + *(data.numeric.config.ot) = (off_t) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_TIME_T) { \ + *(data.numeric.config.tt) = (time_t) val; \ + } + +/* Gets a 'long long val' and sets it with the value from the union, using a + * macro to get compile time type check. */ +#define GET_NUMERIC_TYPE(val) \ + if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { \ + val = *(data.numeric.config.i); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_UINT) { \ + val = *(data.numeric.config.ui); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG) { \ + val = *(data.numeric.config.l); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG) { \ + val = *(data.numeric.config.ul); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { \ + val = *(data.numeric.config.ll); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG) { \ + val = *(data.numeric.config.ull); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { \ + val = *(data.numeric.config.st); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SSIZE_T) { \ + val = *(data.numeric.config.sst); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_OFF_T) { \ + val = *(data.numeric.config.ot); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_TIME_T) { \ + val = *(data.numeric.config.tt); \ + } + +/* Numeric configs */ +static void numericConfigInit(typeData data) { + SET_NUMERIC_TYPE(data.numeric.default_value) +} + +static int numericBoundaryCheck(typeData data, long long ll, const char **err) { + if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG || + data.numeric.numeric_type == NUMERIC_TYPE_UINT || + data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { + /* Boundary check for unsigned types */ + unsigned long long ull = ll; + unsigned long long upper_bound = data.numeric.upper_bound; + unsigned long long lower_bound = data.numeric.lower_bound; + if (ull > upper_bound || ull < lower_bound) { + snprintf(loadbuf, LOADBUF_SIZE, + "argument must be between %llu and %llu inclusive", + lower_bound, + upper_bound); + *err = loadbuf; + return 0; + } + } else { + /* Boundary check for signed types */ + if (ll > data.numeric.upper_bound || ll < data.numeric.lower_bound) { + snprintf(loadbuf, LOADBUF_SIZE, + "argument must be between %lld and %lld inclusive", + data.numeric.lower_bound, + data.numeric.upper_bound); + *err = loadbuf; + return 0; + } + } + return 1; +} + +static int numericConfigSet(typeData data, sds value, int update, const char **err) { + long long ll, prev = 0; + if (data.numeric.is_memory) { + int memerr; + ll = memtoll(value, &memerr); + if (memerr || ll < 0) { + *err = "argument must be a memory value"; + return 0; + } + } else { + if (!string2ll(value, sdslen(value),&ll)) { + *err = "argument couldn't be parsed into an integer" ; + return 0; + } + } + + if (!numericBoundaryCheck(data, ll, err)) + return 0; + + if (data.numeric.is_valid_fn && !data.numeric.is_valid_fn(ll, err)) + return 0; + + GET_NUMERIC_TYPE(prev) + SET_NUMERIC_TYPE(ll) + + if (update && data.numeric.update_fn && !data.numeric.update_fn(ll, prev, err)) { + SET_NUMERIC_TYPE(prev) + return 0; + } + return 1; +} + +static void numericConfigGet(client *c, typeData data) { + char buf[128]; + long long value = 0; + + GET_NUMERIC_TYPE(value) + + ll2string(buf, sizeof(buf), value); + addReplyBulkCString(c, buf); +} + +static void numericConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + long long value = 0; + + GET_NUMERIC_TYPE(value) + + if (data.numeric.is_memory) { + rewriteConfigBytesOption(state, name, value, data.numeric.default_value); + } else { + rewriteConfigNumericalOption(state, name, value, data.numeric.default_value); + } +} + +#define INTEGER_CONFIG 0 +#define MEMORY_CONFIG 1 + +#define embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) { \ + embedCommonConfig(name, alias, flags) \ + embedConfigInterface(numericConfigInit, numericConfigSet, numericConfigGet, numericConfigRewrite) \ + .data.numeric = { \ + .lower_bound = (lower), \ + .upper_bound = (upper), \ + .default_value = (default), \ + .is_valid_fn = (is_valid), \ + .update_fn = (update), \ + .is_memory = (memory), + +#define createIntConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_INT, \ + .config.i = &(config_addr) \ + } \ +} + +#define createUIntConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_UINT, \ + .config.ui = &(config_addr) \ + } \ +} + +#define createLongConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_LONG, \ + .config.l = &(config_addr) \ + } \ +} + +#define createULongConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_ULONG, \ + .config.ul = &(config_addr) \ + } \ +} + +#define createLongLongConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_LONG_LONG, \ + .config.ll = &(config_addr) \ + } \ +} + +#define createULongLongConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_ULONG_LONG, \ + .config.ull = &(config_addr) \ + } \ +} + +#define createSizeTConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_SIZE_T, \ + .config.st = &(config_addr) \ + } \ +} + +#define createSSizeTConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_SSIZE_T, \ + .config.sst = &(config_addr) \ + } \ +} + +#define createTimeTConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_TIME_T, \ + .config.tt = &(config_addr) \ + } \ +} + +#define createOffTConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, flags, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_OFF_T, \ + .config.ot = &(config_addr) \ + } \ +} + +static int isValidActiveDefrag(int val, const char **err) { +#ifndef HAVE_DEFRAG + if (val) { + *err = "Active defragmentation cannot be enabled: it " + "requires a Redis server compiled with a modified Jemalloc " + "like the one shipped by default with the Redis source " + "distribution"; + return 0; + } +#else + UNUSED(val); + UNUSED(err); +#endif + return 1; +} + +static int isValidDBfilename(char *val, const char **err) { + if (!pathIsBaseName(val)) { + *err = "dbfilename can't be a path, just a filename"; + return 0; + } + return 1; +} + +static int isValidAOFfilename(char *val, const char **err) { + if (!pathIsBaseName(val)) { + *err = "appendfilename can't be a path, just a filename"; + return 0; + } + return 1; +} + +/* Validate specified string is a valid proc-title-template */ +static int isValidProcTitleTemplate(char *val, const char **err) { + if (!validateProcTitleTemplate(val)) { + *err = "template format is invalid or contains unknown variables"; + return 0; + } + return 1; +} + +static int updateProcTitleTemplate(char *val, char *prev, const char **err) { + UNUSED(val); + UNUSED(prev); + if (redisSetProcTitle(NULL) == C_ERR) { + *err = "failed to set process title"; + return 0; + } + return 1; +} + +static int updateHZ(long long val, long long prev, const char **err) { + UNUSED(prev); + UNUSED(err); + /* Hz is more a hint from the user, so we accept values out of range + * but cap them to reasonable values. */ + server.config_hz = val; + if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; + if (server.config_hz > CONFIG_MAX_HZ) server.config_hz = CONFIG_MAX_HZ; + server.hz = server.config_hz; + return 1; +} + +static int updatePort(long long val, long long prev, const char **err) { + /* Do nothing if port is unchanged */ + if (val == prev) { + return 1; + } + + if (changeListenPort(val, &server.ipfd, acceptTcpHandler) == C_ERR) { + *err = "Unable to listen on this port. Check server logs."; + return 0; + } + + return 1; +} + +static int updateJemallocBgThread(int val, int prev, const char **err) { + UNUSED(prev); + UNUSED(err); + set_jemalloc_bg_thread(val); + return 1; +} + +static int updateReplBacklogSize(long long val, long long prev, const char **err) { + /* resizeReplicationBacklog sets server.repl_backlog_size, and relies on + * being able to tell when the size changes, so restore prev before calling it. */ + UNUSED(err); + server.repl_backlog_size = prev; + resizeReplicationBacklog(val); + return 1; +} + +static int updateMaxmemory(long long val, long long prev, const char **err) { + UNUSED(prev); + UNUSED(err); + if (val) { + size_t used = zmalloc_used_memory()-freeMemoryGetNotCountedMemory(); + if ((unsigned long long)val < used) { + serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET (%llu) is smaller than the current memory usage (%zu). This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy.", server.maxmemory, used); + } + performEvictions(); + } + return 1; +} + +static int updateGoodSlaves(long long val, long long prev, const char **err) { + UNUSED(val); + UNUSED(prev); + UNUSED(err); + refreshGoodSlavesCount(); + return 1; +} + +static int updateAppendonly(int val, int prev, const char **err) { + UNUSED(prev); + if (val == 0 && server.aof_state != AOF_OFF) { + stopAppendOnly(); + } else if (val && server.aof_state == AOF_OFF) { + if (startAppendOnly() == C_ERR) { + *err = "Unable to turn on AOF. Check server logs."; + return 0; + } + } + return 1; +} + +static int updateSighandlerEnabled(int val, int prev, const char **err) { + UNUSED(err); + UNUSED(prev); + if (val) + setupSignalHandlers(); + else + removeSignalHandlers(); + return 1; +} + +static int updateMaxclients(long long val, long long prev, const char **err) { + /* Try to check if the OS is capable of supporting so many FDs. */ + if (val > prev) { + adjustOpenFilesLimit(); + if (server.maxclients != val) { + static char msg[128]; + sprintf(msg, "The operating system is not able to handle the specified number of clients, try with %d", server.maxclients); + *err = msg; + if (server.maxclients > prev) { + server.maxclients = prev; + adjustOpenFilesLimit(); + } + return 0; + } + if ((unsigned int) aeGetSetSize(server.el) < + server.maxclients + CONFIG_FDSET_INCR) + { + if (aeResizeSetSize(server.el, + server.maxclients + CONFIG_FDSET_INCR) == AE_ERR) + { + *err = "The event loop API used by Redis is not able to handle the specified number of clients"; + return 0; + } + } + } + return 1; +} + +static int updateOOMScoreAdj(int val, int prev, const char **err) { + UNUSED(prev); + + if (val) { + if (setOOMScoreAdj(-1) == C_ERR) { + *err = "Failed to set current oom_score_adj. Check server logs."; + return 0; + } + } + + return 1; +} + + +int updateRequirePass(sds val, sds prev, const char **err) { + UNUSED(prev); + UNUSED(err); + /* The old "requirepass" directive just translates to setting + * a password to the default user. The only thing we do + * additionally is to remember the cleartext password in this + * case, for backward compatibility with Redis <= 5. */ + ACLUpdateDefaultUserPassword(val); + return 1; +} + +#ifdef USE_OPENSSL +static int updateTlsCfg(char *val, char *prev, const char **err) { + UNUSED(val); + UNUSED(prev); + UNUSED(err); + + /* If TLS is enabled, try to configure OpenSSL. */ + if ((server.tls_port || server.tls_replication || server.tls_cluster) + && tlsConfigure(&server.tls_ctx_config) == C_ERR) { + *err = "Unable to update TLS configuration. Check server logs."; + return 0; + } + return 1; +} +static int updateTlsCfgBool(int val, int prev, const char **err) { + UNUSED(val); + UNUSED(prev); + return updateTlsCfg(NULL, NULL, err); +} + +static int updateTlsCfgInt(long long val, long long prev, const char **err) { + UNUSED(val); + UNUSED(prev); + return updateTlsCfg(NULL, NULL, err); +} + +static int updateTLSPort(long long val, long long prev, const char **err) { + /* Do nothing if port is unchanged */ + if (val == prev) { + return 1; + } + + /* Configure TLS if tls is enabled */ + if (prev == 0 && tlsConfigure(&server.tls_ctx_config) == C_ERR) { + *err = "Unable to update TLS configuration. Check server logs."; + return 0; + } + + if (changeListenPort(val, &server.tlsfd, acceptTLSHandler) == C_ERR) { + *err = "Unable to listen on this port. Check server logs."; + return 0; + } + + return 1; +} + +#endif /* USE_OPENSSL */ + +standardConfig configs[] = { + /* Bool configs */ + createBoolConfig("rdbchecksum", NULL, IMMUTABLE_CONFIG, server.rdb_checksum, 1, NULL, NULL), + createBoolConfig("daemonize", NULL, IMMUTABLE_CONFIG, server.daemonize, 0, NULL, NULL), + createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, server.io_threads_do_reads, 0,NULL, NULL), /* Read + parse from threads? */ + createBoolConfig("lua-replicate-commands", NULL, MODIFIABLE_CONFIG, server.lua_always_replicate_commands, 1, NULL, NULL), + createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, 0, NULL, NULL), + createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, 1, NULL, NULL), + createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, 1, NULL, NULL), + createBoolConfig("rdb-del-sync-files", NULL, MODIFIABLE_CONFIG, server.rdb_del_sync_files, 0, NULL, NULL), + createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, 1, NULL, NULL), + createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, 1, NULL, NULL), + createBoolConfig("set-proc-title", NULL, IMMUTABLE_CONFIG, server.set_proc_title, 1, NULL, NULL), /* Should setproctitle be used? */ + createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, server.dynamic_hz, 1, NULL, NULL), /* Adapt hz to # of clients.*/ + createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, 0, NULL, NULL), + createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, 0, NULL, NULL), + createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_server_del, 0, NULL, NULL), + createBoolConfig("lazyfree-lazy-user-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_user_del , 0, NULL, NULL), + createBoolConfig("lazyfree-lazy-user-flush", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_user_flush , 0, NULL, NULL), + createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, server.repl_disable_tcp_nodelay, 0, NULL, NULL), + createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, server.repl_diskless_sync, 0, NULL, NULL), + createBoolConfig("gopher-enabled", NULL, MODIFIABLE_CONFIG, server.gopher_enabled, 0, NULL, NULL), + createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.aof_rewrite_incremental_fsync, 1, NULL, NULL), + createBoolConfig("no-appendfsync-on-rewrite", NULL, MODIFIABLE_CONFIG, server.aof_no_fsync_on_rewrite, 0, NULL, NULL), + createBoolConfig("cluster-require-full-coverage", NULL, MODIFIABLE_CONFIG, server.cluster_require_full_coverage, 1, NULL, NULL), + createBoolConfig("rdb-save-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.rdb_save_incremental_fsync, 1, NULL, NULL), + createBoolConfig("aof-load-truncated", NULL, MODIFIABLE_CONFIG, server.aof_load_truncated, 1, NULL, NULL), + createBoolConfig("aof-use-rdb-preamble", NULL, MODIFIABLE_CONFIG, server.aof_use_rdb_preamble, 1, NULL, NULL), + createBoolConfig("cluster-replica-no-failover", "cluster-slave-no-failover", MODIFIABLE_CONFIG, server.cluster_slave_no_failover, 0, NULL, NULL), /* Failover by default. */ + createBoolConfig("replica-lazy-flush", "slave-lazy-flush", MODIFIABLE_CONFIG, server.repl_slave_lazy_flush, 0, NULL, NULL), + createBoolConfig("replica-serve-stale-data", "slave-serve-stale-data", MODIFIABLE_CONFIG, server.repl_serve_stale_data, 1, NULL, NULL), + createBoolConfig("replica-read-only", "slave-read-only", MODIFIABLE_CONFIG, server.repl_slave_ro, 1, NULL, NULL), + createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, server.repl_slave_ignore_maxmemory, 1, NULL, NULL), + createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, server.jemalloc_bg_thread, 1, NULL, updateJemallocBgThread), + createBoolConfig("activedefrag", NULL, MODIFIABLE_CONFIG, server.active_defrag_enabled, 0, isValidActiveDefrag, NULL), + createBoolConfig("syslog-enabled", NULL, IMMUTABLE_CONFIG, server.syslog_enabled, 0, NULL, NULL), + createBoolConfig("cluster-enabled", NULL, IMMUTABLE_CONFIG, server.cluster_enabled, 0, NULL, NULL), + createBoolConfig("appendonly", NULL, MODIFIABLE_CONFIG, server.aof_enabled, 0, NULL, updateAppendonly), + createBoolConfig("cluster-allow-reads-when-down", NULL, MODIFIABLE_CONFIG, server.cluster_allow_reads_when_down, 0, NULL, NULL), + createBoolConfig("crash-log-enabled", NULL, MODIFIABLE_CONFIG, server.crashlog_enabled, 1, NULL, updateSighandlerEnabled), + createBoolConfig("crash-memcheck-enabled", NULL, MODIFIABLE_CONFIG, server.memcheck_enabled, 1, NULL, NULL), + createBoolConfig("use-exit-on-panic", NULL, MODIFIABLE_CONFIG, server.use_exit_on_panic, 0, NULL, NULL), + createBoolConfig("disable-thp", NULL, MODIFIABLE_CONFIG, server.disable_thp, 1, NULL, NULL), + createBoolConfig("cluster-allow-replica-migration", NULL, MODIFIABLE_CONFIG, server.cluster_allow_replica_migration, 1, NULL, NULL), + createBoolConfig("replica-announced", NULL, MODIFIABLE_CONFIG, server.replica_announced, 1, NULL, NULL), + + /* String Configs */ + createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, "", NULL, NULL), + createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.unixsocket, NULL, NULL, NULL), + createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.pidfile, NULL, NULL, NULL), + createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.slave_announce_ip, NULL, NULL, NULL), + createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, server.masteruser, NULL, NULL, NULL), + createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL, NULL, NULL), + createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.syslog_ident, "redis", NULL, NULL), + createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.rdb_filename, "dump.rdb", isValidDBfilename, NULL), + createStringConfig("appendfilename", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.aof_filename, "appendonly.aof", isValidAOFfilename, NULL), + createStringConfig("server_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.server_cpulist, NULL, NULL, NULL), + createStringConfig("bio_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bio_cpulist, NULL, NULL, NULL), + createStringConfig("aof_rewrite_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.aof_rewrite_cpulist, NULL, NULL, NULL), + createStringConfig("bgsave_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.bgsave_cpulist, NULL, NULL, NULL), + createStringConfig("ignore-warnings", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.ignore_warnings, "", NULL, NULL), + createStringConfig("proc-title-template", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.proc_title_template, CONFIG_DEFAULT_PROC_TITLE_TEMPLATE, isValidProcTitleTemplate, updateProcTitleTemplate), + + /* SDS Configs */ + createSDSConfig("masterauth", NULL, MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL, NULL, NULL), + createSDSConfig("requirepass", NULL, MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, server.requirepass, NULL, NULL, updateRequirePass), + + /* Enum Configs */ + createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE, NULL, NULL), + createEnumConfig("syslog-facility", NULL, IMMUTABLE_CONFIG, syslog_facility_enum, server.syslog_facility, LOG_LOCAL0, NULL, NULL), + createEnumConfig("repl-diskless-load", NULL, MODIFIABLE_CONFIG, repl_diskless_load_enum, server.repl_diskless_load, REPL_DISKLESS_LOAD_DISABLED, NULL, NULL), + createEnumConfig("loglevel", NULL, MODIFIABLE_CONFIG, loglevel_enum, server.verbosity, LL_NOTICE, NULL, NULL), + createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, server.maxmemory_policy, MAXMEMORY_NO_EVICTION, NULL, NULL), + createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, AOF_FSYNC_EVERYSEC, NULL, NULL), + createEnumConfig("oom-score-adj", NULL, MODIFIABLE_CONFIG, oom_score_adj_enum, server.oom_score_adj, OOM_SCORE_ADJ_NO, NULL, updateOOMScoreAdj), + createEnumConfig("acl-pubsub-default", NULL, MODIFIABLE_CONFIG, acl_pubsub_default_enum, server.acl_pubsub_default, USER_FLAG_ALLCHANNELS, NULL, NULL), + createEnumConfig("sanitize-dump-payload", NULL, MODIFIABLE_CONFIG, sanitize_dump_payload_enum, server.sanitize_dump_payload, SANITIZE_DUMP_NO, NULL, NULL), + + /* Integer configs */ + createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, 16, INTEGER_CONFIG, NULL, NULL), + createIntConfig("port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.port, 6379, INTEGER_CONFIG, NULL, updatePort), /* TCP port. */ + createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 128, server.io_threads_num, 1, INTEGER_CONFIG, NULL, NULL), /* Single threaded by default */ + createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.aof_rewrite_perc, 100, INTEGER_CONFIG, NULL, NULL), + createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_slave_validity_factor, 10, INTEGER_CONFIG, NULL, NULL), /* Slave max data age factor. */ + createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.list_max_ziplist_size, -2, INTEGER_CONFIG, NULL, NULL), + createIntConfig("tcp-keepalive", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcpkeepalive, 300, INTEGER_CONFIG, NULL, NULL), + createIntConfig("cluster-migration-barrier", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_migration_barrier, 1, INTEGER_CONFIG, NULL, NULL), + createIntConfig("active-defrag-cycle-min", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_min, 1, INTEGER_CONFIG, NULL, NULL), /* Default: 1% CPU min (at lower threshold) */ + createIntConfig("active-defrag-cycle-max", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_max, 25, INTEGER_CONFIG, NULL, NULL), /* Default: 25% CPU max (at upper threshold) */ + createIntConfig("active-defrag-threshold-lower", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_lower, 10, INTEGER_CONFIG, NULL, NULL), /* Default: don't defrag when fragmentation is below 10% */ + createIntConfig("active-defrag-threshold-upper", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_upper, 100, INTEGER_CONFIG, NULL, NULL), /* Default: maximum defrag force at 100% fragmentation */ + createIntConfig("lfu-log-factor", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_log_factor, 10, INTEGER_CONFIG, NULL, NULL), + createIntConfig("lfu-decay-time", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_decay_time, 1, INTEGER_CONFIG, NULL, NULL), + createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, server.slave_priority, 100, INTEGER_CONFIG, NULL, NULL), + createIntConfig("repl-diskless-sync-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_diskless_sync_delay, 5, INTEGER_CONFIG, NULL, NULL), + createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.maxmemory_samples, 5, INTEGER_CONFIG, NULL, NULL), + createIntConfig("maxmemory-eviction-tenacity", NULL, MODIFIABLE_CONFIG, 0, 100, server.maxmemory_eviction_tenacity, 10, INTEGER_CONFIG, NULL, NULL), + createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.maxidletime, 0, INTEGER_CONFIG, NULL, NULL), /* Default client timeout: infinite */ + createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, server.slave_announce_port, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("tcp-backlog", NULL, IMMUTABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, 511, INTEGER_CONFIG, NULL, NULL), /* TCP listen backlog. */ + createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_bus_port, 0, INTEGER_CONFIG, NULL, NULL), /* Default: Use +10000 offset. */ + createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_port, 0, INTEGER_CONFIG, NULL, NULL), /* Use server.port */ + createIntConfig("cluster-announce-tls-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_tls_port, 0, INTEGER_CONFIG, NULL, NULL), /* Use server.tls_port */ + createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_timeout, 60, INTEGER_CONFIG, NULL, NULL), + createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_ping_slave_period, 10, INTEGER_CONFIG, NULL, NULL), + createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.rdb_key_save_delay, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.key_load_delay, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, 1, INTEGER_CONFIG, NULL, NULL), /* From 1 to 10. */ + createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG, NULL, updateHZ), + createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves), + createIntConfig("min-replicas-max-lag", "min-slaves-max-lag", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_max_lag, 10, INTEGER_CONFIG, NULL, updateGoodSlaves), + + /* Unsigned int configs */ + createUIntConfig("maxclients", NULL, MODIFIABLE_CONFIG, 1, UINT_MAX, server.maxclients, 10000, INTEGER_CONFIG, NULL, updateMaxclients), + + /* Unsigned Long configs */ + createULongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, 1000, INTEGER_CONFIG, NULL, NULL), /* Default: keys with more than 1000 fields will be processed separately */ + createULongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, 128, INTEGER_CONFIG, NULL, NULL), + createULongConfig("acllog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.acllog_max_len, 128, INTEGER_CONFIG, NULL, NULL), + + /* Long Long configs */ + createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, 5000, INTEGER_CONFIG, NULL, NULL),/* milliseconds */ + createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, 15000, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, 10000, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, 0, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 1024*1024, LONG_MAX, server.proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */ + createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.stream_node_max_entries, 100, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */ + + /* Unsigned Long Long configs */ + createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.maxmemory, 0, MEMORY_CONFIG, NULL, updateMaxmemory), + + /* Size_t configs */ + createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_entries, 512, INTEGER_CONFIG, NULL, NULL), + createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, 512, INTEGER_CONFIG, NULL, NULL), + createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_entries, 128, INTEGER_CONFIG, NULL, NULL), + createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.active_defrag_ignore_bytes, 100<<20, MEMORY_CONFIG, NULL, NULL), /* Default: don't defrag if frag overhead is below 100mb */ + createSizeTConfig("hash-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, 4096, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, 3000, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("tracking-table-max-keys", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.tracking_table_max_keys, 1000000, INTEGER_CONFIG, NULL, NULL), /* Default: 1 million keys max. */ + createSizeTConfig("client-query-buffer-limit", NULL, MODIFIABLE_CONFIG, 1024*1024, LONG_MAX, server.client_max_querybuf_len, 1024*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Default: 1GB max query buffer. */ + + /* Other configs */ + createTimeTConfig("repl-backlog-ttl", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.repl_backlog_time_limit, 60*60, INTEGER_CONFIG, NULL, NULL), /* Default: 1 hour */ + createOffTConfig("auto-aof-rewrite-min-size", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.aof_rewrite_min_size, 64*1024*1024, MEMORY_CONFIG, NULL, NULL), + +#ifdef USE_OPENSSL + createIntConfig("tls-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.tls_port, 0, INTEGER_CONFIG, NULL, updateTLSPort), /* TCP port. */ + createIntConfig("tls-session-cache-size", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tls_ctx_config.session_cache_size, 20*1024, INTEGER_CONFIG, NULL, updateTlsCfgInt), + createIntConfig("tls-session-cache-timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tls_ctx_config.session_cache_timeout, 300, INTEGER_CONFIG, NULL, updateTlsCfgInt), + createBoolConfig("tls-cluster", NULL, MODIFIABLE_CONFIG, server.tls_cluster, 0, NULL, updateTlsCfgBool), + createBoolConfig("tls-replication", NULL, MODIFIABLE_CONFIG, server.tls_replication, 0, NULL, updateTlsCfgBool), + createEnumConfig("tls-auth-clients", NULL, MODIFIABLE_CONFIG, tls_auth_clients_enum, server.tls_auth_clients, TLS_CLIENT_AUTH_YES, NULL, NULL), + createBoolConfig("tls-prefer-server-ciphers", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.prefer_server_ciphers, 0, NULL, updateTlsCfgBool), + createBoolConfig("tls-session-caching", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.session_caching, 1, NULL, updateTlsCfgBool), + createStringConfig("tls-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.cert_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.key_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-key-file-pass", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.key_file_pass, NULL, NULL, updateTlsCfg), + createStringConfig("tls-client-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.client_cert_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-client-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.client_key_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-client-key-file-pass", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.client_key_file_pass, NULL, NULL, updateTlsCfg), + createStringConfig("tls-dh-params-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.dh_params_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ca-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ca_cert_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ca-cert-dir", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ca_cert_dir, NULL, NULL, updateTlsCfg), + createStringConfig("tls-protocols", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.protocols, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ciphers", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ciphers, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ciphersuites", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ciphersuites, NULL, NULL, updateTlsCfg), +#endif + + /* NULL Terminator */ + {NULL} +}; + /*----------------------------------------------------------------------------- * CONFIG command entry point *----------------------------------------------------------------------------*/ @@ -2250,12 +2584,17 @@ void configCommand(client *c) { if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { const char *help[] = { -"GET -- Return parameters matching the glob-like and their values.", -"SET -- Set parameter to value.", -"RESETSTAT -- Reset statistics reported by INFO.", -"REWRITE -- Rewrite the configuration file.", +"GET ", +" Return parameters matching the glob-like and their values.", +"SET ", +" Set the configuration to .", +"RESETSTAT", +" Reset statistics reported by the INFO command.", +"REWRITE", +" Rewrite the configuration file.", NULL }; + addReplyHelp(c, help); } else if (!strcasecmp(c->argv[1]->ptr,"set") && c->argc == 4) { configSetCommand(c); @@ -2264,13 +2603,14 @@ NULL } else if (!strcasecmp(c->argv[1]->ptr,"resetstat") && c->argc == 2) { resetServerStats(); resetCommandTableStats(); + resetErrorTableStats(); addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"rewrite") && c->argc == 2) { if (server.configfile == NULL) { addReplyError(c,"The server is running without a config file"); return; } - if (rewriteConfig(server.configfile) == -1) { + if (rewriteConfig(server.configfile, 0) == -1) { serverLog(LL_WARNING,"CONFIG REWRITE failed: %s", strerror(errno)); addReplyErrorFormat(c,"Rewriting config file: %s", strerror(errno)); } else { diff --git a/redis.submodule/src/config.h b/redis.submodule/src/config.h index efa9d11..2f351a1 100644 --- a/redis.submodule/src/config.h +++ b/redis.submodule/src/config.h @@ -35,8 +35,8 @@ #endif #ifdef __linux__ -#include #include +#include #endif /* Define redis_fstat to fstat or fstat64() */ @@ -54,6 +54,7 @@ #define HAVE_PROC_MAPS 1 #define HAVE_PROC_SMAPS 1 #define HAVE_PROC_SOMAXCONN 1 +#define HAVE_PROC_OOM_SCORE_ADJ 1 #endif /* Test for task_info() */ @@ -63,8 +64,8 @@ /* Test for backtrace() */ #if defined(__APPLE__) || (defined(__linux__) && defined(__GLIBC__)) || \ - defined(__FreeBSD__) || (defined(__OpenBSD__) && defined(USE_BACKTRACE))\ - || defined(__DragonFly__) + defined(__FreeBSD__) || ((defined(__OpenBSD__) || defined(__NetBSD__)) && defined(USE_BACKTRACE))\ + || defined(__DragonFly__) || (defined(__UCLIBC__) && defined(__UCLIBC_HAS_BACKTRACE__)) #define HAVE_BACKTRACE 1 #endif @@ -86,6 +87,7 @@ #include #ifdef _DTRACE_VERSION #define HAVE_EVPORT 1 +#define HAVE_PSINFO 1 #endif #endif @@ -96,21 +98,23 @@ #define redis_fsync fsync #endif -/* Define rdb_fsync_range to sync_file_range() on Linux, otherwise we use - * the plain fsync() call. */ -#ifdef __linux__ -#if defined(__GLIBC__) && defined(__GLIBC_PREREQ) -#if (LINUX_VERSION_CODE >= 0x020611 && __GLIBC_PREREQ(2, 6)) -#define HAVE_SYNC_FILE_RANGE 1 -#endif +#if __GNUC__ >= 4 +#define redis_unreachable __builtin_unreachable #else -#if (LINUX_VERSION_CODE >= 0x020611) -#define HAVE_SYNC_FILE_RANGE 1 -#endif +#define redis_unreachable abort #endif + +#if __GNUC__ >= 3 +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else +#define likely(x) (x) +#define unlikely(x) (x) #endif -#ifdef HAVE_SYNC_FILE_RANGE +/* Define rdb_fsync_range to sync_file_range() on Linux, otherwise we use + * the plain fsync() call. */ +#if (defined(__linux__) && defined(SYNC_FILE_RANGE_WAIT_BEFORE)) #define rdb_fsync_range(fd,off,size) sync_file_range(fd,off,size,SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE) #else #define rdb_fsync_range(fd,off,size) fsync(fd) @@ -123,7 +127,11 @@ #define USE_SETPROCTITLE #endif -#if ((defined __linux && defined(__GLIBC__)) || defined __APPLE__) +#if defined(__HAIKU__) +#define ESOCKTNOSUPPORT 0 +#endif + +#if (defined __linux || defined __APPLE__) #define USE_SETPROCTITLE #define INIT_SETPROCTITLE_REPLACEMENT void spt_init(int argc, char *argv[]); @@ -165,7 +173,7 @@ void setproctitle(const char *fmt, ...); #endif /* BYTE_ORDER */ /* Sometimes after including an OS-specific header that defines the - * endianess we end with __BYTE_ORDER but not with BYTE_ORDER that is what + * endianness we end with __BYTE_ORDER but not with BYTE_ORDER that is what * the Redis code uses. In this case let's define everything without the * underscores. */ #ifndef BYTE_ORDER @@ -226,4 +234,34 @@ void setproctitle(const char *fmt, ...); #define USE_ALIGNED_ACCESS #endif +/* Define for redis_set_thread_title */ +#ifdef __linux__ +#define redis_set_thread_title(name) pthread_setname_np(pthread_self(), name) +#else +#if (defined __FreeBSD__ || defined __OpenBSD__) +#include +#define redis_set_thread_title(name) pthread_set_name_np(pthread_self(), name) +#elif defined __NetBSD__ +#include +#define redis_set_thread_title(name) pthread_setname_np(pthread_self(), "%s", name) +#elif defined __HAIKU__ +#include +#define redis_set_thread_title(name) rename_thread(find_thread(0), name) +#else +#if (defined __APPLE__ && defined(MAC_OS_X_VERSION_10_7)) +int pthread_setname_np(const char *name); +#include +#define redis_set_thread_title(name) pthread_setname_np(name) +#else +#define redis_set_thread_title(name) +#endif +#endif +#endif + +/* Check if we can use setcpuaffinity(). */ +#if (defined __linux || defined __NetBSD__ || defined __FreeBSD__ || defined __DragonFly__) +#define USE_SETCPUAFFINITY +void setcpuaffinity(const char *cpulist); +#endif + #endif diff --git a/redis.submodule/src/connection.c b/redis.submodule/src/connection.c new file mode 100644 index 0000000..a594632 --- /dev/null +++ b/redis.submodule/src/connection.c @@ -0,0 +1,433 @@ +/* + * Copyright (c) 2019, Redis Labs + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "server.h" +#include "connhelpers.h" + +/* The connections module provides a lean abstraction of network connections + * to avoid direct socket and async event management across the Redis code base. + * + * It does NOT provide advanced connection features commonly found in similar + * libraries such as complete in/out buffer management, throttling, etc. These + * functions remain in networking.c. + * + * The primary goal is to allow transparent handling of TCP and TLS based + * connections. To do so, connections have the following properties: + * + * 1. A connection may live before its corresponding socket exists. This + * allows various context and configuration setting to be handled before + * establishing the actual connection. + * 2. The caller may register/unregister logical read/write handlers to be + * called when the connection has data to read from/can accept writes. + * These logical handlers may or may not correspond to actual AE events, + * depending on the implementation (for TCP they are; for TLS they aren't). + */ + +ConnectionType CT_Socket; + +/* When a connection is created we must know its type already, but the + * underlying socket may or may not exist: + * + * - For accepted connections, it exists as we do not model the listen/accept + * part; So caller calls connCreateSocket() followed by connAccept(). + * - For outgoing connections, the socket is created by the connection module + * itself; So caller calls connCreateSocket() followed by connConnect(), + * which registers a connect callback that fires on connected/error state + * (and after any transport level handshake was done). + * + * NOTE: An earlier version relied on connections being part of other structs + * and not independently allocated. This could lead to further optimizations + * like using container_of(), etc. However it was discontinued in favor of + * this approach for these reasons: + * + * 1. In some cases conns are created/handled outside the context of the + * containing struct, in which case it gets a bit awkward to copy them. + * 2. Future implementations may wish to allocate arbitrary data for the + * connection. + * 3. The container_of() approach is anyway risky because connections may + * be embedded in different structs, not just client. + */ + +connection *connCreateSocket() { + connection *conn = zcalloc(sizeof(connection)); + conn->type = &CT_Socket; + conn->fd = -1; + + return conn; +} + +/* Create a new socket-type connection that is already associated with + * an accepted connection. + * + * The socket is not ready for I/O until connAccept() was called and + * invoked the connection-level accept handler. + * + * Callers should use connGetState() and verify the created connection + * is not in an error state (which is not possible for a socket connection, + * but could but possible with other protocols). + */ +connection *connCreateAcceptedSocket(int fd) { + connection *conn = connCreateSocket(); + conn->fd = fd; + conn->state = CONN_STATE_ACCEPTING; + return conn; +} + +static int connSocketConnect(connection *conn, const char *addr, int port, const char *src_addr, + ConnectionCallbackFunc connect_handler) { + int fd = anetTcpNonBlockBestEffortBindConnect(NULL,addr,port,src_addr); + if (fd == -1) { + conn->state = CONN_STATE_ERROR; + conn->last_errno = errno; + return C_ERR; + } + + conn->fd = fd; + conn->state = CONN_STATE_CONNECTING; + + conn->conn_handler = connect_handler; + aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE, + conn->type->ae_handler, conn); + + return C_OK; +} + +/* Returns true if a write handler is registered */ +int connHasWriteHandler(connection *conn) { + return conn->write_handler != NULL; +} + +/* Returns true if a read handler is registered */ +int connHasReadHandler(connection *conn) { + return conn->read_handler != NULL; +} + +/* Associate a private data pointer with the connection */ +void connSetPrivateData(connection *conn, void *data) { + conn->private_data = data; +} + +/* Get the associated private data pointer */ +void *connGetPrivateData(connection *conn) { + return conn->private_data; +} + +/* ------ Pure socket connections ------- */ + +/* A very incomplete list of implementation-specific calls. Much of the above shall + * move here as we implement additional connection types. + */ + +/* Close the connection and free resources. */ +static void connSocketClose(connection *conn) { + if (conn->fd != -1) { + aeDeleteFileEvent(server.el,conn->fd, AE_READABLE | AE_WRITABLE); + close(conn->fd); + conn->fd = -1; + } + + /* If called from within a handler, schedule the close but + * keep the connection until the handler returns. + */ + if (connHasRefs(conn)) { + conn->flags |= CONN_FLAG_CLOSE_SCHEDULED; + return; + } + + zfree(conn); +} + +static int connSocketWrite(connection *conn, const void *data, size_t data_len) { + int ret = write(conn->fd, data, data_len); + if (ret < 0 && errno != EAGAIN) { + conn->last_errno = errno; + + /* Don't overwrite the state of a connection that is not already + * connected, not to mess with handler callbacks. + */ + if (conn->state == CONN_STATE_CONNECTED) + conn->state = CONN_STATE_ERROR; + } + + return ret; +} + +static int connSocketRead(connection *conn, void *buf, size_t buf_len) { + int ret = read(conn->fd, buf, buf_len); + if (!ret) { + conn->state = CONN_STATE_CLOSED; + } else if (ret < 0 && errno != EAGAIN) { + conn->last_errno = errno; + + /* Don't overwrite the state of a connection that is not already + * connected, not to mess with handler callbacks. + */ + if (conn->state == CONN_STATE_CONNECTED) + conn->state = CONN_STATE_ERROR; + } + + return ret; +} + +static int connSocketAccept(connection *conn, ConnectionCallbackFunc accept_handler) { + int ret = C_OK; + + if (conn->state != CONN_STATE_ACCEPTING) return C_ERR; + conn->state = CONN_STATE_CONNECTED; + + connIncrRefs(conn); + if (!callHandler(conn, accept_handler)) ret = C_ERR; + connDecrRefs(conn); + + return ret; +} + +/* Register a write handler, to be called when the connection is writable. + * If NULL, the existing handler is removed. + * + * The barrier flag indicates a write barrier is requested, resulting with + * CONN_FLAG_WRITE_BARRIER set. This will ensure that the write handler is + * always called before and not after the read handler in a single event + * loop. + */ +static int connSocketSetWriteHandler(connection *conn, ConnectionCallbackFunc func, int barrier) { + if (func == conn->write_handler) return C_OK; + + conn->write_handler = func; + if (barrier) + conn->flags |= CONN_FLAG_WRITE_BARRIER; + else + conn->flags &= ~CONN_FLAG_WRITE_BARRIER; + if (!conn->write_handler) + aeDeleteFileEvent(server.el,conn->fd,AE_WRITABLE); + else + if (aeCreateFileEvent(server.el,conn->fd,AE_WRITABLE, + conn->type->ae_handler,conn) == AE_ERR) return C_ERR; + return C_OK; +} + +/* Register a read handler, to be called when the connection is readable. + * If NULL, the existing handler is removed. + */ +static int connSocketSetReadHandler(connection *conn, ConnectionCallbackFunc func) { + if (func == conn->read_handler) return C_OK; + + conn->read_handler = func; + if (!conn->read_handler) + aeDeleteFileEvent(server.el,conn->fd,AE_READABLE); + else + if (aeCreateFileEvent(server.el,conn->fd, + AE_READABLE,conn->type->ae_handler,conn) == AE_ERR) return C_ERR; + return C_OK; +} + +static const char *connSocketGetLastError(connection *conn) { + return strerror(conn->last_errno); +} + +static void connSocketEventHandler(struct aeEventLoop *el, int fd, void *clientData, int mask) +{ + UNUSED(el); + UNUSED(fd); + connection *conn = clientData; + + if (conn->state == CONN_STATE_CONNECTING && + (mask & AE_WRITABLE) && conn->conn_handler) { + + int conn_error = connGetSocketError(conn); + if (conn_error) { + conn->last_errno = conn_error; + conn->state = CONN_STATE_ERROR; + } else { + conn->state = CONN_STATE_CONNECTED; + } + + if (!conn->write_handler) aeDeleteFileEvent(server.el,conn->fd,AE_WRITABLE); + + if (!callHandler(conn, conn->conn_handler)) return; + conn->conn_handler = NULL; + } + + /* Normally we execute the readable event first, and the writable + * event later. This is useful as sometimes we may be able + * to serve the reply of a query immediately after processing the + * query. + * + * However if WRITE_BARRIER is set in the mask, our application is + * asking us to do the reverse: never fire the writable event + * after the readable. In such a case, we invert the calls. + * This is useful when, for instance, we want to do things + * in the beforeSleep() hook, like fsync'ing a file to disk, + * before replying to a client. */ + int invert = conn->flags & CONN_FLAG_WRITE_BARRIER; + + int call_write = (mask & AE_WRITABLE) && conn->write_handler; + int call_read = (mask & AE_READABLE) && conn->read_handler; + + /* Handle normal I/O flows */ + if (!invert && call_read) { + if (!callHandler(conn, conn->read_handler)) return; + } + /* Fire the writable event. */ + if (call_write) { + if (!callHandler(conn, conn->write_handler)) return; + } + /* If we have to invert the call, fire the readable event now + * after the writable one. */ + if (invert && call_read) { + if (!callHandler(conn, conn->read_handler)) return; + } +} + +static int connSocketBlockingConnect(connection *conn, const char *addr, int port, long long timeout) { + int fd = anetTcpNonBlockConnect(NULL,addr,port); + if (fd == -1) { + conn->state = CONN_STATE_ERROR; + conn->last_errno = errno; + return C_ERR; + } + + if ((aeWait(fd, AE_WRITABLE, timeout) & AE_WRITABLE) == 0) { + conn->state = CONN_STATE_ERROR; + conn->last_errno = ETIMEDOUT; + } + + conn->fd = fd; + conn->state = CONN_STATE_CONNECTED; + return C_OK; +} + +/* Connection-based versions of syncio.c functions. + * NOTE: This should ideally be refactored out in favor of pure async work. + */ + +static ssize_t connSocketSyncWrite(connection *conn, char *ptr, ssize_t size, long long timeout) { + return syncWrite(conn->fd, ptr, size, timeout); +} + +static ssize_t connSocketSyncRead(connection *conn, char *ptr, ssize_t size, long long timeout) { + return syncRead(conn->fd, ptr, size, timeout); +} + +static ssize_t connSocketSyncReadLine(connection *conn, char *ptr, ssize_t size, long long timeout) { + return syncReadLine(conn->fd, ptr, size, timeout); +} + +static int connSocketGetType(connection *conn) { + (void) conn; + + return CONN_TYPE_SOCKET; +} + +ConnectionType CT_Socket = { + .ae_handler = connSocketEventHandler, + .close = connSocketClose, + .write = connSocketWrite, + .read = connSocketRead, + .accept = connSocketAccept, + .connect = connSocketConnect, + .set_write_handler = connSocketSetWriteHandler, + .set_read_handler = connSocketSetReadHandler, + .get_last_error = connSocketGetLastError, + .blocking_connect = connSocketBlockingConnect, + .sync_write = connSocketSyncWrite, + .sync_read = connSocketSyncRead, + .sync_readline = connSocketSyncReadLine, + .get_type = connSocketGetType +}; + + +int connGetSocketError(connection *conn) { + int sockerr = 0; + socklen_t errlen = sizeof(sockerr); + + if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, &sockerr, &errlen) == -1) + sockerr = errno; + return sockerr; +} + +int connPeerToString(connection *conn, char *ip, size_t ip_len, int *port) { + return anetFdToString(conn ? conn->fd : -1, ip, ip_len, port, FD_TO_PEER_NAME); +} + +int connSockName(connection *conn, char *ip, size_t ip_len, int *port) { + return anetFdToString(conn->fd, ip, ip_len, port, FD_TO_SOCK_NAME); +} + +int connFormatFdAddr(connection *conn, char *buf, size_t buf_len, int fd_to_str_type) { + return anetFormatFdAddr(conn ? conn->fd : -1, buf, buf_len, fd_to_str_type); +} + +int connBlock(connection *conn) { + if (conn->fd == -1) return C_ERR; + return anetBlock(NULL, conn->fd); +} + +int connNonBlock(connection *conn) { + if (conn->fd == -1) return C_ERR; + return anetNonBlock(NULL, conn->fd); +} + +int connEnableTcpNoDelay(connection *conn) { + if (conn->fd == -1) return C_ERR; + return anetEnableTcpNoDelay(NULL, conn->fd); +} + +int connDisableTcpNoDelay(connection *conn) { + if (conn->fd == -1) return C_ERR; + return anetDisableTcpNoDelay(NULL, conn->fd); +} + +int connKeepAlive(connection *conn, int interval) { + if (conn->fd == -1) return C_ERR; + return anetKeepAlive(NULL, conn->fd, interval); +} + +int connSendTimeout(connection *conn, long long ms) { + return anetSendTimeout(NULL, conn->fd, ms); +} + +int connRecvTimeout(connection *conn, long long ms) { + return anetRecvTimeout(NULL, conn->fd, ms); +} + +int connGetState(connection *conn) { + return conn->state; +} + +/* Return a text that describes the connection, suitable for inclusion + * in CLIENT LIST and similar outputs. + * + * For sockets, we always return "fd=" to maintain compatibility. + */ +const char *connGetInfo(connection *conn, char *buf, size_t buf_len) { + snprintf(buf, buf_len-1, "fd=%i", conn == NULL ? -1 : conn->fd); + return buf; +} + diff --git a/redis.submodule/src/connection.h b/redis.submodule/src/connection.h new file mode 100644 index 0000000..60f1f70 --- /dev/null +++ b/redis.submodule/src/connection.h @@ -0,0 +1,237 @@ + +/* + * Copyright (c) 2019, Redis Labs + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __REDIS_CONNECTION_H +#define __REDIS_CONNECTION_H + +#define CONN_INFO_LEN 32 + +struct aeEventLoop; +typedef struct connection connection; + +typedef enum { + CONN_STATE_NONE = 0, + CONN_STATE_CONNECTING, + CONN_STATE_ACCEPTING, + CONN_STATE_CONNECTED, + CONN_STATE_CLOSED, + CONN_STATE_ERROR +} ConnectionState; + +#define CONN_FLAG_CLOSE_SCHEDULED (1<<0) /* Closed scheduled by a handler */ +#define CONN_FLAG_WRITE_BARRIER (1<<1) /* Write barrier requested */ + +#define CONN_TYPE_SOCKET 1 +#define CONN_TYPE_TLS 2 + +typedef void (*ConnectionCallbackFunc)(struct connection *conn); + +typedef struct ConnectionType { + void (*ae_handler)(struct aeEventLoop *el, int fd, void *clientData, int mask); + int (*connect)(struct connection *conn, const char *addr, int port, const char *source_addr, ConnectionCallbackFunc connect_handler); + int (*write)(struct connection *conn, const void *data, size_t data_len); + int (*read)(struct connection *conn, void *buf, size_t buf_len); + void (*close)(struct connection *conn); + int (*accept)(struct connection *conn, ConnectionCallbackFunc accept_handler); + int (*set_write_handler)(struct connection *conn, ConnectionCallbackFunc handler, int barrier); + int (*set_read_handler)(struct connection *conn, ConnectionCallbackFunc handler); + const char *(*get_last_error)(struct connection *conn); + int (*blocking_connect)(struct connection *conn, const char *addr, int port, long long timeout); + ssize_t (*sync_write)(struct connection *conn, char *ptr, ssize_t size, long long timeout); + ssize_t (*sync_read)(struct connection *conn, char *ptr, ssize_t size, long long timeout); + ssize_t (*sync_readline)(struct connection *conn, char *ptr, ssize_t size, long long timeout); + int (*get_type)(struct connection *conn); +} ConnectionType; + +struct connection { + ConnectionType *type; + ConnectionState state; + short int flags; + short int refs; + int last_errno; + void *private_data; + ConnectionCallbackFunc conn_handler; + ConnectionCallbackFunc write_handler; + ConnectionCallbackFunc read_handler; + int fd; +}; + +/* The connection module does not deal with listening and accepting sockets, + * so we assume we have a socket when an incoming connection is created. + * + * The fd supplied should therefore be associated with an already accept()ed + * socket. + * + * connAccept() may directly call accept_handler(), or return and call it + * at a later time. This behavior is a bit awkward but aims to reduce the need + * to wait for the next event loop, if no additional handshake is required. + * + * IMPORTANT: accept_handler may decide to close the connection, calling connClose(). + * To make this safe, the connection is only marked with CONN_FLAG_CLOSE_SCHEDULED + * in this case, and connAccept() returns with an error. + * + * connAccept() callers must always check the return value and on error (C_ERR) + * a connClose() must be called. + */ + +static inline int connAccept(connection *conn, ConnectionCallbackFunc accept_handler) { + return conn->type->accept(conn, accept_handler); +} + +/* Establish a connection. The connect_handler will be called when the connection + * is established, or if an error has occurred. + * + * The connection handler will be responsible to set up any read/write handlers + * as needed. + * + * If C_ERR is returned, the operation failed and the connection handler shall + * not be expected. + */ +static inline int connConnect(connection *conn, const char *addr, int port, const char *src_addr, + ConnectionCallbackFunc connect_handler) { + return conn->type->connect(conn, addr, port, src_addr, connect_handler); +} + +/* Blocking connect. + * + * NOTE: This is implemented in order to simplify the transition to the abstract + * connections, but should probably be refactored out of cluster.c and replication.c, + * in favor of a pure async implementation. + */ +static inline int connBlockingConnect(connection *conn, const char *addr, int port, long long timeout) { + return conn->type->blocking_connect(conn, addr, port, timeout); +} + +/* Write to connection, behaves the same as write(2). + * + * Like write(2), a short write is possible. A -1 return indicates an error. + * + * The caller should NOT rely on errno. Testing for an EAGAIN-like condition, use + * connGetState() to see if the connection state is still CONN_STATE_CONNECTED. + */ +static inline int connWrite(connection *conn, const void *data, size_t data_len) { + return conn->type->write(conn, data, data_len); +} + +/* Read from the connection, behaves the same as read(2). + * + * Like read(2), a short read is possible. A return value of 0 will indicate the + * connection was closed, and -1 will indicate an error. + * + * The caller should NOT rely on errno. Testing for an EAGAIN-like condition, use + * connGetState() to see if the connection state is still CONN_STATE_CONNECTED. + */ +static inline int connRead(connection *conn, void *buf, size_t buf_len) { + return conn->type->read(conn, buf, buf_len); +} + +/* Register a write handler, to be called when the connection is writable. + * If NULL, the existing handler is removed. + */ +static inline int connSetWriteHandler(connection *conn, ConnectionCallbackFunc func) { + return conn->type->set_write_handler(conn, func, 0); +} + +/* Register a read handler, to be called when the connection is readable. + * If NULL, the existing handler is removed. + */ +static inline int connSetReadHandler(connection *conn, ConnectionCallbackFunc func) { + return conn->type->set_read_handler(conn, func); +} + +/* Set a write handler, and possibly enable a write barrier, this flag is + * cleared when write handler is changed or removed. + * With barrier enabled, we never fire the event if the read handler already + * fired in the same event loop iteration. Useful when you want to persist + * things to disk before sending replies, and want to do that in a group fashion. */ +static inline int connSetWriteHandlerWithBarrier(connection *conn, ConnectionCallbackFunc func, int barrier) { + return conn->type->set_write_handler(conn, func, barrier); +} + +static inline void connClose(connection *conn) { + conn->type->close(conn); +} + +/* Returns the last error encountered by the connection, as a string. If no error, + * a NULL is returned. + */ +static inline const char *connGetLastError(connection *conn) { + return conn->type->get_last_error(conn); +} + +static inline ssize_t connSyncWrite(connection *conn, char *ptr, ssize_t size, long long timeout) { + return conn->type->sync_write(conn, ptr, size, timeout); +} + +static inline ssize_t connSyncRead(connection *conn, char *ptr, ssize_t size, long long timeout) { + return conn->type->sync_read(conn, ptr, size, timeout); +} + +static inline ssize_t connSyncReadLine(connection *conn, char *ptr, ssize_t size, long long timeout) { + return conn->type->sync_readline(conn, ptr, size, timeout); +} + +/* Return CONN_TYPE_* for the specified connection */ +static inline int connGetType(connection *conn) { + return conn->type->get_type(conn); +} + +connection *connCreateSocket(); +connection *connCreateAcceptedSocket(int fd); + +connection *connCreateTLS(); +connection *connCreateAcceptedTLS(int fd, int require_auth); + +void connSetPrivateData(connection *conn, void *data); +void *connGetPrivateData(connection *conn); +int connGetState(connection *conn); +int connHasWriteHandler(connection *conn); +int connHasReadHandler(connection *conn); +int connGetSocketError(connection *conn); + +/* anet-style wrappers to conns */ +int connBlock(connection *conn); +int connNonBlock(connection *conn); +int connEnableTcpNoDelay(connection *conn); +int connDisableTcpNoDelay(connection *conn); +int connKeepAlive(connection *conn, int interval); +int connSendTimeout(connection *conn, long long ms); +int connRecvTimeout(connection *conn, long long ms); +int connPeerToString(connection *conn, char *ip, size_t ip_len, int *port); +int connFormatFdAddr(connection *conn, char *buf, size_t buf_len, int fd_to_str_type); +int connSockName(connection *conn, char *ip, size_t ip_len, int *port); +const char *connGetInfo(connection *conn, char *buf, size_t buf_len); + +/* Helpers for tls special considerations */ +sds connTLSGetPeerCert(connection *conn); +int tlsHasPendingData(); +int tlsProcessPendingData(); + +#endif /* __REDIS_CONNECTION_H */ diff --git a/redis.submodule/src/connhelpers.h b/redis.submodule/src/connhelpers.h new file mode 100644 index 0000000..86250d0 --- /dev/null +++ b/redis.submodule/src/connhelpers.h @@ -0,0 +1,88 @@ + +/* + * Copyright (c) 2019, Redis Labs + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __REDIS_CONNHELPERS_H +#define __REDIS_CONNHELPERS_H + +#include "connection.h" + +/* These are helper functions that are common to different connection + * implementations (currently sockets in connection.c and TLS in tls.c). + * + * Currently helpers implement the mechanisms for invoking connection + * handlers and tracking connection references, to allow safe destruction + * of connections from within a handler. + */ + +/* Incremenet connection references. + * + * Inside a connection handler, we guarantee refs >= 1 so it is always + * safe to connClose(). + * + * In other cases where we don't want to prematurely lose the connection, + * it can go beyond 1 as well; currently it is only done by connAccept(). + */ +static inline void connIncrRefs(connection *conn) { + conn->refs++; +} + +/* Decrement connection references. + * + * Note that this is not intended to provide any automatic free logic! + * callHandler() takes care of that for the common flows, and anywhere an + * explicit connIncrRefs() is used, the caller is expected to take care of + * that. + */ + +static inline void connDecrRefs(connection *conn) { + conn->refs--; +} + +static inline int connHasRefs(connection *conn) { + return conn->refs; +} + +/* Helper for connection implementations to call handlers: + * 1. Increment refs to protect the connection. + * 2. Execute the handler (if set). + * 3. Decrement refs and perform deferred close, if refs==0. + */ +static inline int callHandler(connection *conn, ConnectionCallbackFunc handler) { + connIncrRefs(conn); + if (handler) handler(conn); + connDecrRefs(conn); + if (conn->flags & CONN_FLAG_CLOSE_SCHEDULED) { + if (!connHasRefs(conn)) connClose(conn); + return 0; + } + return 1; +} + +#endif /* __REDIS_CONNHELPERS_H */ diff --git a/redis.submodule/src/crc16_slottable.h b/redis.submodule/src/crc16_slottable.h new file mode 100644 index 0000000..652aea9 --- /dev/null +++ b/redis.submodule/src/crc16_slottable.h @@ -0,0 +1,835 @@ +#ifndef _CRC16_TABLE_H__ +#define _CRC16_TABLE_H__ + +/* A table of the shortest possible alphanumeric string that is mapped by redis' crc16 + * to any given redis cluster slot. + * + * The array indexes are slot numbers, so that given a desired slot, this string is guaranteed + * to make redis cluster route a request to the shard holding this slot + */ + +const char *crc16_slot_table[] = { +"06S", "Qi", "5L5", "4Iu", "4gY", "460", "1Y7", "1LV", "0QG", "ru", "7Ok", "4ji", "4DE", "65n", "2JH", "I8", "F9", "SX", "7nF", "4KD", +"4eh", "6PK", "2ke", "1Ng", "0Sv", "4L", "491", "4hX", "4Ft", "5C4", "2Hy", "09R", "021", "0cX", "4Xv", "6mU", "6Cy", "42R", "0Mt", "nF", +"cv", "1Pe", "5kK", "6NI", "74L", "4UF", "0nh", "MZ", "2TJ", "0ai", "4ZG", "6od", "6AH", "40c", "0OE", "lw", "aG", "0Bu", "5iz", "6Lx", +"5R7", "4Ww", "0lY", "Ok", "5n3", "4ks", "8YE", "7g", "2KR", "1nP", "714", "64t", "69D", "4Ho", "07I", "Ps", "2hN", "1ML", "4fC", "7CA", +"avs", "4iB", "0Rl", "5V", "2Ic", "08H", "4Gn", "66E", "aUo", "b4e", "05x", "RB", "8f", "8VD", "4dr", "5a2", "4zp", "6OS", "bl", "355", +"0or", "1j2", "75V", "bno", "4Yl", "6lO", "Ap", "0bB", "0Ln", "2yM", "6Bc", "43H", "4xA", "6Mb", "22D", "14", "0mC", "Nq", "6cN", "4Vm", +"ban", "aDl", "CA", "14Z", "8GG", "mm", "549", "41y", "53t", "464", "1Y3", "1LR", "06W", "Qm", "5L1", "4Iq", "4DA", "65j", "2JL", "1oN", +"0QC", "6y", "7Oo", "4jm", "4el", "6PO", "9x", "1Nc", "04f", "2EM", "7nB", "bqs", "4Fp", "5C0", "d6F", "09V", "0Sr", "4H", "495", "bRo", +"aio", "42V", "0Mp", "nB", "025", "17u", "4Xr", "6mQ", "74H", "4UB", "0nl", "3Kn", "cr", "1Pa", "5kO", "6NM", "6AL", "40g", "0OA", "ls", +"2TN", "0am", "4ZC", "aEr", "5R3", "4Ws", "18t", "Oo", "aC", "0Bq", "bCl", "afn", "2KV", "1nT", "5Uz", "64p", "5n7", "4kw", "0PY", "7c", +"2hJ", "1MH", "4fG", "6Sd", "7mi", "4Hk", "07M", "Pw", "2Ig", "08L", "4Gj", "66A", "7LD", "4iF", "0Rh", "5R", "8b", "1Oy", "4dv", "5a6", +"7oX", "4JZ", "0qt", "RF", "0ov", "LD", "4A9", "4TX", "4zt", "6OW", "bh", "0AZ", "z9", "oX", "6Bg", "43L", "4Yh", "6lK", "At", "0bF", +"0mG", "Nu", "6cJ", "4Vi", "4xE", "6Mf", "2vH", "10", "8GC", "mi", "5p5", "4uu", "5Kx", "4N8", "CE", "1pV", "0QO", "6u", "7Oc", "4ja", +"4DM", "65f", "3Za", "I0", "0rS", "Qa", "68V", "b7F", "4gQ", "468", "dSo", "285", "274", "4D", "499", "4hP", "b8G", "67W", "0h3", "09Z", +"F1", "SP", "7nN", "4KL", "51I", "6PC", "9t", "1No", "21g", "1Pm", "5kC", "6NA", "74D", "4UN", "X3", "MR", "029", "0cP", "bbM", "79t", +"4c3", "42Z", "8Dd", "nN", "aO", "8Ke", "4yS", "4l2", "76u", "635", "0lQ", "Oc", "BS", "W2", "4ZO", "6ol", "7Qa", "40k", "0OM", "2zn", +"69L", "4Hg", "07A", "2Fj", "2hF", "k6", "4fK", "6Sh", "7Ny", "6K9", "0PU", "7o", "2KZ", "1nX", "4EW", "4P6", "7oT", "4JV", "05p", "RJ", +"8n", "1Ou", "4dz", "6QY", "7LH", "4iJ", "d7", "qV", "2Ik", "1li", "4Gf", "66M", "4Yd", "6lG", "Ax", "0bJ", "z5", "oT", "6Bk", "4wH", +"4zx", "aeI", "bd", "0AV", "0oz", "LH", "4A5", "4TT", "5Kt", "4N4", "CI", "14R", "0NW", "me", "541", "41q", "4xI", "6Mj", "22L", "u4", +"0mK", "Ny", "6cF", "4Ve", "4DI", "65b", "2JD", "I4", "0QK", "6q", "7Og", "4je", "4gU", "4r4", "2iX", "1LZ", "0rW", "Qe", "5L9", "4Iy", +"4Fx", "5C8", "0h7", "1mw", "0Sz", "pH", "7MV", "4hT", "4ed", "6PG", "9p", "1Nk", "F5", "ST", "7nJ", "4KH", "7pH", "4UJ", "X7", "MV", +"cz", "1Pi", "5kG", "6NE", "4c7", "4vV", "0Mx", "nJ", "0v5", "0cT", "4Xz", "6mY", "6bX", "5GZ", "0lU", "Og", "aK", "0By", "4yW", "4l6", +"6AD", "40o", "0OI", "2zj", "BW", "W6", "4ZK", "6oh", "2hB", "k2", "4fO", "6Sl", "69H", "4Hc", "07E", "2Fn", "d5e", "83m", "4ES", "4P2", +"a0F", "bQL", "0PQ", "7k", "8j", "1Oq", "50W", "hbv", "7oP", "4JR", "05t", "RN", "2Io", "08D", "4Gb", "66I", "7LL", "4iN", "d3", "5Z", +"z1", "oP", "6Bo", "43D", "5IA", "6lC", "2Wm", "0bN", "8ff", "LL", "4A1", "4TP", "cPn", "aeM", "0T3", "0AR", "0NS", "ma", "545", "41u", +"5Kp", "4N0", "CM", "14V", "0mO", "2Xl", "6cB", "4Va", "4xM", "6Mn", "22H", "18", "04s", "SI", "7nW", "4KU", "4ey", "6PZ", "9m", "1Nv", +"e4", "pU", "7MK", "4hI", "4Fe", "67N", "2Hh", "09C", "06B", "Qx", "68O", "4Id", "4gH", "6Rk", "2iE", "j5", "0QV", "6l", "5o8", "4jx", +"4DT", "4Q5", "2JY", "82j", "BJ", "0ax", "4ZV", "4O7", "552", "40r", "0OT", "lf", "aV", "t7", "4yJ", "6Li", "6bE", "4Wf", "0lH", "Oz", +"2Vj", "0cI", "4Xg", "6mD", "6Ch", "42C", "0Me", "nW", "cg", "1Pt", "5kZ", "6NX", "7pU", "4UW", "0ny", "MK", "7LQ", "4iS", "267", "5G", +"0i0", "08Y", "b9D", "66T", "7oM", "4JO", "G2", "RS", "8w", "1Ol", "4dc", "7Aa", "atS", "4kb", "0PL", "7v", "2KC", "H3", "4EN", "64e", +"69U", "b6E", "07X", "Pb", "dRl", "296", "4fR", "4s3", "4xP", "4m1", "22U", "8Jf", "0mR", "0x3", "77v", "626", "5Km", "6no", "CP", "V1", +"0NN", "3kL", "7Pb", "41h", "4za", "6OB", "20d", "0AO", "Y0", "LQ", "6an", "4TM", "bcN", "78w", "Aa", "0bS", "8Eg", "oM", "4b0", "43Y", +"51T", "azL", "9i", "1Nr", "04w", "SM", "7nS", "4KQ", "4Fa", "67J", "2Hl", "09G", "e0", "4Y", "7MO", "4hM", "4gL", "6Ro", "2iA", "j1", +"06F", "2Gm", "68K", "5YA", "4DP", "4Q1", "d4f", "82n", "0QR", "6h", "a1E", "bPO", "556", "40v", "0OP", "lb", "BN", "15U", "4ZR", "4O3", +"6bA", "4Wb", "0lL", "2Yo", "aR", "t3", "4yN", "6Lm", "6Cl", "42G", "0Ma", "nS", "2Vn", "0cM", "4Xc", "79i", "74Y", "4US", "8ge", "MO", +"cc", "1Pp", "bAL", "adN", "0i4", "1lt", "5WZ", "66P", "7LU", "4iW", "0Ry", "5C", "8s", "1Oh", "4dg", "6QD", "7oI", "4JK", "G6", "RW", +"2KG", "H7", "4EJ", "64a", "7Nd", "4kf", "0PH", "7r", "1X8", "1MY", "4fV", "4s7", "69Q", "4Hz", "0sT", "Pf", "0mV", "Nd", "5S8", "4Vx", +"4xT", "4m5", "22Q", "0Cz", "0NJ", "mx", "7Pf", "41l", "5Ki", "6nk", "CT", "V5", "Y4", "LU", "6aj", "4TI", "4ze", "6OF", "by", "0AK", +"2l9", "oI", "4b4", "4wU", "4Yy", "6lZ", "Ae", "0bW", "0So", "4U", "7MC", "4hA", "4Fm", "67F", "3XA", "09K", "0ps", "SA", "aTl", "b5f", +"4eq", "6PR", "9e", "8WG", "8XF", "6d", "5o0", "4jp", "707", "65w", "1z2", "1oS", "06J", "Qp", "68G", "4Il", "53i", "6Rc", "2iM", "1LO", +"23G", "07", "4yB", "6La", "6bM", "4Wn", "18i", "Or", "BB", "0ap", "c4D", "aEo", "5q2", "40z", "8FD", "ln", "co", "346", "5kR", "6NP", +"74U", "bol", "0nq", "MC", "2Vb", "0cA", "4Xo", "6mL", "7SA", "42K", "0Mm", "2xN", "7oE", "4JG", "05a", "2DJ", "2jf", "1Od", "4dk", "6QH", +"482", "5yz", "0Ru", "5O", "0i8", "08Q", "4Gw", "5B7", "5M6", "4Hv", "07P", "Pj", "1X4", "1MU", "4fZ", "473", "7Nh", "4kj", "0PD", "sv", +"2KK", "1nI", "4EF", "64m", "5Ke", "6ng", "CX", "V9", "0NF", "mt", "7Pj", "4uh", "4xX", "4m9", "1F6", "0Cv", "0mZ", "Nh", "5S4", "4Vt", +"4Yu", "6lV", "Ai", "16r", "0Lw", "oE", "4b8", "43Q", "4zi", "6OJ", "bu", "0AG", "Y8", "LY", "6af", "4TE", "4Fi", "67B", "2Hd", "09O", +"e8", "4Q", "7MG", "4hE", "4eu", "6PV", "9a", "1Nz", "0pw", "SE", "aTh", "4KY", "4DX", "4Q9", "1z6", "1oW", "0QZ", "rh", "5o4", "4jt", +"4gD", "6Rg", "2iI", "j9", "06N", "Qt", "68C", "4Ih", "6bI", "4Wj", "0lD", "Ov", "aZ", "03", "4yF", "6Le", "5q6", "4tv", "0OX", "lj", +"BF", "0at", "4ZZ", "6oy", "74Q", "5Ez", "0nu", "MG", "ck", "1Px", "5kV", "6NT", "6Cd", "42O", "0Mi", "2xJ", "2Vf", "0cE", "4Xk", "6mH", +"2jb", "8VY", "4do", "6QL", "7oA", "4JC", "05e", "2DN", "d7E", "08U", "4Gs", "5B3", "486", "bSl", "0Rq", "5K", "1X0", "1MQ", "52w", "477", +"5M2", "4Hr", "07T", "Pn", "2KO", "1nM", "4EB", "64i", "7Nl", "4kn", "8YX", "7z", "0NB", "mp", "7Pn", "41d", "5Ka", "6nc", "2UM", "14G", +"19w", "Nl", "5S0", "4Vp", "bBo", "agm", "1F2", "0Cr", "0Ls", "oA", "ahl", "43U", "4Yq", "6lR", "Am", "16v", "0oo", "2ZL", "6ab", "4TA", +"4zm", "6ON", "bq", "0AC", "2VY", "0cz", "4XT", "4M5", "570", "42p", "0MV", "nd", "cT", "v5", "5ki", "6Nk", "74n", "4Ud", "0nJ", "Mx", +"By", "0aK", "4Ze", "6oF", "6Aj", "40A", "y4", "lU", "ae", "0BW", "4yy", "581", "4B4", "4WU", "18R", "OI", "06q", "QK", "7lU", "4IW", +"53R", "6RX", "0I4", "1Lt", "g6", "rW", "7OI", "4jK", "4Dg", "65L", "2Jj", "1oh", "0pH", "Sz", "7nd", "4Kf", "4eJ", "6Pi", "2kG", "h7", +"0ST", "4n", "7Mx", "4hz", "4FV", "4S7", "1x8", "09p", "4zR", "4o3", "bN", "8Hd", "0oP", "Lb", "75t", "604", "4YN", "6lm", "AR", "T3", +"0LL", "2yo", "6BA", "43j", "4xc", "agR", "22f", "0CM", "0ma", "NS", "6cl", "4VO", "baL", "aDN", "Cc", "14x", "8Ge", "mO", "7PQ", "4uS", +"7NS", "4kQ", "245", "7E", "0k2", "1nr", "coo", "64V", "69f", "4HM", "E0", "PQ", "2hl", "1Mn", "4fa", "6SB", "7Lb", "5yA", "0RN", "5t", +"2IA", "J1", "4GL", "66g", "aUM", "b4G", "05Z", "0d3", "8D", "8Vf", "4dP", "459", "574", "42t", "0MR", "0X3", "dln", "17W", "4XP", "4M1", +"74j", "5EA", "0nN", "3KL", "cP", "29", "5km", "6No", "6An", "40E", "y0", "lQ", "2Tl", "0aO", "4Za", "6oB", "4B0", "4WQ", "18V", "OM", +"aa", "0BS", "bCN", "585", "53V", "axN", "0I0", "1Lp", "06u", "QO", "68x", "4IS", "4Dc", "65H", "2Jn", "1ol", "g2", "rS", "7OM", "4jO", +"4eN", "6Pm", "9Z", "h3", "04D", "2Eo", "aTS", "4Kb", "4FR", "4S3", "d6d", "09t", "0SP", "4j", "a3G", "bRM", "0oT", "Lf", "6aY", "4Tz", +"4zV", "4o7", "bJ", "0Ax", "0LH", "oz", "6BE", "43n", "4YJ", "6li", "AV", "T7", "0me", "NW", "6ch", "4VK", "4xg", "6MD", "22b", "0CI", +"0Ny", "mK", "7PU", "4uW", "5KZ", "6nX", "Cg", "1pt", "0k6", "1nv", "4Ey", "64R", "7NW", "4kU", "241", "7A", "2hh", "1Mj", "4fe", "6SF", +"69b", "4HI", "E4", "PU", "2IE", "J5", "4GH", "66c", "7Lf", "4id", "0RJ", "5p", "2jY", "8Vb", "4dT", "4q5", "5O8", "4Jx", "0qV", "Rd", +"21E", "25", "5ka", "6Nc", "74f", "4Ul", "0nB", "Mp", "1f2", "0cr", "bbo", "79V", "578", "42x", "395", "nl", "am", "364", "4yq", "589", +"76W", "bmn", "0ls", "OA", "Bq", "0aC", "4Zm", "6oN", "6Ab", "40I", "0Oo", "2zL", "0Qm", "6W", "7OA", "4jC", "4Do", "65D", "2Jb", "82Q", +"06y", "QC", "68t", "b7d", "4gs", "5b3", "dSM", "8UE", "8ZD", "4f", "5m2", "4hr", "725", "67u", "1x0", "09x", "04H", "Sr", "7nl", "4Kn", +"4eB", "6Pa", "9V", "1NM", "4YF", "6le", "AZ", "0bh", "0LD", "ov", "6BI", "43b", "4zZ", "6Oy", "bF", "0At", "0oX", "Lj", "5Q6", "4Tv", +"5KV", "6nT", "Ck", "14p", "0Nu", "mG", "7PY", "41S", "4xk", "6MH", "22n", "0CE", "0mi", "2XJ", "6cd", "4VG", "69n", "4HE", "E8", "PY", +"2hd", "1Mf", "4fi", "6SJ", "ath", "4kY", "0Pw", "7M", "2Kx", "1nz", "4Eu", "6pV", "5O4", "4Jt", "05R", "Rh", "8L", "1OW", "4dX", "451", +"7Lj", "4ih", "0RF", "qt", "2II", "J9", "4GD", "66o", "74b", "4Uh", "0nF", "Mt", "cX", "21", "5ke", "6Ng", "5s4", "4vt", "0MZ", "nh", +"1f6", "0cv", "4XX", "4M9", "4B8", "4WY", "0lw", "OE", "ai", "1Rz", "4yu", "6LV", "6Af", "40M", "y8", "lY", "Bu", "0aG", "4Zi", "6oJ", +"4Dk", "6qH", "2Jf", "1od", "0Qi", "6S", "7OE", "4jG", "4gw", "5b7", "0I8", "1Lx", "0ru", "QG", "68p", "5Yz", "4FZ", "67q", "1x4", "1mU", +"0SX", "4b", "5m6", "4hv", "4eF", "6Pe", "9R", "1NI", "04L", "Sv", "7nh", "4Kj", "8EX", "or", "6BM", "43f", "4YB", "6la", "2WO", "0bl", +"8fD", "Ln", "5Q2", "4Tr", "cPL", "aeo", "bB", "0Ap", "0Nq", "mC", "ajn", "41W", "5KR", "6nP", "Co", "14t", "0mm", "2XN", "77I", "4VC", +"4xo", "6ML", "22j", "0CA", "3xA", "1Mb", "4fm", "6SN", "69j", "4HA", "07g", "2FL", "d5G", "83O", "4Eq", "64Z", "a0d", "bQn", "0Ps", "7I", +"8H", "1OS", "50u", "455", "5O0", "4Jp", "05V", "Rl", "2IM", "08f", "5Wa", "66k", "7Ln", "4il", "0RB", "5x", "Bh", "0aZ", "4Zt", "6oW", +"4a9", "40P", "0Ov", "lD", "at", "0BF", "4yh", "6LK", "6bg", "4WD", "Z9", "OX", "2VH", "U8", "4XE", "6mf", "6CJ", "42a", "0MG", "nu", +"cE", "1PV", "5kx", "4n8", "5P5", "4Uu", "8gC", "Mi", "04Q", "Sk", "5N7", "4Kw", "51r", "442", "9O", "1NT", "0SE", "pw", "7Mi", "4hk", +"4FG", "67l", "2HJ", "09a", "3", "QZ", "68m", "4IF", "4gj", "6RI", "2ig", "1Le", "0Qt", "6N", "7OX", "4jZ", "4Dv", "5A6", "0j9", "1oy", +"4xr", "6MQ", "22w", "377", "0mp", "NB", "77T", "blm", "5KO", "6nM", "Cr", "14i", "0Nl", "3kn", "ajs", "41J", "4zC", "aer", "20F", "36", +"0oA", "Ls", "6aL", "4To", "bcl", "78U", "AC", "0bq", "386", "oo", "5r3", "4ws", "5l1", "4iq", "9Kf", "5e", "1y3", "1lR", "736", "66v", +"7oo", "4Jm", "05K", "Rq", "8U", "1ON", "4dA", "6Qb", "7NB", "bQs", "0Pn", "7T", "2Ka", "1nc", "4El", "64G", "69w", "b6g", "07z", "1v2", +"dRN", "8TF", "4fp", "5c0", "akm", "40T", "0Or", "1J2", "Bl", "15w", "4Zp", "6oS", "6bc", "5Ga", "0ln", "2YM", "ap", "0BB", "4yl", "6LO", +"6CN", "42e", "0MC", "nq", "2VL", "0co", "4XA", "6mb", "5P1", "4Uq", "8gG", "Mm", "cA", "1PR", "bAn", "adl", "51v", "446", "9K", "1NP", +"04U", "So", "5N3", "4Ks", "4FC", "67h", "2HN", "09e", "0SA", "ps", "7Mm", "4ho", "4gn", "6RM", "2ic", "1La", "7", "2GO", "68i", "4IB", +"4Dr", "5A2", "d4D", "82L", "0Qp", "6J", "a1g", "bPm", "0mt", "NF", "6cy", "4VZ", "4xv", "6MU", "0V9", "0CX", "0Nh", "mZ", "7PD", "41N", +"5KK", "6nI", "Cv", "14m", "0oE", "Lw", "6aH", "4Tk", "4zG", "6Od", "20B", "32", "0LY", "ok", "5r7", "4ww", "5Iz", "6lx", "AG", "0bu", +"1y7", "1lV", "4GY", "4R8", "5l5", "4iu", "1Bz", "5a", "8Q", "i8", "4dE", "6Qf", "7ok", "4Ji", "05O", "Ru", "2Ke", "1ng", "4Eh", "64C", +"7NF", "4kD", "f9", "7P", "2hy", "3m9", "4ft", "5c4", "69s", "4HX", "0sv", "PD", "23e", "0BN", "5iA", "6LC", "6bo", "4WL", "Z1", "OP", +"0t3", "0aR", "c4f", "aEM", "4a1", "40X", "8Ff", "lL", "cM", "8Ig", "5kp", "4n0", "74w", "617", "0nS", "Ma", "3Fa", "U0", "4XM", "6mn", +"6CB", "42i", "0MO", "2xl", "0SM", "4w", "7Ma", "4hc", "4FO", "67d", "2HB", "K2", "04Y", "Sc", "aTN", "b5D", "4eS", "4p2", "9G", "8We", +"256", "6F", "7OP", "4jR", "cnl", "65U", "0j1", "1oq", "D3", "QR", "68e", "4IN", "4gb", "6RA", "2io", "1Lm", "5KG", "6nE", "Cz", "14a", +"x7", "mV", "7PH", "41B", "4xz", "592", "0V5", "0CT", "0mx", "NJ", "4C7", "4VV", "4YW", "4L6", "AK", "0by", "0LU", "og", "563", "43s", +"4zK", "6Oh", "bW", "w6", "0oI", "2Zj", "6aD", "4Tg", "7og", "4Je", "05C", "Ry", "2jD", "i4", "4dI", "6Qj", "5l9", "4iy", "0RW", "5m", +"2IX", "08s", "4GU", "4R4", "7mV", "4HT", "07r", "PH", "0H7", "1Mw", "4fx", "5c8", "7NJ", "4kH", "f5", "sT", "2Ki", "1nk", "4Ed", "64O", +"6bk", "4WH", "Z5", "OT", "ax", "0BJ", "4yd", "6LG", "4a5", "4tT", "0Oz", "lH", "Bd", "0aV", "4Zx", "aEI", "5P9", "4Uy", "0nW", "Me", +"cI", "1PZ", "5kt", "4n4", "6CF", "42m", "0MK", "ny", "2VD", "U4", "4XI", "6mj", "4FK", "6sh", "2HF", "K6", "0SI", "4s", "7Me", "4hg", +"4eW", "4p6", "9C", "1NX", "0pU", "Sg", "7ny", "6k9", "4Dz", "65Q", "0j5", "1ou", "0Qx", "6B", "7OT", "4jV", "4gf", "6RE", "2ik", "1Li", +"D7", "QV", "68a", "4IJ", "x3", "mR", "7PL", "41F", "5KC", "6nA", "2Uo", "14e", "19U", "NN", "4C3", "4VR", "bBM", "596", "0V1", "0CP", +"0LQ", "oc", "567", "43w", "4YS", "4L2", "AO", "16T", "0oM", "2Zn", "75i", "4Tc", "4zO", "6Ol", "bS", "w2", "8Y", "i0", "4dM", "6Qn", +"7oc", "4Ja", "05G", "2Dl", "d7g", "08w", "4GQ", "4R0", "a2D", "bSN", "0RS", "5i", "0H3", "1Ms", "52U", "ayM", "7mR", "4HP", "07v", "PL", +"2Km", "1no", "5UA", "64K", "7NN", "4kL", "f1", "7X", "5nw", "4k7", "fJ", "0Ex", "0kT", "Hf", "6eY", "4Pz", "5Mk", "6hi", "EV", "P7", +"0HH", "kz", "6FE", "47n", "48o", "6ID", "26b", "0GI", "0ie", "JW", "6gh", "4RK", "5OZ", "6jX", "Gg", "0dU", "0Jy", "iK", "4d6", "4qW", +"4z4", "4oU", "1DZ", "3A", "Ye", "0zW", "4Ay", "5D9", "6yj", "4LI", "A4", "TU", "zy", "0YK", "4be", "6WF", "6XG", "4md", "0VJ", "1p", +"2ME", "N5", "4CH", "62c", "5K8", "4Nx", "0uV", "Vd", "xH", "8Rb", "5pu", "4u5", "D", "13W", "5Lq", "4I1", "534", "46t", "0IR", "28y", +"gP", "69", "5om", "6Jo", "6dC", "5AA", "0jN", "3OL", "2Pl", "0eO", "aT1", "6kB", "6En", "44E", "98", "hQ", "ea", "0FS", "49u", "abL", +"4F0", "4SQ", "8ag", "KM", "02u", "UO", "4X2", "4MS", "57V", "a8F", "0M0", "0XQ", "c2", "vS", "7KM", "4nO", "5PB", "61H", "2Nn", "1kl", +"00D", "2Ao", "6zA", "4Ob", "4aN", "6Tm", "yR", "l3", "0WP", "0j", "a7G", "58W", "4BR", "4W3", "ZN", "84l", "0kP", "Hb", "71t", "644", +"5ns", "4k3", "fN", "8Ld", "0HL", "29g", "6FA", "47j", "5Mo", "6hm", "ER", "P3", "0ia", "JS", "6gl", "4RO", "48k", "7Ya", "26f", "0GM", +"8Ce", "iO", "4d2", "4qS", "beL", "hYw", "Gc", "0dQ", "Ya", "0zS", "cko", "60V", "4z0", "4oQ", "205", "3E", "2ll", "0YO", "4ba", "6WB", +"6yn", "4LM", "A0", "TQ", "2MA", "N1", "4CL", "62g", "6XC", "59I", "0VN", "1t", "xL", "8Rf", "54y", "419", "aQM", "b0G", "01Z", "3PP", +"530", "46p", "0IV", "jd", "DH", "0gz", "5Lu", "4I5", "6dG", "4Qd", "0jJ", "Ix", "gT", "r5", "5oi", "6Jk", "6Ej", "44A", "0Kg", "hU", +"Fy", "0eK", "5ND", "6kF", "4F4", "4SU", "1xZ", "KI", "ee", "0FW", "49q", "5x9", "57R", "6VX", "0M4", "0XU", "02q", "UK", "4X6", "4MW", +"5PF", "61L", "2Nj", "1kh", "c6", "vW", "7KI", "4nK", "4aJ", "6Ti", "yV", "l7", "0tH", "Wz", "6zE", "4Of", "4BV", "4W7", "ZJ", "0yx", +"0WT", "0n", "6YY", "4lz", "5Mc", "6ha", "2SO", "0fl", "1Xa", "kr", "6FM", "47f", "bDm", "aao", "fB", "0Ep", "8bD", "Hn", "5U2", "4Pr", +"5OR", "5Z3", "Go", "10t", "0Jq", "iC", "ann", "45W", "48g", "6IL", "ds", "0GA", "0im", "3Lo", "73I", "4RC", "6yb", "4LA", "03g", "2BL", +"zq", "0YC", "4bm", "6WN", "a4d", "bUn", "0Ts", "3I", "Ym", "87O", "4Aq", "5D1", "5K0", "4Np", "01V", "Vl", "2nQ", "1KS", "54u", "415", +"6XO", "4ml", "0VB", "1x", "2MM", "0xn", "5Sa", "62k", "gX", "61", "5oe", "6Jg", "6dK", "4Qh", "0jF", "It", "L", "0gv", "5Ly", "4I9", +"5w4", "4rt", "0IZ", "jh", "ei", "1Vz", "5mT", "5x5", "4F8", "4SY", "0hw", "KE", "Fu", "0eG", "5NH", "6kJ", "6Ef", "44M", "90", "hY", +"0Ui", "2S", "7KE", "4nG", "5PJ", "6uH", "Xw", "1kd", "0vu", "UG", "6xx", "790", "4cw", "5f7", "0M8", "0XY", "0WX", "0b", "5i6", "4lv", +"4BZ", "63q", "ZF", "0yt", "00L", "Wv", "6zI", "4Oj", "4aF", "6Te", "yZ", "0Zh", "0HD", "kv", "6FI", "47b", "5Mg", "6he", "EZ", "0fh", +"0kX", "Hj", "5U6", "4Pv", "7N9", "6Ky", "fF", "0Et", "0Ju", "iG", "6Dx", "45S", "5OV", "5Z7", "Gk", "0dY", "0ii", "3Lk", "6gd", "4RG", +"48c", "6IH", "dw", "0GE", "zu", "0YG", "4bi", "6WJ", "6yf", "4LE", "A8", "TY", "Yi", "1jz", "4Au", "5D5", "4z8", "4oY", "0Tw", "3M", +"xD", "1KW", "54q", "411", "5K4", "4Nt", "01R", "Vh", "2MI", "N9", "4CD", "62o", "6XK", "4mh", "0VF", "ut", "6dO", "4Ql", "0jB", "Ip", +"25E", "65", "5oa", "6Jc", "538", "46x", "9Pg", "jl", "H", "0gr", "bfo", "aCm", "72W", "bin", "0hs", "KA", "em", "324", "49y", "5x1", +"6Eb", "44I", "94", "3nm", "Fq", "0eC", "5NL", "6kN", "5PN", "61D", "Xs", "86Q", "0Um", "2W", "7KA", "4nC", "4cs", "5f3", "39W", "8QE", +"02y", "UC", "aRn", "794", "765", "63u", "ZB", "0yp", "9Ne", "0f", "5i2", "4lr", "4aB", "6Ta", "2oO", "0Zl", "00H", "Wr", "6zM", "4On", +"5lW", "5y6", "dj", "0GX", "0it", "JF", "6gy", "4RZ", "5OK", "6jI", "Gv", "0dD", "83", "iZ", "6De", "45N", "5nf", "6Kd", "24B", "72", +"0kE", "Hw", "6eH", "4Pk", "5Mz", "6hx", "EG", "0fu", "0HY", "kk", "5v7", "4sw", "5h5", "4mu", "1Fz", "1a", "2MT", "0xw", "4CY", "4V8", +"7kk", "4Ni", "01O", "Vu", "xY", "m8", "54l", "6Uf", "6Zg", "4oD", "b9", "3P", "Yt", "0zF", "4Ah", "60C", "4Y9", "4LX", "0wv", "TD", +"zh", "0YZ", "4bt", "5g4", "Fl", "11w", "5NQ", "6kS", "aom", "44T", "0Kr", "1N2", "ep", "0FB", "49d", "6HO", "6fc", "5Ca", "0hn", "3Ml", +"U", "0go", "bfr", "6ib", "6GN", "46e", "0IC", "jq", "gA", "0Ds", "bEn", "hyU", "5T1", "4Qq", "8cG", "Im", "00U", "Wo", "5J3", "4Os", +"55v", "406", "yC", "0Zq", "0WA", "ts", "6YL", "4lo", "4BC", "63h", "2LN", "0ym", "02d", "2CO", "6xa", "4MB", "4cn", "6VM", "2mc", "1Ha", +"0Up", "2J", "a5g", "bTm", "5PS", "5E2", "Xn", "86L", "0ip", "JB", "73T", "bhm", "48z", "5y2", "dn", "337", "87", "3on", "6Da", "45J", +"5OO", "6jM", "Gr", "10i", "0kA", "Hs", "6eL", "4Po", "5nb", "aar", "24F", "76", "8AE", "ko", "5v3", "4ss", "bgl", "aBn", "EC", "0fq", +"2MP", "0xs", "776", "62v", "5h1", "4mq", "9Of", "1e", "2nL", "1KN", "54h", "6Ub", "7ko", "4Nm", "01K", "Vq", "Yp", "0zB", "4Al", "60G", +"6Zc", "bUs", "0Tn", "3T", "zl", "8PF", "4bp", "5g0", "aSm", "787", "03z", "1r2", "4e9", "44P", "0Kv", "hD", "Fh", "0eZ", "5NU", "6kW", +"6fg", "4SD", "0hj", "KX", "et", "0FF", "5mI", "6HK", "6GJ", "46a", "0IG", "ju", "Q", "Q8", "5Ld", "6if", "5T5", "4Qu", "1zz", "Ii", +"gE", "0Dw", "5ox", "4j8", "55r", "402", "yG", "0Zu", "00Q", "Wk", "5J7", "4Ow", "4BG", "63l", "2LJ", "0yi", "0WE", "tw", "6YH", "4lk", +"4cj", "6VI", "2mg", "0XD", "0vh", "UZ", "6xe", "4MF", "5PW", "5E6", "Xj", "1ky", "0Ut", "2N", "7KX", "4nZ", "5OC", "6jA", "2Qo", "0dL", +"1ZA", "iR", "6Dm", "45F", "48v", "acO", "db", "0GP", "94M", "JN", "4G3", "4RR", "5Mr", "4H2", "EO", "12T", "0HQ", "kc", "527", "47w", +"5nn", "6Kl", "fS", "s2", "0kM", "3NO", "71i", "4Pc", "7kc", "4Na", "01G", "3PM", "xQ", "m0", "54d", "6Un", "a6D", "59T", "0VS", "1i", +"197", "85o", "4CQ", "4V0", "4Y1", "4LP", "03v", "TL", "0L3", "0YR", "56U", "a9E", "6Zo", "4oL", "b1", "3X", "2Om", "0zN", "5QA", "60K", +"ex", "0FJ", "49l", "6HG", "6fk", "4SH", "0hf", "KT", "Fd", "0eV", "5NY", "aAI", "4e5", "4pT", "0Kz", "hH", "gI", "1TZ", "5ot", "4j4", +"5T9", "4Qy", "0jW", "Ie", "DU", "Q4", "5Lh", "6ij", "6GF", "46m", "0IK", "jy", "0WI", "0s", "6YD", "4lg", "4BK", "6wh", "ZW", "O6", +"0tU", "Wg", "6zX", "6o9", "4aW", "4t6", "yK", "0Zy", "0Ux", "2B", "7KT", "4nV", "bzI", "61Q", "Xf", "1ku", "02l", "UV", "6xi", "4MJ", +"4cf", "6VE", "2mk", "0XH", "0Jd", "iV", "6Di", "45B", "5OG", "6jE", "Gz", "0dH", "0ix", "JJ", "4G7", "4RV", "48r", "6IY", "df", "0GT", +"0HU", "kg", "523", "47s", "5Mv", "4H6", "EK", "0fy", "0kI", "3NK", "6eD", "4Pg", "5nj", "6Kh", "fW", "s6", "xU", "m4", "5ph", "6Uj", +"7kg", "4Ne", "01C", "Vy", "193", "1hZ", "4CU", "4V4", "5h9", "4my", "0VW", "1m", "zd", "0YV", "4bx", "5g8", "4Y5", "4LT", "03r", "TH", +"Yx", "0zJ", "4Ad", "60O", "6Zk", "4oH", "b5", "wT", "6fo", "4SL", "0hb", "KP", "27e", "0FN", "49h", "6HC", "4e1", "44X", "8Bf", "hL", +"0p3", "0eR", "bdO", "aAM", "70w", "657", "0jS", "Ia", "gM", "8Mg", "5op", "4j0", "6GB", "46i", "0IO", "28d", "Y", "Q0", "5Ll", "6in", +"4BO", "63d", "ZS", "O2", "0WM", "0w", "7Ia", "4lc", "4aS", "4t2", "yO", "8Se", "00Y", "Wc", "aPN", "b1D", "bzM", "61U", "Xb", "1kq", +"216", "2F", "7KP", "4nR", "4cb", "6VA", "2mo", "0XL", "02h", "UR", "6xm", "4MN", "5j7", "4ow", "0TY", "3c", "YG", "0zu", "5Qz", "60p", +"6yH", "4Lk", "03M", "Tw", "2lJ", "0Yi", "4bG", "6Wd", "6Xe", "4mF", "0Vh", "1R", "2Mg", "0xD", "4Cj", "62A", "7kX", "4NZ", "0ut", "VF", +"xj", "1Ky", "5pW", "5e6", "5nU", "6KW", "fh", "0EZ", "0kv", "HD", "4E9", "4PX", "5MI", "6hK", "Et", "0fF", "0Hj", "kX", "6Fg", "47L", +"48M", "6If", "dY", "50", "0iG", "Ju", "6gJ", "4Ri", "5Ox", "4J8", "GE", "0dw", "1Zz", "ii", "5t5", "4qu", "02W", "Um", "5H1", "4Mq", +"57t", "424", "2mP", "0Xs", "0UC", "2y", "7Ko", "4nm", "bzr", "61j", "2NL", "1kN", "00f", "2AM", "6zc", "bus", "4al", "6TO", "yp", "0ZB", +"0Wr", "0H", "a7e", "58u", "4Bp", "5G0", "Zl", "84N", "f", "13u", "5LS", "5Y2", "amo", "46V", "0Ip", "jB", "gr", "1Ta", "5oO", "6JM", +"6da", "4QB", "0jl", "3On", "2PN", "0em", "5Nb", "aAr", "6EL", "44g", "0KA", "hs", "eC", "0Fq", "49W", "abn", "5V3", "4Ss", "8aE", "Ko", +"YC", "0zq", "754", "60t", "5j3", "4os", "9Md", "3g", "2lN", "0Ym", "4bC", "7GA", "6yL", "4Lo", "03I", "Ts", "2Mc", "1ha", "4Cn", "62E", +"6Xa", "4mB", "0Vl", "1V", "xn", "8RD", "5pS", "5e2", "aQo", "b0e", "01x", "VB", "0kr", "1n2", "71V", "bjo", "5nQ", "6KS", "fl", "315", +"0Hn", "29E", "6Fc", "47H", "5MM", "6hO", "Ep", "0fB", "0iC", "Jq", "6gN", "4Rm", "48I", "6Ib", "26D", "54", "8CG", "im", "509", "45y", +"ben", "hYU", "GA", "0ds", "4cY", "420", "2mT", "0Xw", "02S", "Ui", "5H5", "4Mu", "5Pd", "61n", "XY", "M8", "0UG", "vu", "7Kk", "4ni", +"4ah", "6TK", "yt", "0ZF", "B9", "WX", "6zg", "4OD", "4Bt", "5G4", "Zh", "0yZ", "0Wv", "0L", "4y9", "4lX", "6Gy", "46R", "0It", "jF", +"b", "0gX", "5LW", "5Y6", "6de", "4QF", "0jh", "IZ", "gv", "0DD", "5oK", "6JI", "6EH", "44c", "0KE", "hw", "2PJ", "0ei", "5Nf", "6kd", +"5V7", "4Sw", "0hY", "Kk", "eG", "0Fu", "49S", "6Hx", "7ia", "4Lc", "03E", "2Bn", "zS", "o2", "4bO", "6Wl", "a4F", "bUL", "0TQ", "3k", +"YO", "87m", "4AS", "4T2", "7kP", "4NR", "01t", "VN", "xb", "1Kq", "54W", "hfv", "6Xm", "4mN", "1FA", "1Z", "2Mo", "0xL", "4Cb", "62I", +"5MA", "6hC", "2Sm", "0fN", "0Hb", "kP", "6Fo", "47D", "bDO", "aaM", "0P3", "0ER", "8bf", "HL", "4E1", "4PP", "5Op", "4J0", "GM", "10V", +"0JS", "ia", "505", "45u", "48E", "6In", "dQ", "58", "0iO", "3LM", "6gB", "4Ra", "0UK", "2q", "7Kg", "4ne", "5Ph", "61b", "XU", "M4", +"0vW", "Ue", "5H9", "4My", "4cU", "4v4", "2mX", "1HZ", "0Wz", "tH", "4y5", "4lT", "4Bx", "5G8", "Zd", "0yV", "B5", "WT", "6zk", "4OH", +"4ad", "6TG", "yx", "0ZJ", "gz", "0DH", "5oG", "6JE", "6di", "4QJ", "0jd", "IV", "n", "0gT", "680", "6iY", "4g7", "4rV", "0Ix", "jJ", +"eK", "0Fy", "5mv", "4h6", "6fX", "5CZ", "0hU", "Kg", "FW", "S6", "5Nj", "6kh", "6ED", "44o", "0KI", "3nK", "zW", "o6", "4bK", "6Wh", +"6yD", "4Lg", "03A", "2Bj", "YK", "0zy", "4AW", "4T6", "6ZX", "6O9", "0TU", "3o", "xf", "1Ku", "54S", "6UY", "7kT", "4NV", "01p", "VJ", +"2Mk", "0xH", "4Cf", "62M", "6Xi", "4mJ", "0Vd", "uV", "0Hf", "kT", "6Fk", "4sH", "5ME", "6hG", "Ex", "0fJ", "0kz", "HH", "4E5", "4PT", +"5nY", "aaI", "fd", "0EV", "0JW", "ie", "501", "45q", "5Ot", "4J4", "GI", "10R", "0iK", "Jy", "6gF", "4Re", "48A", "6Ij", "dU", "q4", +"5Pl", "61f", "XQ", "M0", "0UO", "2u", "7Kc", "4na", "4cQ", "428", "39u", "8Qg", "0vS", "Ua", "aRL", "b3F", "bxO", "63W", "0l3", "0yR", +"234", "0D", "4y1", "4lP", "55I", "6TC", "2om", "0ZN", "B1", "WP", "6zo", "4OL", "6dm", "4QN", "1zA", "IR", "25g", "0DL", "5oC", "6JA", +"4g3", "46Z", "9PE", "jN", "j", "0gP", "684", "aCO", "72u", "675", "0hQ", "Kc", "eO", "8Oe", "5mr", "4h2", "7Ua", "44k", "0KM", "3nO", +"FS", "S2", "5Nn", "6kl", "4x6", "4mW", "0Vy", "1C", "0m4", "0xU", "5SZ", "62P", "7kI", "4NK", "C6", "VW", "2nj", "1Kh", "54N", "6UD", +"6ZE", "4of", "0TH", "3r", "YV", "L7", "4AJ", "60a", "6yY", "4Lz", "0wT", "Tf", "zJ", "0Yx", "4bV", "4w7", "5lu", "4i5", "dH", "0Gz", +"0iV", "Jd", "5W8", "4Rx", "5Oi", "6jk", "GT", "R5", "0JJ", "ix", "6DG", "45l", "5nD", "6KF", "fy", "0EK", "0kg", "HU", "6ej", "4PI", +"5MX", "5X9", "Ee", "0fW", "1XZ", "kI", "4f4", "4sU", "00w", "WM", "4Z0", "4OQ", "55T", "hgu", "ya", "0ZS", "a0", "0Y", "6Yn", "4lM", +"4Ba", "63J", "2Ll", "0yO", "02F", "2Cm", "6xC", "aG0", "4cL", "6Vo", "2mA", "n1", "0UR", "2h", "a5E", "bTO", "5Pq", "4U1", "XL", "86n", +"FN", "11U", "5Ns", "4K3", "516", "44v", "0KP", "hb", "eR", "p3", "49F", "6Hm", "6fA", "4Sb", "0hL", "3MN", "w", "0gM", "5LB", "7ya", +"6Gl", "46G", "0Ia", "jS", "gc", "0DQ", "bEL", "hyw", "4D2", "4QS", "8ce", "IO", "0m0", "0xQ", "byL", "62T", "4x2", "4mS", "227", "1G", +"2nn", "1Kl", "54J", "7Ea", "7kM", "4NO", "C2", "VS", "YR", "L3", "4AN", "60e", "6ZA", "4ob", "0TL", "3v", "zN", "8Pd", "4bR", "4w3", +"aSO", "b2E", "03X", "Tb", "0iR", "3LP", "73v", "666", "48X", "4i1", "dL", "8Nf", "0JN", "3oL", "6DC", "45h", "5Om", "6jo", "GP", "R1", +"0kc", "HQ", "6en", "4PM", "a09", "6KB", "24d", "0EO", "8Ag", "kM", "4f0", "47Y", "697", "aBL", "Ea", "0fS", "4ay", "5d9", "ye", "0ZW", +"00s", "WI", "4Z4", "4OU", "4Be", "63N", "Zy", "0yK", "a4", "tU", "6Yj", "4lI", "4cH", "6Vk", "2mE", "n5", "02B", "Ux", "6xG", "4Md", +"5Pu", "4U5", "XH", "86j", "0UV", "2l", "5k8", "4nx", "512", "44r", "0KT", "hf", "FJ", "0ex", "5Nw", "4K7", "6fE", "4Sf", "0hH", "Kz", +"eV", "p7", "49B", "6Hi", "6Gh", "46C", "0Ie", "jW", "s", "0gI", "5LF", "6iD", "4D6", "4QW", "0jy", "IK", "gg", "0DU", "5oZ", "6JX", +"7kA", "4NC", "01e", "3Po", "xs", "8RY", "54F", "6UL", "a6f", "59v", "0Vq", "1K", "d3E", "85M", "4Cs", "5F3", "5I2", "4Lr", "03T", "Tn", +"zB", "0Yp", "56w", "437", "6ZM", "4on", "1Da", "3z", "2OO", "0zl", "4AB", "60i", "5Oa", "6jc", "2QM", "0dn", "0JB", "ip", "6DO", "45d", +"48T", "acm", "1B2", "0Gr", "94o", "Jl", "5W0", "4Rp", "5MP", "5X1", "Em", "12v", "0Hs", "kA", "all", "47U", "5nL", "6KN", "fq", "0EC", +"0ko", "3Nm", "6eb", "4PA", "a8", "0Q", "6Yf", "4lE", "4Bi", "63B", "Zu", "0yG", "0tw", "WE", "4Z8", "4OY", "4au", "5d5", "yi", "1Jz", +"0UZ", "vh", "5k4", "4nt", "5Py", "4U9", "XD", "1kW", "02N", "Ut", "6xK", "4Mh", "4cD", "6Vg", "2mI", "n9", "eZ", "43", "49N", "6He", +"6fI", "4Sj", "0hD", "Kv", "FF", "0et", "7n9", "6ky", "5u6", "4pv", "0KX", "hj", "gk", "0DY", "5oV", "5z7", "6dx", "5Az", "0ju", "IG", +"Dw", "0gE", "5LJ", "6iH", "6Gd", "46O", "0Ii", "28B", "xw", "1Kd", "54B", "6UH", "7kE", "4NG", "01a", "3Pk", "0m8", "0xY", "4Cw", "5F7", +"6Xx", "59r", "0Vu", "1O", "zF", "0Yt", "4bZ", "433", "5I6", "4Lv", "03P", "Tj", "YZ", "0zh", "4AF", "60m", "6ZI", "4oj", "0TD", "wv", +"0JF", "it", "6DK", "4qh", "5Oe", "6jg", "GX", "R9", "0iZ", "Jh", "5W4", "4Rt", "48P", "4i9", "dD", "0Gv", "0Hw", "kE", "4f8", "47Q", +"5MT", "5X5", "Ei", "12r", "0kk", "HY", "6ef", "4PE", "5nH", "6KJ", "fu", "0EG", "4Bm", "63F", "Zq", "0yC", "0Wo", "0U", "6Yb", "4lA", +"4aq", "5d1", "ym", "8SG", "0ts", "WA", "aPl", "b1f", "747", "61w", "2NQ", "1kS", "9Lg", "2d", "5k0", "4np", "57i", "6Vc", "2mM", "0Xn", +"02J", "Up", "6xO", "4Ml", "6fM", "4Sn", "1xa", "Kr", "27G", "47", "49J", "6Ha", "5u2", "44z", "8BD", "hn", "FB", "0ep", "bdm", "aAo", +"70U", "bkl", "0jq", "IC", "go", "306", "5oR", "5z3", "7WA", "46K", "0Im", "28F", "Ds", "0gA", "5LN", "6iL", "0cY", "020", "6mT", "4Xw", +"42S", "6Cx", "nG", "0Mu", "1Pd", "cw", "6NH", "5kJ", "4UG", "74M", "3Kk", "0ni", "0ah", "BZ", "6oe", "4ZF", "40b", "6AI", "lv", "0OD", +"0Bt", "aF", "6Ly", "4yZ", "4Wv", "5R6", "Oj", "0lX", "Qh", "06R", "4It", "5L4", "461", "4gX", "1LW", "1Y6", "rt", "0QF", "4jh", "7Oj", +"65o", "4DD", "I9", "2JI", "SY", "F8", "4KE", "7nG", "6PJ", "4ei", "1Nf", "2kd", "4M", "0Sw", "4hY", "490", "5C5", "4Fu", "09S", "2Hx", +"6OR", "4zq", "354", "bm", "LA", "0os", "bnn", "75W", "6lN", "4Ym", "0bC", "Aq", "2yL", "0Lo", "43I", "6Bb", "6Mc", "5ha", "15", "22E", +"Np", "0mB", "4Vl", "6cO", "aDm", "bao", "1pS", "1e2", "ml", "8GF", "41x", "548", "4kr", "5n2", "7f", "8YD", "1nQ", "2KS", "64u", "715", +"4Hn", "69E", "Pr", "07H", "1MM", "2hO", "6Sa", "4fB", "4iC", "7LA", "5W", "0Rm", "08I", "2Ib", "66D", "4Go", "b4d", "aUn", "RC", "05y", +"8VE", "8g", "5a3", "4ds", "42W", "ain", "nC", "0Mq", "17t", "024", "6mP", "4Xs", "4UC", "74I", "3Ko", "0nm", "8IY", "cs", "6NL", "5kN", +"40f", "6AM", "lr", "8FX", "0al", "2TO", "6oa", "4ZB", "4Wr", "5R2", "On", "18u", "0Bp", "aB", "afo", "bCm", "465", "53u", "1LS", "1Y2", +"Ql", "06V", "4Ip", "5L0", "65k", "5Ta", "1oO", "2JM", "6x", "0QB", "4jl", "7On", "6PN", "4em", "1Nb", "9y", "2EL", "04g", "4KA", "7nC", +"5C1", "4Fq", "09W", "d6G", "4I", "0Ss", "bRn", "494", "LE", "0ow", "4TY", "4A8", "6OV", "4zu", "1Qz", "bi", "oY", "z8", "43M", "6Bf", +"6lJ", "4Yi", "0bG", "Au", "Nt", "0mF", "4Vh", "6cK", "6Mg", "4xD", "11", "22A", "mh", "0NZ", "4ut", "5p4", "4N9", "5Ky", "1pW", "CD", +"1nU", "2KW", "64q", "4EZ", "4kv", "5n6", "7b", "0PX", "1MI", "2hK", "6Se", "4fF", "4Hj", "69A", "Pv", "07L", "08M", "2If", "6rH", "4Gk", +"4iG", "7LE", "5S", "0Ri", "1Ox", "8c", "5a7", "4dw", "5Zz", "7oY", "RG", "0qu", "1Pl", "21f", "adR", "5kB", "4UO", "74E", "MS", "X2", +"0cQ", "028", "79u", "bbL", "4vS", "4c2", "nO", "8De", "8Kd", "aN", "4l3", "4yR", "634", "76t", "Ob", "0lP", "W3", "BR", "6om", "4ZN", +"40j", "6AA", "2zo", "0OL", "6t", "0QN", "5zA", "7Ob", "65g", "4DL", "I1", "2JA", "0g3", "06Z", "b7G", "68W", "469", "4gP", "284", "dSn", +"4E", "275", "4hQ", "498", "67V", "b8F", "1mr", "0h2", "SQ", "F0", "4KM", "7nO", "6PB", "4ea", "1Nn", "9u", "6lF", "4Ye", "0bK", "Ay", +"oU", "z4", "43A", "6Bj", "6OZ", "4zy", "0AW", "be", "LI", "2O9", "4TU", "4A4", "4N5", "5Ku", "14S", "CH", "md", "0NV", "41p", "540", +"6Mk", "4xH", "u5", "22M", "Nx", "0mJ", "4Vd", "6cG", "4Hf", "69M", "Pz", "0sH", "k7", "2hG", "6Si", "4fJ", "4kz", "7Nx", "7n", "0PT", +"1nY", "dqh", "4P7", "4EV", "4JW", "7oU", "RK", "05q", "1Ot", "8o", "6QX", "50R", "4iK", "7LI", "qW", "d6", "08A", "2Ij", "66L", "4Gg", +"4UK", "74A", "MW", "X6", "1Ph", "21b", "6ND", "5kF", "4vW", "4c6", "nK", "0My", "0cU", "0v4", "6mX", "5HZ", "4Wz", "6bY", "Of", "0lT", +"0Bx", "aJ", "4l7", "4yV", "40n", "6AE", "lz", "0OH", "W7", "BV", "6oi", "4ZJ", "65c", "4DH", "I5", "2JE", "6p", "0QJ", "4jd", "7Of", +"4r5", "4gT", "280", "2iY", "Qd", "0rV", "4Ix", "5L8", "5C9", "4Fy", "1mv", "0h6", "4A", "1CZ", "4hU", "7MW", "6PF", "4ee", "1Nj", "9q", +"SU", "F4", "4KI", "7nK", "oQ", "z0", "43E", "6Bn", "6lB", "4Ya", "0bO", "2Wl", "LM", "8fg", "4TQ", "4A0", "aeL", "cPo", "0AS", "ba", +"3kP", "0NR", "41t", "544", "4N1", "5Kq", "14W", "CL", "2Xm", "0mN", "5FA", "6cC", "6Mo", "4xL", "19", "22I", "k3", "2hC", "6Sm", "4fN", +"4Hb", "69I", "2Fo", "07D", "83l", "d5d", "4P3", "4ER", "bQM", "a0G", "7j", "0PP", "1Op", "8k", "hbw", "50V", "4JS", "7oQ", "RO", "05u", +"08E", "2In", "66H", "4Gc", "4iO", "7LM", "qS", "d2", "0ay", "BK", "4O6", "4ZW", "40s", "553", "lg", "0OU", "t6", "aW", "6Lh", "4yK", +"4Wg", "6bD", "2Yj", "0lI", "0cH", "2Vk", "6mE", "4Xf", "42B", "6Ci", "nV", "0Md", "1Pu", "cf", "6NY", "bAI", "4UV", "7pT", "MJ", "0nx", +"SH", "04r", "4KT", "7nV", "azI", "4ex", "1Nw", "9l", "pT", "e5", "4hH", "7MJ", "67O", "4Fd", "09B", "2Hi", "Qy", "06C", "4Ie", "68N", +"6Rj", "4gI", "j4", "2iD", "6m", "0QW", "4jy", "5o9", "4Q4", "4DU", "1oZ", "2JX", "4m0", "4xQ", "8Jg", "22T", "Na", "0mS", "627", "77w", +"6nn", "5Kl", "V0", "CQ", "3kM", "0NO", "41i", "7Pc", "6OC", "5jA", "0AN", "20e", "LP", "Y1", "4TL", "6ao", "78v", "bcO", "0bR", "0w3", +"oL", "8Ef", "43X", "4b1", "4iR", "7LP", "5F", "266", "08X", "0i1", "66U", "b9E", "4JN", "7oL", "RR", "G3", "1Om", "8v", "6QA", "4db", +"4kc", "7Na", "7w", "0PM", "H2", "2KB", "64d", "4EO", "b6D", "69T", "Pc", "07Y", "297", "dRm", "4s2", "4fS", "40w", "557", "lc", "0OQ", +"15T", "BO", "4O2", "4ZS", "4Wc", "76i", "2Yn", "0lM", "t2", "aS", "6Ll", "4yO", "42F", "6Cm", "nR", "8Dx", "0cL", "2Vo", "6mA", "4Xb", +"4UR", "74X", "MN", "8gd", "1Pq", "cb", "adO", "bAM", "azM", "51U", "1Ns", "9h", "SL", "04v", "4KP", "7nR", "67K", "5VA", "09F", "2Hm", +"4X", "e1", "4hL", "7MN", "6Rn", "4gM", "j0", "3ya", "2Gl", "06G", "4Ia", "68J", "4Q0", "4DQ", "82o", "d4g", "6i", "0QS", "bPN", "a1D", +"Ne", "0mW", "4Vy", "5S9", "4m4", "4xU", "1SZ", "22P", "my", "0NK", "41m", "7Pg", "6nj", "5Kh", "V4", "CU", "LT", "Y5", "4TH", "6ak", +"6OG", "4zd", "0AJ", "bx", "oH", "0Lz", "4wT", "4b5", "78r", "4Yx", "0bV", "Ad", "1lu", "0i5", "66Q", "4Gz", "4iV", "7LT", "5B", "0Rx", +"1Oi", "8r", "6QE", "4df", "4JJ", "7oH", "RV", "G7", "H6", "2KF", "6ph", "4EK", "4kg", "7Ne", "7s", "0PI", "1MX", "1X9", "4s6", "4fW", +"5XZ", "69P", "Pg", "0sU", "06", "23F", "afr", "4yC", "4Wo", "6bL", "Os", "0lA", "0aq", "BC", "aEn", "c4E", "4ts", "5q3", "lo", "8FE", +"347", "cn", "6NQ", "5kS", "bom", "74T", "MB", "0np", "17i", "2Vc", "6mM", "4Xn", "42J", "6Ca", "2xO", "0Ml", "4T", "0Sn", "5xa", "7MB", +"67G", "4Fl", "09J", "2Ha", "1u2", "04z", "b5g", "aTm", "6PS", "4ep", "8WF", "9d", "6e", "8XG", "4jq", "5o1", "65v", "706", "1oR", "1z3", +"Qq", "06K", "4Im", "68F", "6Rb", "4gA", "1LN", "2iL", "6nf", "5Kd", "V8", "CY", "mu", "0NG", "41a", "7Pk", "4m8", "4xY", "0Cw", "1F7", +"Ni", "19r", "4Vu", "5S5", "6lW", "4Yt", "0bZ", "Ah", "oD", "0Lv", "43P", "4b9", "6OK", "4zh", "0AF", "bt", "LX", "Y9", "4TD", "6ag", +"4JF", "7oD", "RZ", "0qh", "1Oe", "2jg", "6QI", "4dj", "4iZ", "483", "5N", "0Rt", "08P", "0i9", "5B6", "4Gv", "4Hw", "5M7", "Pk", "07Q", +"1MT", "1X5", "472", "52r", "4kk", "7Ni", "sw", "0PE", "1nH", "2KJ", "64l", "4EG", "4Wk", "6bH", "Ow", "0lE", "02", "23B", "6Ld", "4yG", +"4tw", "5q7", "lk", "0OY", "0au", "BG", "6ox", "5Jz", "4UZ", "74P", "MF", "0nt", "1Py", "cj", "6NU", "5kW", "42N", "6Ce", "nZ", "0Mh", +"0cD", "2Vg", "6mI", "4Xj", "67C", "4Fh", "09N", "2He", "4P", "e9", "4hD", "7MF", "6PW", "4et", "3n9", "2ky", "SD", "0pv", "4KX", "7nZ", +"4Q8", "4DY", "1oV", "1z7", "6a", "1Az", "4ju", "5o5", "6Rf", "4gE", "j8", "2iH", "Qu", "06O", "4Ii", "68B", "mq", "0NC", "41e", "7Po", +"6nb", "bar", "14F", "2UL", "Nm", "19v", "4Vq", "5S1", "agl", "bBn", "0Cs", "1F3", "1I2", "0Lr", "43T", "ahm", "6lS", "4Yp", "16w", "Al", +"2ZM", "0on", "5Da", "6ac", "6OO", "4zl", "0AB", "bp", "1Oa", "8z", "6QM", "4dn", "4JB", "aUs", "2DO", "05d", "08T", "d7D", "5B2", "4Gr", +"bSm", "487", "5J", "0Rp", "1MP", "1X1", "476", "52v", "4Hs", "5M3", "Po", "07U", "1nL", "2KN", "64h", "4EC", "4ko", "7Nm", "ss", "0PA", +"QJ", "06p", "4IV", "7lT", "6RY", "4gz", "1Lu", "0I5", "rV", "g7", "4jJ", "7OH", "65M", "4Df", "1oi", "2Jk", "2Ej", "04A", "4Kg", "7ne", +"6Ph", "4eK", "h6", "2kF", "4o", "0SU", "5xZ", "7My", "4S6", "4FW", "09q", "1x9", "17R", "2VX", "4M4", "4XU", "42q", "571", "ne", "0MW", +"v4", "cU", "6Nj", "5kh", "4Ue", "74o", "My", "0nK", "0aJ", "Bx", "6oG", "4Zd", "4tH", "6Ak", "lT", "y5", "0BV", "ad", "580", "4yx", +"4WT", "4B5", "OH", "0lz", "4kP", "7NR", "7D", "244", "1ns", "0k3", "64W", "con", "4HL", "69g", "PP", "E1", "1Mo", "2hm", "6SC", "52I", +"4ia", "7Lc", "5u", "0RO", "J0", "3Ya", "66f", "4GM", "b4F", "aUL", "Ra", "0qS", "8Vg", "8E", "458", "4dQ", "4o2", "4zS", "8He", "bO", +"Lc", "0oQ", "605", "75u", "6ll", "4YO", "T2", "AS", "2yn", "0LM", "43k", "7Ra", "6MA", "4xb", "0CL", "22g", "NR", "19I", "4VN", "6cm", +"aDO", "baM", "14y", "Cb", "mN", "8Gd", "41Z", "7PP", "axO", "53W", "1Lq", "0I1", "QN", "06t", "4IR", "68y", "65I", "4Db", "1om", "2Jo", +"6Z", "g3", "4jN", "7OL", "6Pl", "4eO", "h2", "2kB", "2En", "04E", "4Kc", "7na", "4S2", "4FS", "09u", "d6e", "4k", "0SQ", "bRL", "a3F", +"42u", "575", "na", "0MS", "17V", "dlo", "4M0", "4XQ", "4Ua", "74k", "3KM", "0nO", "28", "cQ", "6Nn", "5kl", "40D", "6Ao", "lP", "y1", +"0aN", "2Tm", "6oC", "5JA", "4WP", "4B1", "OL", "18W", "0BR", "0W3", "584", "bCO", "1nw", "0k7", "64S", "4Ex", "4kT", "7NV", "sH", "0Pz", +"1Mk", "2hi", "6SG", "4fd", "4HH", "69c", "PT", "E5", "J4", "2ID", "66b", "4GI", "4ie", "7Lg", "5q", "0RK", "1OZ", "8A", "4q4", "4dU", +"4Jy", "5O9", "Re", "0qW", "Lg", "0oU", "5DZ", "6aX", "4o6", "4zW", "0Ay", "bK", "2yj", "0LI", "43o", "6BD", "6lh", "4YK", "T6", "AW", +"NV", "0md", "4VJ", "6ci", "6ME", "4xf", "0CH", "22c", "mJ", "0Nx", "4uV", "7PT", "6nY", "baI", "1pu", "Cf", "6V", "0Ql", "4jB", "aus", +"65E", "4Dn", "1oa", "2Jc", "QB", "06x", "b7e", "68u", "5b2", "4gr", "8UD", "dSL", "4g", "8ZE", "4hs", "5m3", "67t", "724", "09y", "1x1", +"Ss", "04I", "4Ko", "7nm", "azr", "4eC", "1NL", "9W", "24", "21D", "6Nb", "bAr", "4Um", "74g", "Mq", "0nC", "0cs", "1f3", "79W", "bbn", +"42y", "579", "nm", "394", "365", "al", "588", "4yp", "bmo", "76V", "1i2", "0lr", "0aB", "Bp", "6oO", "4Zl", "40H", "6Ac", "2zM", "0On", +"4HD", "69o", "PX", "E9", "1Mg", "2he", "6SK", "4fh", "4kX", "7NZ", "7L", "0Pv", "3N9", "2Ky", "6pW", "4Et", "4Ju", "5O5", "Ri", "05S", +"1OV", "8M", "450", "4dY", "4ii", "7Lk", "qu", "0RG", "J8", "2IH", "66n", "4GE", "6ld", "4YG", "0bi", "2WJ", "ow", "0LE", "43c", "6BH", +"6Ox", "5jz", "0Au", "bG", "Lk", "0oY", "4Tw", "5Q7", "6nU", "5KW", "14q", "Cj", "mF", "0Nt", "41R", "7PX", "6MI", "4xj", "0CD", "22o", +"NZ", "0mh", "4VF", "6ce", "65A", "4Dj", "1oe", "2Jg", "6R", "0Qh", "4jF", "7OD", "5b6", "4gv", "1Ly", "0I9", "QF", "0rt", "4IZ", "68q", +"67p", "5Vz", "1mT", "1x5", "4c", "0SY", "4hw", "5m7", "6Pd", "4eG", "1NH", "9S", "Sw", "04M", "4Kk", "7ni", "4Ui", "74c", "Mu", "0nG", +"20", "cY", "6Nf", "5kd", "4vu", "5s5", "ni", "390", "0cw", "1f7", "4M8", "4XY", "4WX", "4B9", "OD", "0lv", "0BZ", "ah", "6LW", "4yt", +"40L", "6Ag", "lX", "y9", "0aF", "Bt", "6oK", "4Zh", "1Mc", "2ha", "6SO", "4fl", "5Xa", "69k", "2FM", "07f", "83N", "d5F", "6pS", "4Ep", +"bQo", "a0e", "7H", "0Pr", "1OR", "8I", "454", "50t", "4Jq", "5O1", "Rm", "05W", "08g", "2IL", "66j", "4GA", "4im", "7Lo", "5y", "0RC", +"os", "0LA", "43g", "6BL", "78I", "4YC", "0bm", "2WN", "Lo", "8fE", "4Ts", "5Q3", "aen", "cPM", "0Aq", "bC", "mB", "0Np", "41V", "ajo", +"6nQ", "5KS", "14u", "Cn", "2XO", "0ml", "4VB", "6ca", "6MM", "4xn", "1Sa", "22k", "Sj", "04P", "4Kv", "5N6", "443", "4eZ", "1NU", "9N", +"pv", "0SD", "4hj", "7Mh", "67m", "4FF", "1mI", "2HK", "2GJ", "2", "4IG", "68l", "6RH", "4gk", "1Ld", "2if", "6O", "0Qu", "5zz", "7OY", +"5A7", "4Dw", "1ox", "0j8", "15r", "Bi", "6oV", "4Zu", "40Q", "4a8", "lE", "0Ow", "0BG", "au", "6LJ", "4yi", "4WE", "6bf", "OY", "Z8", +"U9", "2VI", "6mg", "4XD", "4vh", "6CK", "nt", "0MF", "1PW", "cD", "4n9", "5ky", "4Ut", "5P4", "Mh", "0nZ", "4ip", "5l0", "5d", "9Kg", +"08z", "1y2", "66w", "737", "4Jl", "7on", "Rp", "05J", "1OO", "8T", "6Qc", "50i", "4kA", "7NC", "7U", "0Po", "1nb", "dqS", "64F", "4Em", +"b6f", "69v", "PA", "0ss", "8TG", "dRO", "5c1", "4fq", "6MP", "4xs", "376", "22v", "NC", "0mq", "bll", "77U", "6nL", "5KN", "14h", "Cs", +"3ko", "0Nm", "41K", "7PA", "6Oa", "4zB", "37", "20G", "Lr", "8fX", "4Tn", "6aM", "78T", "bcm", "0bp", "AB", "on", "387", "43z", "5r2", +"447", "51w", "1NQ", "9J", "Sn", "04T", "4Kr", "5N2", "67i", "4FB", "09d", "2HO", "4z", "1Ca", "4hn", "7Ml", "6RL", "4go", "8UY", "2ib", +"2GN", "6", "4IC", "68h", "5A3", "4Ds", "82M", "d4E", "6K", "0Qq", "bPl", "a1f", "40U", "akl", "lA", "0Os", "15v", "Bm", "6oR", "4Zq", +"4WA", "6bb", "2YL", "0lo", "0BC", "aq", "6LN", "4ym", "42d", "6CO", "np", "0MB", "0cn", "2VM", "6mc", "5Ha", "4Up", "5P0", "Ml", "8gF", +"1PS", "1E2", "adm", "bAo", "1lW", "1y6", "4R9", "4GX", "4it", "5l4", "qh", "0RZ", "i9", "8P", "6Qg", "4dD", "4Jh", "7oj", "Rt", "05N", +"1nf", "2Kd", "64B", "4Ei", "4kE", "7NG", "7Q", "f8", "1Mz", "2hx", "5c5", "4fu", "4HY", "69r", "PE", "0sw", "NG", "0mu", "5Fz", "6cx", +"6MT", "4xw", "0CY", "0V8", "3kk", "0Ni", "41O", "7PE", "6nH", "5KJ", "14l", "Cw", "Lv", "0oD", "4Tj", "6aI", "6Oe", "4zF", "33", "bZ", +"oj", "0LX", "4wv", "5r6", "6ly", "4YZ", "0bt", "AF", "4v", "0SL", "4hb", "awS", "67e", "4FN", "K3", "2HC", "Sb", "04X", "b5E", "aTO", +"4p3", "4eR", "8Wd", "9F", "6G", "257", "4jS", "7OQ", "65T", "cnm", "1op", "0j0", "QS", "D2", "4IO", "68d", "7Ba", "4gc", "1Ll", "2in", +"0BO", "23d", "6LB", "4ya", "4WM", "6bn", "OQ", "Z0", "0aS", "Ba", "aEL", "c4g", "40Y", "4a0", "lM", "8Fg", "8If", "cL", "4n1", "5kq", +"616", "74v", "3KP", "0nR", "U1", "2VA", "6mo", "4XL", "42h", "6CC", "2xm", "0MN", "4Jd", "7of", "Rx", "05B", "i5", "2jE", "6Qk", "4dH", +"4ix", "5l8", "5l", "0RV", "08r", "2IY", "4R5", "4GT", "4HU", "7mW", "PI", "07s", "1Mv", "0H6", "5c9", "4fy", "4kI", "7NK", "sU", "f4", +"1nj", "2Kh", "64N", "4Ee", "6nD", "5KF", "1ph", "2Uj", "mW", "x6", "41C", "7PI", "593", "5hZ", "0CU", "0V4", "NK", "0my", "4VW", "4C6", +"4L7", "4YV", "0bx", "AJ", "of", "0LT", "43r", "562", "6Oi", "4zJ", "w7", "bV", "Lz", "0oH", "4Tf", "6aE", "67a", "4FJ", "K7", "2HG", +"4r", "0SH", "4hf", "7Md", "4p7", "4eV", "1NY", "9B", "Sf", "0pT", "4Kz", "7nx", "65P", "5TZ", "1ot", "0j4", "6C", "0Qy", "4jW", "7OU", +"6RD", "4gg", "1Lh", "2ij", "QW", "D6", "4IK", "7lI", "4WI", "6bj", "OU", "Z4", "0BK", "ay", "6LF", "4ye", "4tU", "4a4", "lI", "2o9", +"0aW", "Be", "6oZ", "4Zy", "4Ux", "5P8", "Md", "0nV", "8Ib", "cH", "4n5", "5ku", "42l", "6CG", "nx", "0MJ", "U5", "2VE", "6mk", "4XH", +"i1", "8X", "6Qo", "4dL", "5ZA", "7ob", "2Dm", "05F", "08v", "d7f", "4R1", "4GP", "bSO", "a2E", "5h", "0RR", "1Mr", "0H2", "ayL", "52T", +"4HQ", "69z", "PM", "07w", "1nn", "2Kl", "64J", "4Ea", "4kM", "7NO", "7Y", "f0", "mS", "x2", "41G", "7PM", "aDR", "5KB", "14d", "2Un", +"NO", "19T", "4VS", "4C2", "597", "bBL", "0CQ", "0V0", "ob", "0LP", "43v", "566", "4L3", "4YR", "16U", "AN", "2Zo", "0oL", "4Tb", "6aA", +"6Om", "4zN", "w3", "bR", "4oT", "4z5", "wH", "0Tz", "0zV", "Yd", "5D8", "4Ax", "4LH", "6yk", "TT", "A5", "0YJ", "zx", "6WG", "4bd", +"4me", "6XF", "1q", "0VK", "N4", "2MD", "62b", "4CI", "4Ny", "5K9", "Ve", "0uW", "1KZ", "xI", "4u4", "5pt", "4k6", "5nv", "0Ey", "fK", +"Hg", "0kU", "641", "6eX", "6hh", "5Mj", "P6", "EW", "29b", "0HI", "47o", "6FD", "6IE", "48n", "0GH", "dz", "JV", "0id", "4RJ", "6gi", +"6jY", "beI", "0dT", "Gf", "iJ", "0Jx", "4qV", "4d7", "UN", "02t", "4MR", "4X3", "a8G", "57W", "0XP", "0M1", "2Z", "c3", "4nN", "7KL", +"61I", "5PC", "1km", "2No", "2An", "00E", "4Oc", "7ja", "6Tl", "4aO", "l2", "yS", "0k", "0WQ", "58V", "a7F", "4W2", "4BS", "84m", "ZO", +"13V", "E", "4I0", "5Lp", "46u", "535", "ja", "0IS", "68", "gQ", "6Jn", "5ol", "4Qa", "6dB", "3OM", "0jO", "0eN", "2Pm", "6kC", "5NA", +"44D", "6Eo", "hP", "99", "0FR", "0S3", "abM", "49t", "4SP", "4F1", "KL", "8af", "0zR", "0o3", "60W", "ckn", "4oP", "4z1", "3D", "204", +"0YN", "2lm", "6WC", "56I", "4LL", "6yo", "TP", "A1", "N0", "903", "62f", "4CM", "4ma", "6XB", "1u", "0VO", "8Rg", "xM", "418", "54x", +"b0F", "aQL", "Va", "0uS", "Hc", "0kQ", "645", "71u", "4k2", "5nr", "8Le", "fO", "29f", "0HM", "47k", "7Va", "6hl", "5Mn", "P2", "ES", +"JR", "1yA", "4RN", "6gm", "6IA", "48j", "0GL", "26g", "iN", "8Cd", "45Z", "4d3", "hYv", "beM", "0dP", "Gb", "6VY", "4cz", "0XT", "0M5", +"UJ", "02p", "4MV", "4X7", "61M", "5PG", "1ki", "Xz", "vV", "c7", "4nJ", "7KH", "6Th", "4aK", "l6", "yW", "2Aj", "00A", "4Og", "6zD", +"4W6", "4BW", "0yy", "ZK", "0o", "0WU", "58R", "6YX", "46q", "531", "je", "0IW", "13R", "A", "4I4", "5Lt", "4Qe", "6dF", "Iy", "0jK", +"r4", "gU", "6Jj", "5oh", "4pH", "6Ek", "hT", "0Kf", "0eJ", "Fx", "6kG", "5NE", "4ST", "4F5", "KH", "0hz", "0FV", "ed", "5x8", "49p", +"bvs", "6yc", "2BM", "03f", "0YB", "zp", "6WO", "4bl", "bUo", "a4e", "3H", "0Tr", "87N", "Yl", "5D0", "4Ap", "4Nq", "5K1", "Vm", "01W", +"1KR", "xA", "414", "54t", "4mm", "6XN", "1y", "0VC", "0xo", "2ML", "62j", "4CA", "7xA", "5Mb", "0fm", "2SN", "ks", "0HA", "47g", "6FL", +"aan", "bDl", "0Eq", "fC", "Ho", "8bE", "4Ps", "5U3", "5Z2", "5OS", "10u", "Gn", "iB", "0Jp", "45V", "ano", "6IM", "48f", "1Wa", "dr", +"3Ln", "0il", "4RB", "6ga", "2R", "0Uh", "4nF", "7KD", "61A", "5PK", "1ke", "Xv", "UF", "0vt", "4MZ", "6xy", "5f6", "4cv", "0XX", "0M9", +"0c", "0WY", "4lw", "5i7", "63p", "5Rz", "0yu", "ZG", "Ww", "00M", "4Ok", "6zH", "6Td", "4aG", "0Zi", "2oJ", "60", "gY", "6Jf", "5od", +"4Qi", "6dJ", "Iu", "0jG", "0gw", "M", "4I8", "5Lx", "4ru", "5w5", "ji", "1Yz", "0FZ", "eh", "5x4", "5mU", "4SX", "4F9", "KD", "0hv", +"0eF", "Ft", "6kK", "5NI", "44L", "6Eg", "hX", "91", "0YF", "zt", "6WK", "4bh", "4LD", "6yg", "TX", "A9", "0zZ", "Yh", "5D4", "4At", +"4oX", "4z9", "3L", "0Tv", "1KV", "xE", "410", "54p", "4Nu", "5K5", "Vi", "01S", "N8", "2MH", "62n", "4CE", "4mi", "6XJ", "uu", "0VG", +"kw", "0HE", "47c", "6FH", "6hd", "5Mf", "0fi", "2SJ", "Hk", "0kY", "4Pw", "5U7", "6Kx", "5nz", "0Eu", "fG", "iF", "0Jt", "45R", "6Dy", +"5Z6", "5OW", "0dX", "Gj", "JZ", "0ih", "4RF", "6ge", "6II", "48b", "0GD", "dv", "61E", "5PO", "1ka", "Xr", "2V", "0Ul", "4nB", "aqs", +"5f2", "4cr", "8QD", "39V", "UB", "02x", "795", "aRo", "63t", "764", "0yq", "ZC", "0g", "9Nd", "4ls", "5i3", "7DA", "4aC", "0Zm", "2oN", +"Ws", "00I", "4Oo", "6zL", "4Qm", "6dN", "Iq", "0jC", "64", "25D", "6Jb", "bEr", "46y", "539", "jm", "9Pf", "0gs", "I", "aCl", "bfn", +"bio", "72V", "1m2", "0hr", "325", "el", "5x0", "49x", "44H", "6Ec", "3nl", "95", "0eB", "Fp", "6kO", "5NM", "4mt", "5h4", "uh", "0VZ", +"0xv", "2MU", "4V9", "4CX", "4Nh", "7kj", "Vt", "01N", "m9", "xX", "6Ug", "54m", "4oE", "6Zf", "3Q", "b8", "0zG", "Yu", "60B", "4Ai", +"4LY", "4Y8", "TE", "0ww", "1Iz", "zi", "5g5", "4bu", "5y7", "5lV", "0GY", "dk", "JG", "0iu", "5Bz", "6gx", "6jH", "5OJ", "0dE", "Gw", +"3ok", "82", "45O", "6Dd", "6Ke", "5ng", "73", "fZ", "Hv", "0kD", "4Pj", "6eI", "6hy", "7m9", "0ft", "EF", "kj", "0HX", "4sv", "5v6", +"Wn", "00T", "4Or", "5J2", "407", "55w", "0Zp", "yB", "0z", "1Ga", "4ln", "6YM", "63i", "4BB", "0yl", "2LO", "2CN", "02e", "4MC", "7hA", +"6VL", "4co", "0XA", "2mb", "2K", "0Uq", "bTl", "a5f", "5E3", "5PR", "86M", "Xo", "11v", "Fm", "6kR", "5NP", "44U", "aol", "hA", "0Ks", +"0FC", "eq", "6HN", "49e", "4SA", "6fb", "3Mm", "0ho", "0gn", "T", "6ic", "5La", "46d", "6GO", "jp", "0IB", "0Dr", "1A2", "hyT", "bEo", +"4Qp", "5T0", "Il", "8cF", "0xr", "2MQ", "62w", "777", "4mp", "5h0", "1d", "9Og", "1KO", "2nM", "6Uc", "54i", "4Nl", "7kn", "Vp", "01J", +"0zC", "Yq", "60F", "4Am", "4oA", "6Zb", "3U", "0To", "8PG", "zm", "5g1", "4bq", "786", "aSl", "TA", "0ws", "JC", "0iq", "bhl", "73U", +"5y3", "5lR", "336", "do", "3oo", "86", "45K", "7TA", "6jL", "5ON", "0dA", "Gs", "Hr", "8bX", "4Pn", "6eM", "6Ka", "5nc", "77", "24G", +"kn", "8AD", "47z", "5v2", "aBo", "bgm", "0fp", "EB", "403", "4aZ", "0Zt", "yF", "Wj", "00P", "4Ov", "5J6", "63m", "4BF", "0yh", "ZZ", +"tv", "0WD", "4lj", "6YI", "6VH", "4ck", "0XE", "2mf", "2CJ", "02a", "4MG", "6xd", "5E7", "5PV", "1kx", "Xk", "2O", "0Uu", "bTh", "7KY", +"44Q", "4e8", "hE", "0Kw", "11r", "Fi", "6kV", "5NT", "4SE", "6ff", "KY", "0hk", "0FG", "eu", "6HJ", "49a", "4rh", "6GK", "jt", "0IF", +"Q9", "P", "6ig", "5Le", "4Qt", "5T4", "Ih", "0jZ", "0Dv", "gD", "4j9", "5oy", "aD0", "7kb", "3PL", "01F", "m1", "xP", "6Uo", "54e", +"59U", "a6E", "1h", "0VR", "85n", "196", "4V1", "4CP", "4LQ", "4Y0", "TM", "03w", "0YS", "za", "a9D", "56T", "4oM", "6Zn", "3Y", "b0", +"0zO", "2Ol", "60J", "4Aa", "7za", "5OB", "0dM", "2Qn", "iS", "0Ja", "45G", "6Dl", "acN", "48w", "0GQ", "dc", "JO", "94L", "4RS", "4G2", +"4H3", "5Ms", "12U", "EN", "kb", "0HP", "47v", "526", "6Km", "5no", "s3", "fR", "3NN", "0kL", "4Pb", "6eA", "0r", "0WH", "4lf", "6YE", +"63a", "4BJ", "O7", "ZV", "Wf", "0tT", "4Oz", "6zY", "4t7", "4aV", "0Zx", "yJ", "2C", "0Uy", "4nW", "7KU", "61P", "5PZ", "1kt", "Xg", +"UW", "02m", "4MK", "6xh", "6VD", "4cg", "0XI", "2mj", "0FK", "ey", "6HF", "49m", "4SI", "6fj", "KU", "0hg", "0eW", "Fe", "6kZ", "5NX", +"4pU", "4e4", "hI", "2k9", "0Dz", "gH", "4j5", "5ou", "4Qx", "5T8", "Id", "0jV", "Q5", "DT", "6ik", "5Li", "46l", "6GG", "jx", "0IJ", +"m5", "xT", "6Uk", "54a", "4Nd", "7kf", "Vx", "01B", "0xz", "192", "4V5", "4CT", "4mx", "5h8", "1l", "0VV", "0YW", "ze", "5g9", "4by", +"4LU", "4Y4", "TI", "03s", "0zK", "Yy", "60N", "4Ae", "4oI", "6Zj", "wU", "b4", "iW", "0Je", "45C", "6Dh", "6jD", "5OF", "0dI", "2Qj", +"JK", "0iy", "4RW", "4G6", "6IX", "48s", "0GU", "dg", "kf", "0HT", "47r", "522", "4H7", "5Mw", "0fx", "EJ", "Hz", "0kH", "4Pf", "6eE", +"6Ki", "5nk", "s7", "fV", "63e", "4BN", "O3", "ZR", "0v", "0WL", "4lb", "6YA", "4t3", "4aR", "8Sd", "yN", "Wb", "00X", "b1E", "aPO", +"61T", "bzL", "1kp", "Xc", "2G", "217", "4nS", "7KQ", "7Fa", "4cc", "0XM", "2mn", "US", "02i", "4MO", "6xl", "4SM", "6fn", "KQ", "0hc", +"0FO", "27d", "6HB", "49i", "44Y", "4e0", "hM", "8Bg", "0eS", "Fa", "aAL", "bdN", "656", "70v", "3OP", "0jR", "8Mf", "gL", "4j1", "5oq", +"46h", "6GC", "28e", "0IN", "Q1", "X", "6io", "5Lm", "6KV", "5nT", "1Uz", "fi", "HE", "0kw", "4PY", "4E8", "6hJ", "5MH", "0fG", "Eu", +"kY", "0Hk", "47M", "6Ff", "6Ig", "48L", "51", "dX", "Jt", "0iF", "4Rh", "6gK", "4J9", "5Oy", "0dv", "GD", "ih", "0JZ", "4qt", "5t4", +"4ov", "5j6", "3b", "0TX", "0zt", "YF", "60q", "4AZ", "4Lj", "6yI", "Tv", "03L", "0Yh", "zZ", "6We", "4bF", "4mG", "6Xd", "1S", "0Vi", +"0xE", "2Mf", "6vH", "4Ck", "bth", "7kY", "VG", "0uu", "1Kx", "xk", "5e7", "5pV", "13t", "g", "5Y3", "5LR", "46W", "amn", "jC", "0Iq", +"0DA", "gs", "6JL", "5oN", "4QC", "70I", "3Oo", "0jm", "0el", "2PO", "6ka", "5Nc", "44f", "6EM", "hr", "8BX", "0Fp", "eB", "abo", "49V", +"4Sr", "5V2", "Kn", "8aD", "Ul", "02V", "4Mp", "5H0", "425", "57u", "0Xr", "2mQ", "2x", "0UB", "4nl", "7Kn", "61k", "5Pa", "1kO", "2NM", +"2AL", "00g", "4OA", "6zb", "6TN", "4am", "0ZC", "yq", "0I", "0Ws", "58t", "a7d", "5G1", "4Bq", "84O", "Zm", "HA", "0ks", "bjn", "71W", +"6KR", "5nP", "314", "fm", "29D", "0Ho", "47I", "6Fb", "6hN", "5ML", "0fC", "Eq", "Jp", "0iB", "4Rl", "6gO", "6Ic", "48H", "55", "26E", +"il", "8CF", "45x", "508", "hYT", "beo", "0dr", "1a2", "0zp", "YB", "60u", "755", "4or", "5j2", "3f", "9Me", "0Yl", "2lO", "6Wa", "4bB", +"4Ln", "6yM", "Tr", "03H", "0xA", "2Mb", "62D", "4Co", "4mC", "7HA", "1W", "0Vm", "8RE", "xo", "5e3", "54Z", "b0d", "aQn", "VC", "01y", +"46S", "6Gx", "jG", "0Iu", "0gY", "c", "5Y7", "5LV", "4QG", "6dd", "3Ok", "0ji", "0DE", "gw", "6JH", "5oJ", "44b", "6EI", "hv", "0KD", +"0eh", "FZ", "6ke", "5Ng", "4Sv", "5V6", "Kj", "0hX", "0Ft", "eF", "6Hy", "49R", "421", "4cX", "0Xv", "2mU", "Uh", "02R", "4Mt", "5H4", +"61o", "5Pe", "M9", "XX", "vt", "0UF", "4nh", "7Kj", "6TJ", "4ai", "0ZG", "yu", "WY", "B8", "4OE", "6zf", "5G5", "4Bu", "1iz", "Zi", +"0M", "0Ww", "4lY", "4y8", "6hB", "aW1", "0fO", "2Sl", "kQ", "0Hc", "47E", "6Fn", "aaL", "bDN", "0ES", "fa", "HM", "8bg", "4PQ", "4E0", +"4J1", "5Oq", "10W", "GL", "3oP", "0JR", "45t", "504", "6Io", "48D", "59", "dP", "3LL", "0iN", "5BA", "6gC", "4Lb", "6yA", "2Bo", "03D", +"o3", "zR", "6Wm", "4bN", "bUM", "a4G", "3j", "0TP", "87l", "YN", "4T3", "4AR", "4NS", "7kQ", "VO", "01u", "1Kp", "xc", "hfw", "54V", +"4mO", "6Xl", "uS", "0Va", "0xM", "2Mn", "62H", "4Cc", "0DI", "25b", "6JD", "5oF", "4QK", "6dh", "IW", "0je", "0gU", "o", "6iX", "5LZ", +"4rW", "4g6", "jK", "0Iy", "0Fx", "eJ", "4h7", "5mw", "4Sz", "6fY", "Kf", "0hT", "S7", "FV", "6ki", "5Nk", "44n", "6EE", "hz", "0KH", +"2p", "0UJ", "4nd", "7Kf", "61c", "5Pi", "M5", "XT", "Ud", "0vV", "4Mx", "5H8", "4v5", "4cT", "0Xz", "2mY", "0A", "1GZ", "4lU", "4y4", +"5G9", "4By", "0yW", "Ze", "WU", "B4", "4OI", "6zj", "6TF", "4ae", "0ZK", "yy", "kU", "0Hg", "47A", "6Fj", "6hF", "5MD", "0fK", "Ey", +"HI", "2K9", "4PU", "4E4", "6KZ", "5nX", "0EW", "fe", "id", "0JV", "45p", "500", "4J5", "5Ou", "0dz", "GH", "Jx", "0iJ", "4Rd", "6gG", +"6Ik", "5li", "q5", "dT", "o7", "zV", "6Wi", "4bJ", "4Lf", "6yE", "Tz", "0wH", "0zx", "YJ", "4T7", "4AV", "4oz", "6ZY", "3n", "0TT", +"1Kt", "xg", "6UX", "54R", "4NW", "7kU", "VK", "01q", "0xI", "2Mj", "62L", "4Cg", "4mK", "6Xh", "uW", "0Ve", "4QO", "6dl", "IS", "0ja", +"0DM", "25f", "7Za", "5oB", "4rS", "4g2", "jO", "9PD", "0gQ", "k", "aCN", "685", "674", "72t", "Kb", "0hP", "8Od", "eN", "4h3", "49Z", +"44j", "6EA", "3nN", "0KL", "S3", "FR", "6km", "5No", "61g", "5Pm", "M1", "XP", "2t", "0UN", "ad0", "7Kb", "429", "4cP", "8Qf", "39t", +"0c3", "02Z", "b3G", "aRM", "63V", "bxN", "0yS", "Za", "0E", "235", "4lQ", "4y0", "6TB", "4aa", "0ZO", "2ol", "WQ", "B0", "4OM", "6zn", +"4i4", "5lt", "1WZ", "dI", "Je", "0iW", "4Ry", "5W9", "6jj", "5Oh", "R4", "GU", "iy", "0JK", "45m", "6DF", "6KG", "5nE", "0EJ", "fx", +"HT", "0kf", "4PH", "6ek", "5X8", "5MY", "0fV", "Ed", "kH", "0Hz", "4sT", "4f5", "4mV", "4x7", "1B", "0Vx", "0xT", "0m5", "62Q", "4Cz", +"4NJ", "7kH", "VV", "C7", "1Ki", "xz", "6UE", "54O", "4og", "6ZD", "3s", "0TI", "L6", "YW", "6th", "4AK", "6l9", "6yX", "Tg", "0wU", +"0Yy", "zK", "4w6", "4bW", "11T", "FO", "4K2", "5Nr", "44w", "517", "hc", "0KQ", "p2", "eS", "6Hl", "49G", "4Sc", "72i", "3MO", "0hM", +"0gL", "v", "6iA", "5LC", "46F", "6Gm", "jR", "1YA", "0DP", "gb", "hyv", "bEM", "4QR", "4D3", "IN", "8cd", "WL", "00v", "4OP", "4Z1", +"hgt", "55U", "0ZR", "0O3", "0X", "a1", "4lL", "6Yo", "63K", "5RA", "0yN", "2Lm", "2Cl", "02G", "4Ma", "6xB", "6Vn", "4cM", "n0", "39i", +"2i", "0US", "bTN", "a5D", "4U0", "5Pp", "86o", "XM", "Ja", "0iS", "667", "73w", "4i0", "48Y", "8Ng", "dM", "3oM", "0JO", "45i", "6DB", +"6jn", "5Ol", "R0", "GQ", "HP", "0kb", "4PL", "6eo", "6KC", "5nA", "0EN", "24e", "kL", "8Af", "47X", "4f1", "aBM", "696", "0fR", "0s3", +"0xP", "0m1", "62U", "byM", "4mR", "4x3", "1F", "226", "1Km", "2no", "6UA", "54K", "4NN", "7kL", "VR", "C3", "L2", "YS", "60d", "4AO", +"4oc", "7Ja", "3w", "0TM", "8Pe", "zO", "4w2", "4bS", "b2D", "aSN", "Tc", "03Y", "44s", "513", "hg", "0KU", "0ey", "FK", "4K6", "5Nv", +"4Sg", "6fD", "3MK", "0hI", "p6", "eW", "6Hh", "49C", "46B", "6Gi", "jV", "0Id", "0gH", "r", "6iE", "5LG", "4QV", "4D7", "IJ", "0jx", +"0DT", "gf", "6JY", "bEI", "5d8", "4ax", "0ZV", "yd", "WH", "00r", "4OT", "4Z5", "63O", "4Bd", "0yJ", "Zx", "tT", "a5", "4lH", "6Yk", +"6Vj", "4cI", "n4", "2mD", "Uy", "02C", "4Me", "6xF", "4U4", "5Pt", "1kZ", "XI", "2m", "0UW", "4ny", "5k9", "6jb", "ber", "0do", "2QL", +"iq", "0JC", "45e", "6DN", "acl", "48U", "0Gs", "dA", "Jm", "94n", "4Rq", "5W1", "5X0", "5MQ", "12w", "El", "1M2", "0Hr", "47T", "alm", +"6KO", "5nM", "0EB", "fp", "3Nl", "0kn", "bjs", "6ec", "4NB", "aQs", "3Pn", "01d", "1Ka", "xr", "6UM", "54G", "59w", "a6g", "1J", "0Vp", +"85L", "d3D", "5F2", "4Cr", "4Ls", "5I3", "To", "03U", "0Yq", "zC", "436", "56v", "4oo", "6ZL", "ws", "0TA", "0zm", "2ON", "60h", "4AC", +"42", "27B", "6Hd", "49O", "4Sk", "6fH", "Kw", "0hE", "0eu", "FG", "6kx", "5Nz", "4pw", "5u7", "hk", "0KY", "0DX", "gj", "5z6", "5oW", +"4QZ", "6dy", "IF", "0jt", "0gD", "Dv", "6iI", "5LK", "46N", "6Ge", "jZ", "0Ih", "0P", "a9", "4lD", "6Yg", "63C", "4Bh", "0yF", "Zt", +"WD", "0tv", "4OX", "4Z9", "5d4", "4at", "0ZZ", "yh", "2a", "1Ez", "4nu", "5k5", "4U8", "5Px", "1kV", "XE", "Uu", "02O", "4Mi", "6xJ", +"6Vf", "4cE", "n8", "2mH", "iu", "0JG", "45a", "6DJ", "6jf", "5Od", "R8", "GY", "Ji", "1yz", "4Ru", "5W5", "4i8", "48Q", "0Gw", "dE", +"kD", "0Hv", "47P", "4f9", "5X4", "5MU", "0fZ", "Eh", "HX", "0kj", "4PD", "6eg", "6KK", "5nI", "0EF", "ft", "1Ke", "xv", "6UI", "54C", +"4NF", "7kD", "VZ", "0uh", "0xX", "0m9", "5F6", "4Cv", "4mZ", "6Xy", "1N", "0Vt", "0Yu", "zG", "432", "56r", "4Lw", "5I7", "Tk", "03Q", +"0zi", "2OJ", "60l", "4AG", "4ok", "6ZH", "ww", "0TE", "4So", "6fL", "Ks", "0hA", "46", "27F", "7XA", "49K", "4ps", "5u3", "ho", "8BE", +"0eq", "FC", "aAn", "bdl", "bkm", "70T", "IB", "0jp", "307", "gn", "5z2", "5oS", "46J", "6Ga", "28G", "0Il", "13i", "z", "6iM", "5LO", +"63G", "4Bl", "0yB", "Zp", "0T", "0Wn", "58i", "6Yc", "5d0", "4ap", "8SF", "yl", "1q2", "00z", "b1g", "aPm", "61v", "746", "1kR", "XA", +"2e", "9Lf", "4nq", "5k1", "6Vb", "4cA", "0Xo", "2mL", "Uq", "02K", "4Mm", "6xN", "8YG", "7e", "5n1", "4kq", "716", "64v", "2KP", "1nR", +"07K", "Pq", "69F", "4Hm", "4fA", "6Sb", "2hL", "1MN", "0Rn", "5T", "7LB", "5ya", "4Gl", "66G", "2Ia", "08J", "05z", "1t2", "aUm", "b4g", +"4dp", "5a0", "8d", "8VF", "bn", "357", "4zr", "6OQ", "75T", "bnm", "0op", "LB", "Ar", "16i", "4Yn", "6lM", "6Ba", "43J", "0Ll", "2yO", +"22F", "16", "4xC", "agr", "6cL", "4Vo", "0mA", "Ns", "CC", "14X", "bal", "aDn", "5p3", "4us", "8GE", "mo", "5L7", "4Iw", "06Q", "Qk", +"1Y5", "1LT", "53r", "462", "7Oi", "4jk", "0QE", "rw", "2JJ", "1oH", "4DG", "65l", "7nD", "4KF", "0ph", "SZ", "2kg", "1Ne", "4ej", "6PI", +"493", "4hZ", "0St", "4N", "0h9", "09P", "4Fv", "5C6", "4Xt", "6mW", "023", "0cZ", "0Mv", "nD", "4c9", "42P", "5kI", "6NK", "ct", "1Pg", +"X9", "MX", "74N", "4UD", "4ZE", "6of", "BY", "W8", "0OG", "lu", "6AJ", "40a", "4yY", "4l8", "aE", "0Bw", "18r", "Oi", "5R5", "4Wu", +"4EY", "4P8", "2KT", "1nV", "8YC", "7a", "5n5", "4ku", "4fE", "6Sf", "2hH", "k8", "07O", "Pu", "69B", "4Hi", "4Gh", "66C", "2Ie", "08N", +"d9", "5P", "7LF", "4iD", "4dt", "5a4", "2jy", "3o9", "0qv", "RD", "7oZ", "4JX", "6ay", "4TZ", "0ot", "LF", "bj", "0AX", "4zv", "6OU", +"6Be", "43N", "0Lh", "oZ", "Av", "0bD", "4Yj", "6lI", "6cH", "4Vk", "0mE", "Nw", "22B", "12", "4xG", "6Md", "5p7", "4uw", "0NY", "mk", +"CG", "1pT", "5Kz", "6nx", "1Y1", "1LP", "53v", "466", "5L3", "4Is", "06U", "Qo", "2JN", "1oL", "4DC", "65h", "7Om", "4jo", "0QA", "rs", +"9z", "1Na", "4en", "6PM", "aTs", "4KB", "04d", "2EO", "d6D", "09T", "4Fr", "5C2", "497", "bRm", "0Sp", "4J", "0Mr", "1H2", "aim", "42T", +"4Xp", "6mS", "027", "17w", "0nn", "3Kl", "74J", "5Ea", "5kM", "6NO", "cp", "1Pc", "0OC", "lq", "6AN", "40e", "4ZA", "6ob", "2TL", "0ao", +"18v", "Om", "5R1", "4Wq", "bCn", "afl", "aA", "0Bs", "07C", "Py", "69N", "4He", "4fI", "6Sj", "2hD", "k4", "0PW", "7m", "5n9", "4ky", +"4EU", "4P4", "2KX", "1nZ", "05r", "RH", "7oV", "4JT", "4dx", "5a8", "8l", "1Ow", "d5", "qT", "7LJ", "4iH", "4Gd", "66O", "2Ii", "08B", +"Az", "0bH", "4Yf", "6lE", "6Bi", "43B", "z7", "oV", "bf", "0AT", "4zz", "6OY", "4A7", "4TV", "0ox", "LJ", "CK", "14P", "5Kv", "4N6", +"543", "41s", "0NU", "mg", "22N", "u6", "4xK", "6Mh", "6cD", "4Vg", "0mI", "2Xj", "7Oa", "4jc", "0QM", "6w", "2JB", "I2", "4DO", "65d", +"68T", "b7D", "06Y", "Qc", "dSm", "287", "4gS", "4r2", "7MP", "4hR", "276", "4F", "0h1", "09X", "b8E", "67U", "7nL", "4KN", "F3", "SR", +"9v", "1Nm", "4eb", "6PA", "5kA", "6NC", "21e", "1Po", "X1", "MP", "74F", "4UL", "bbO", "79v", "0v3", "0cR", "8Df", "nL", "4c1", "42X", +"4yQ", "4l0", "aM", "8Kg", "0lS", "Oa", "76w", "637", "4ZM", "6on", "BQ", "W0", "0OO", "2zl", "6AB", "40i", "4fM", "6Sn", "3xa", "k0", +"07G", "2Fl", "69J", "4Ha", "4EQ", "4P0", "d5g", "83o", "0PS", "7i", "a0D", "bQN", "50U", "hbt", "8h", "1Os", "05v", "RL", "7oR", "4JP", +"5WA", "66K", "2Im", "08F", "d1", "5X", "7LN", "4iL", "6Bm", "43F", "z3", "oR", "2Wo", "0bL", "4Yb", "6lA", "4A3", "4TR", "8fd", "LN", +"bb", "0AP", "cPl", "aeO", "547", "41w", "0NQ", "mc", "CO", "14T", "5Kr", "4N2", "77i", "4Vc", "0mM", "2Xn", "22J", "u2", "4xO", "6Ml", +"2JF", "I6", "4DK", "6qh", "7Oe", "4jg", "0QI", "6s", "1Y9", "1LX", "4gW", "4r6", "68P", "5YZ", "0rU", "Qg", "0h5", "1mu", "4Fz", "67Q", +"7MT", "4hV", "0Sx", "4B", "9r", "1Ni", "4ef", "6PE", "7nH", "4KJ", "F7", "SV", "X5", "MT", "74B", "4UH", "5kE", "6NG", "cx", "1Pk", +"0Mz", "nH", "4c5", "4vT", "4Xx", "79r", "0v7", "0cV", "0lW", "Oe", "5R9", "4Wy", "4yU", "4l4", "aI", "1RZ", "0OK", "ly", "6AF", "40m", +"4ZI", "6oj", "BU", "W4", "265", "5E", "488", "4iQ", "b9F", "66V", "0i2", "1lr", "G0", "RQ", "7oO", "4JM", "4da", "6QB", "8u", "1On", +"0PN", "7t", "7Nb", "aa0", "4EL", "64g", "2KA", "H1", "07Z", "0f3", "69W", "b6G", "4fP", "479", "dRn", "294", "22W", "8Jd", "4xR", "4m3", +"77t", "624", "0mP", "Nb", "CR", "V3", "5Ko", "6nm", "ajS", "41j", "0NL", "3kN", "20f", "0AM", "4zc", "aeR", "6al", "4TO", "Y2", "LS", +"Ac", "0bQ", "bcL", "78u", "4b2", "4wS", "8Ee", "oO", "7nU", "4KW", "04q", "SK", "9o", "1Nt", "51R", "6PX", "7MI", "4hK", "e6", "pW", +"2Hj", "09A", "4Fg", "67L", "68M", "4If", "0rH", "Qz", "2iG", "j7", "4gJ", "6Ri", "7Ox", "4jz", "0QT", "6n", "1z8", "1oY", "4DV", "4Q7", +"4ZT", "4O5", "BH", "0az", "0OV", "ld", "550", "40p", "4yH", "6Lk", "aT", "t5", "0lJ", "Ox", "6bG", "4Wd", "4Xe", "6mF", "2Vh", "0cK", +"0Mg", "nU", "6Cj", "42A", "5kX", "6NZ", "ce", "1Pv", "2N9", "MI", "7pW", "4UU", "4Gy", "5B9", "0i6", "1lv", "1BZ", "5A", "7LW", "4iU", +"4de", "6QF", "8q", "1Oj", "G4", "RU", "7oK", "4JI", "4EH", "64c", "2KE", "H5", "0PJ", "7p", "7Nf", "4kd", "4fT", "4s5", "2hY", "290", +"0sV", "Pd", "5M8", "4Hx", "6cY", "4Vz", "0mT", "Nf", "1F8", "0Cx", "4xV", "4m7", "7Pd", "41n", "0NH", "mz", "CV", "V7", "5Kk", "6ni", +"6ah", "4TK", "Y6", "LW", "20b", "0AI", "4zg", "6OD", "4b6", "4wW", "0Ly", "oK", "Ag", "0bU", "5IZ", "6lX", "9k", "1Np", "51V", "azN", +"7nQ", "4KS", "04u", "SO", "2Hn", "09E", "4Fc", "67H", "7MM", "4hO", "e2", "pS", "2iC", "j3", "4gN", "6Rm", "68I", "4Ib", "06D", "2Go", +"d4d", "82l", "4DR", "4Q3", "a1G", "bPM", "0QP", "6j", "0OR", "0Z3", "554", "40t", "4ZP", "4O1", "BL", "15W", "0lN", "2Ym", "6bC", "5GA", +"4yL", "6Lo", "aP", "09", "0Mc", "nQ", "6Cn", "42E", "4Xa", "6mB", "2Vl", "0cO", "8gg", "MM", "7pS", "4UQ", "bAN", "adL", "ca", "1Pr", +"G8", "RY", "7oG", "4JE", "4di", "6QJ", "2jd", "1Of", "0Rw", "5M", "480", "4iY", "4Gu", "5B5", "2Ix", "08S", "07R", "Ph", "5M4", "4Ht", +"4fX", "471", "1X6", "1MW", "0PF", "st", "7Nj", "4kh", "4ED", "64o", "2KI", "H9", "CZ", "14A", "5Kg", "6ne", "7Ph", "41b", "0ND", "mv", +"1F4", "0Ct", "4xZ", "6My", "5S6", "4Vv", "0mX", "Nj", "Ak", "0bY", "4Yw", "6lT", "6Bx", "43S", "0Lu", "oG", "bw", "0AE", "4zk", "6OH", +"6ad", "4TG", "0oi", "2ZJ", "7MA", "4hC", "0Sm", "4W", "2Hb", "09I", "4Fo", "67D", "aTn", "b5d", "04y", "SC", "9g", "8WE", "4es", "6PP", +"5o2", "4jr", "8XD", "6f", "1z0", "1oQ", "705", "65u", "68E", "4In", "06H", "Qr", "2iO", "1LM", "4gB", "6Ra", "5ia", "6Lc", "23E", "05", +"0lB", "Op", "6bO", "4Wl", "c4F", "aEm", "1d2", "0ar", "8FF", "ll", "558", "40x", "5kP", "6NR", "cm", "344", "0ns", "MA", "74W", "bon", +"4Xm", "6mN", "3FA", "0cC", "0Mo", "2xL", "6Cb", "42I", "4dm", "6QN", "8y", "1Ob", "05g", "2DL", "7oC", "4JA", "4Gq", "5B1", "d7G", "08W", +"0Rs", "5I", "484", "bSn", "52u", "475", "1X2", "1MS", "07V", "Pl", "5M0", "4Hp", "5Ua", "64k", "2KM", "1nO", "0PB", "7x", "7Nn", "4kl", +"7Pl", "41f", "8GX", "mr", "2UO", "14E", "5Kc", "6na", "5S2", "4Vr", "19u", "Nn", "1F0", "0Cp", "bBm", "ago", "ahn", "43W", "0Lq", "oC", +"Ao", "16t", "4Ys", "6lP", "75I", "4TC", "0om", "2ZN", "bs", "0AA", "4zo", "6OL", "2Hf", "09M", "4Fk", "6sH", "7ME", "4hG", "0Si", "4S", +"9c", "1Nx", "4ew", "6PT", "7nY", "bqh", "0pu", "SG", "1z4", "1oU", "4DZ", "65q", "5o6", "4jv", "0QX", "6b", "2iK", "1LI", "4gF", "6Re", +"68A", "4Ij", "06L", "Qv", "0lF", "Ot", "6bK", "4Wh", "4yD", "6Lg", "aX", "01", "0OZ", "lh", "5q4", "4tt", "4ZX", "4O9", "BD", "0av", +"0nw", "ME", "74S", "4UY", "5kT", "6NV", "ci", "1Pz", "0Mk", "nY", "6Cf", "42M", "4Xi", "6mJ", "2Vd", "0cG", "bL", "8Hf", "4zP", "4o1", +"75v", "606", "0oR", "0z3", "AP", "T1", "4YL", "6lo", "6BC", "43h", "0LN", "2ym", "22d", "0CO", "4xa", "6MB", "6cn", "4VM", "0mc", "NQ", +"Ca", "14z", "baN", "aDL", "7PS", "41Y", "8Gg", "mM", "247", "7G", "7NQ", "4kS", "com", "64T", "0k0", "1np", "E2", "PS", "69d", "4HO", +"4fc", "7Ca", "2hn", "1Ml", "0RL", "5v", "avS", "4ib", "4GN", "66e", "2IC", "J3", "05X", "Rb", "aUO", "b4E", "4dR", "4q3", "8F", "8Vd", +"4XV", "4M7", "1f8", "0cx", "0MT", "nf", "572", "42r", "5kk", "6Ni", "cV", "v7", "0nH", "Mz", "74l", "4Uf", "4Zg", "6oD", "2Tj", "0aI", +"y6", "lW", "6Ah", "40C", "5iZ", "583", "ag", "0BU", "0ly", "OK", "4B6", "4WW", "7lW", "4IU", "06s", "QI", "0I6", "1Lv", "4gy", "5b9", +"7OK", "4jI", "g4", "rU", "2Jh", "1oj", "4De", "65N", "7nf", "4Kd", "04B", "Sx", "2kE", "h5", "4eH", "6Pk", "5m8", "4hx", "0SV", "4l", +"2HY", "09r", "4FT", "4S5", "5Q8", "4Tx", "0oV", "Ld", "bH", "0Az", "4zT", "4o5", "6BG", "43l", "0LJ", "ox", "AT", "T5", "4YH", "6lk", +"6cj", "4VI", "0mg", "NU", "2vh", "0CK", "4xe", "6MF", "7PW", "4uU", "2n9", "mI", "Ce", "1pv", "5KX", "6nZ", "5UZ", "64P", "0k4", "1nt", +"0Py", "7C", "7NU", "4kW", "4fg", "6SD", "2hj", "1Mh", "E6", "PW", "7mI", "4HK", "4GJ", "66a", "2IG", "J7", "0RH", "5r", "7Ld", "4if", +"4dV", "4q7", "8B", "1OY", "0qT", "Rf", "7ox", "4Jz", "0MP", "nb", "576", "42v", "4XR", "4M3", "dll", "17U", "0nL", "3KN", "74h", "4Ub", +"5ko", "6Nm", "cR", "v3", "y2", "lS", "6Al", "40G", "4Zc", "aER", "2Tn", "0aM", "18T", "OO", "4B2", "4WS", "bCL", "587", "ac", "0BQ", +"0I2", "1Lr", "53T", "axL", "68z", "4IQ", "06w", "QM", "2Jl", "1on", "4Da", "65J", "7OO", "4jM", "g0", "6Y", "9X", "h1", "4eL", "6Po", +"7nb", "aA0", "04F", "2Em", "d6f", "09v", "4FP", "4S1", "a3E", "bRO", "0SR", "4h", "AX", "T9", "4YD", "6lg", "6BK", "4wh", "0LF", "ot", +"bD", "0Av", "4zX", "4o9", "5Q4", "4Tt", "0oZ", "Lh", "Ci", "14r", "5KT", "6nV", "ajh", "41Q", "0Nw", "mE", "22l", "0CG", "4xi", "6MJ", +"6cf", "4VE", "0mk", "NY", "07a", "2FJ", "69l", "4HG", "4fk", "6SH", "2hf", "1Md", "0Pu", "7O", "7NY", "bQh", "4Ew", "6pT", "0k8", "1nx", +"05P", "Rj", "5O6", "4Jv", "4dZ", "453", "8N", "1OU", "0RD", "qv", "7Lh", "4ij", "4GF", "66m", "2IK", "1lI", "5kc", "6Na", "21G", "27", +"8gX", "Mr", "74d", "4Un", "bbm", "79T", "1f0", "0cp", "397", "nn", "5s2", "42z", "4ys", "6LP", "ao", "366", "0lq", "OC", "76U", "bml", +"4Zo", "6oL", "Bs", "0aA", "0Om", "2zN", "7QA", "40K", "7OC", "4jA", "0Qo", "6U", "3ZA", "1ob", "4Dm", "65F", "68v", "b7f", "0rs", "QA", +"dSO", "8UG", "4gq", "5b1", "5m0", "4hp", "8ZF", "4d", "1x2", "09z", "727", "67w", "7nn", "4Kl", "04J", "Sp", "9T", "1NO", "51i", "6Pc", +"6BO", "43d", "0LB", "op", "2WM", "0bn", "5Ia", "6lc", "5Q0", "4Tp", "8fF", "Ll", "1D2", "0Ar", "cPN", "aem", "ajl", "41U", "0Ns", "mA", +"Cm", "14v", "5KP", "6nR", "6cb", "4VA", "0mo", "2XL", "22h", "0CC", "4xm", "6MN", "4fo", "6SL", "2hb", "8TY", "07e", "2FN", "69h", "4HC", +"4Es", "64X", "d5E", "83M", "0Pq", "7K", "a0f", "bQl", "50w", "457", "8J", "1OQ", "05T", "Rn", "5O2", "4Jr", "4GB", "66i", "2IO", "08d", +"1Ba", "5z", "7Ll", "4in", "0nD", "Mv", "7ph", "4Uj", "5kg", "6Ne", "cZ", "23", "0MX", "nj", "5s6", "4vv", "4XZ", "6my", "1f4", "0ct", +"0lu", "OG", "6bx", "5Gz", "4yw", "6LT", "ak", "0BY", "0Oi", "2zJ", "6Ad", "40O", "4Zk", "6oH", "Bw", "0aE", "2Jd", "1of", "4Di", "65B", +"7OG", "4jE", "g8", "6Q", "2ix", "1Lz", "4gu", "5b5", "68r", "4IY", "0rw", "QE", "1x6", "1mW", "4FX", "4S9", "5m4", "4ht", "0SZ", "ph", +"9P", "h9", "4eD", "6Pg", "7nj", "4Kh", "04N", "St", "22u", "375", "4xp", "598", "77V", "blo", "0mr", "1h2", "Cp", "14k", "5KM", "6nO", +"7PB", "41H", "0Nn", "3kl", "20D", "34", "4zA", "6Ob", "6aN", "4Tm", "0oC", "Lq", "AA", "0bs", "bcn", "78W", "569", "43y", "384", "om", +"9Kd", "5g", "5l3", "4is", "734", "66t", "1y1", "08y", "05I", "Rs", "7om", "4Jo", "4dC", "7AA", "8W", "1OL", "0Pl", "7V", "ats", "4kB", +"4En", "64E", "2Kc", "1na", "07x", "PB", "69u", "b6e", "4fr", "5c2", "dRL", "8TD", "4Zv", "6oU", "Bj", "0aX", "0Ot", "lF", "6Ay", "40R", +"4yj", "6LI", "av", "0BD", "0lh", "OZ", "6be", "4WF", "4XG", "6md", "2VJ", "0ci", "0ME", "nw", "6CH", "42c", "5kz", "6Nx", "cG", "1PT", +"0nY", "Mk", "5P7", "4Uw", "5N5", "4Ku", "04S", "Si", "9M", "1NV", "4eY", "440", "7Mk", "4hi", "0SG", "pu", "2HH", "K8", "4FE", "67n", +"68o", "4ID", "1", "QX", "2ie", "1Lg", "4gh", "6RK", "7OZ", "4jX", "0Qv", "6L", "2Jy", "3O9", "4Dt", "5A4", "4C9", "4VX", "0mv", "ND", +"22q", "0CZ", "4xt", "6MW", "7PF", "41L", "x9", "mX", "Ct", "14o", "5KI", "6nK", "6aJ", "4Ti", "0oG", "Lu", "bY", "30", "4zE", "6Of", +"5r5", "4wu", "380", "oi", "AE", "0bw", "4YY", "4L8", "5Wz", "66p", "1y5", "1lT", "0RY", "5c", "5l7", "4iw", "4dG", "6Qd", "8S", "1OH", +"05M", "Rw", "7oi", "4Jk", "4Ej", "64A", "2Kg", "1ne", "0Ph", "7R", "7ND", "4kF", "4fv", "5c6", "0H9", "1My", "0st", "PF", "69q", "4HZ", +"0Op", "lB", "ako", "40V", "4Zr", "6oQ", "Bn", "15u", "0ll", "2YO", "6ba", "4WB", "4yn", "6LM", "ar", "1Ra", "0MA", "ns", "6CL", "42g", +"4XC", "79I", "2VN", "0cm", "8gE", "Mo", "5P3", "4Us", "bAl", "adn", "cC", "1PP", "9I", "1NR", "51t", "444", "5N1", "4Kq", "04W", "Sm", +"2HL", "09g", "4FA", "67j", "7Mo", "4hm", "0SC", "4y", "2ia", "1Lc", "4gl", "6RO", "68k", "5Ya", "5", "2GM", "d4F", "82N", "4Dp", "5A0", +"a1e", "bPo", "0Qr", "6H", "Cx", "14c", "5KE", "6nG", "7PJ", "4uH", "x5", "mT", "0V7", "0CV", "4xx", "590", "4C5", "4VT", "0mz", "NH", +"AI", "16R", "4YU", "4L4", "561", "43q", "0LW", "oe", "bU", "w4", "4zI", "6Oj", "6aF", "4Te", "0oK", "Ly", "05A", "2Dj", "7oe", "4Jg", +"4dK", "6Qh", "2jF", "i6", "0RU", "5o", "7Ly", "5yZ", "4GW", "4R6", "1y9", "08q", "07p", "PJ", "7mT", "4HV", "4fz", "6SY", "0H5", "1Mu", +"f7", "sV", "7NH", "4kJ", "4Ef", "64M", "2Kk", "1ni", "4yb", "6LA", "23g", "0BL", "Z3", "OR", "6bm", "4WN", "c4d", "aEO", "Bb", "0aP", +"8Fd", "lN", "4a3", "40Z", "5kr", "4n2", "cO", "8Ie", "0nQ", "Mc", "74u", "615", "4XO", "6ml", "2VB", "U2", "0MM", "2xn", "7Sa", "42k", +"7Mc", "4ha", "0SO", "4u", "3Xa", "K0", "4FM", "67f", "aTL", "b5F", "0pS", "Sa", "9E", "8Wg", "4eQ", "448", "7OR", "4jP", "254", "6D", +"0j3", "1os", "cnn", "65W", "68g", "4IL", "9", "QP", "2im", "1Lo", "53I", "6RC", "7PN", "41D", "x1", "mP", "2Um", "14g", "5KA", "6nC", +"4C1", "4VP", "19W", "NL", "0V3", "0CR", "bBO", "594", "565", "43u", "0LS", "oa", "AM", "16V", "4YQ", "4L0", "6aB", "4Ta", "0oO", "2Zl", +"bQ", "38", "4zM", "6On", "4dO", "6Ql", "2jB", "i2", "05E", "2Dn", "7oa", "4Jc", "4GS", "4R2", "d7e", "08u", "0RQ", "5k", "a2F", "bSL", +"52W", "ayO", "0H1", "1Mq", "07t", "PN", "69y", "4HR", "4Eb", "64I", "2Ko", "1nm", "f3", "7Z", "7NL", "4kN", "Z7", "OV", "6bi", "4WJ", +"4yf", "6LE", "az", "0BH", "0Ox", "lJ", "4a7", "4tV", "4Zz", "6oY", "Bf", "0aT", "0nU", "Mg", "74q", "5EZ", "5kv", "4n6", "cK", "1PX", +"0MI", "2xj", "6CD", "42o", "4XK", "6mh", "2VF", "U6", "2HD", "K4", "4FI", "67b", "7Mg", "4he", "0SK", "4q", "9A", "1NZ", "4eU", "4p4", +"5N9", "4Ky", "0pW", "Se", "0j7", "1ow", "4Dx", "5A8", "7OV", "4jT", "0Qz", "rH", "2ii", "1Lk", "4gd", "6RG", "68c", "4IH", "D5", "QT", +"5Ls", "4I3", "F", "13U", "0IP", "jb", "536", "46v", "5oo", "6Jm", "gR", "r3", "0jL", "3ON", "6dA", "4Qb", "5NB", "aAR", "2Pn", "0eM", +"0Ka", "hS", "6El", "44G", "49w", "abN", "ec", "0FQ", "8ae", "KO", "4F2", "4SS", "4X0", "4MQ", "02w", "UM", "0M2", "0XS", "57T", "a8D", +"7KO", "4nM", "c0", "2Y", "2Nl", "1kn", "aJ1", "61J", "6zC", "aE0", "00F", "2Am", "yP", "l1", "4aL", "6To", "a7E", "58U", "0WR", "0h", +"ZL", "84n", "4BP", "4W1", "fH", "0Ez", "5nu", "4k5", "5U8", "4Px", "0kV", "Hd", "ET", "P5", "5Mi", "6hk", "6FG", "47l", "0HJ", "kx", +"dy", "0GK", "48m", "6IF", "6gj", "4RI", "0ig", "JU", "Ge", "0dW", "5OX", "5Z9", "4d4", "4qU", "1ZZ", "iI", "0Ty", "3C", "4z6", "4oW", +"5QZ", "60P", "Yg", "0zU", "A6", "TW", "6yh", "4LK", "4bg", "6WD", "2lj", "0YI", "0VH", "1r", "6XE", "4mf", "4CJ", "62a", "2MG", "N7", +"0uT", "Vf", "7kx", "4Nz", "5pw", "4u7", "xJ", "1KY", "0IT", "jf", "532", "46r", "5Lw", "4I7", "B", "0gx", "0jH", "Iz", "6dE", "4Qf", +"5ok", "6Ji", "gV", "r7", "0Ke", "hW", "6Eh", "44C", "5NF", "6kD", "2Pj", "0eI", "0hy", "KK", "4F6", "4SW", "49s", "6HX", "eg", "0FU", +"0M6", "0XW", "4cy", "5f9", "4X4", "4MU", "02s", "UI", "Xy", "1kj", "5PD", "61N", "7KK", "4nI", "c4", "vU", "yT", "l5", "4aH", "6Tk", +"6zG", "4Od", "00B", "Wx", "ZH", "0yz", "4BT", "4W5", "5i8", "4lx", "0WV", "0l", "71v", "646", "0kR", "3NP", "fL", "8Lf", "5nq", "4k1", +"6FC", "47h", "0HN", "29e", "EP", "P1", "5Mm", "6ho", "6gn", "4RM", "0ic", "JQ", "26d", "0GO", "48i", "6IB", "4d0", "45Y", "8Cg", "iM", +"Ga", "0dS", "beN", "hYu", "ckm", "60T", "Yc", "0zQ", "207", "3G", "4z2", "4oS", "4bc", "7Ga", "2ln", "0YM", "A2", "TS", "6yl", "4LO", +"4CN", "62e", "2MC", "N3", "0VL", "1v", "6XA", "4mb", "5ps", "4u3", "xN", "8Rd", "01X", "Vb", "aQO", "b0E", "5og", "6Je", "gZ", "63", +"0jD", "Iv", "6dI", "4Qj", "7l9", "6iy", "N", "0gt", "0IX", "jj", "5w6", "4rv", "5mV", "5x7", "ek", "0FY", "0hu", "KG", "6fx", "5Cz", +"5NJ", "6kH", "Fw", "0eE", "92", "3nk", "6Ed", "44O", "7KG", "4nE", "c8", "2Q", "Xu", "1kf", "5PH", "61B", "4X8", "4MY", "0vw", "UE", +"2mx", "1Hz", "4cu", "5f5", "5i4", "4lt", "0WZ", "th", "ZD", "0yv", "4BX", "4W9", "6zK", "4Oh", "00N", "Wt", "yX", "l9", "4aD", "6Tg", +"2SM", "0fn", "5Ma", "6hc", "6FO", "47d", "0HB", "kp", "24Y", "0Er", "bDo", "aam", "5U0", "4Pp", "8bF", "Hl", "Gm", "10v", "5OP", "5Z1", +"anl", "45U", "0Js", "iA", "dq", "0GC", "48e", "6IN", "6gb", "4RA", "0io", "3Lm", "03e", "2BN", "7iA", "4LC", "4bo", "6WL", "zs", "0YA", +"0Tq", "3K", "a4f", "bUl", "4As", "5D3", "Yo", "87M", "01T", "Vn", "5K2", "4Nr", "54w", "417", "xB", "1KQ", "1Fa", "1z", "6XM", "4mn", +"4CB", "62i", "2MO", "0xl", "1za", "Ir", "6dM", "4Qn", "5oc", "6Ja", "25G", "67", "9Pe", "jn", "5w2", "46z", "bfm", "aCo", "J", "0gp", +"0hq", "KC", "72U", "bil", "5mR", "5x3", "eo", "326", "96", "3no", "7UA", "44K", "5NN", "6kL", "Fs", "0eA", "Xq", "1kb", "5PL", "61F", +"7KC", "4nA", "0Uo", "2U", "39U", "8QG", "4cq", "5f1", "aRl", "796", "0vs", "UA", "2LQ", "0yr", "767", "63w", "5i0", "4lp", "9Ng", "0d", +"2oM", "0Zn", "55i", "6Tc", "6zO", "4Ol", "00J", "Wp", "6FK", "4sh", "0HF", "kt", "EX", "P9", "5Me", "6hg", "5U4", "4Pt", "0kZ", "Hh", +"fD", "0Ev", "5ny", "4k9", "4d8", "45Q", "0Jw", "iE", "Gi", "10r", "5OT", "5Z5", "6gf", "4RE", "0ik", "JY", "du", "0GG", "48a", "6IJ", +"4bk", "6WH", "zw", "0YE", "03a", "2BJ", "6yd", "4LG", "4Aw", "5D7", "Yk", "0zY", "0Tu", "3O", "6Zx", "bUh", "54s", "413", "xF", "1KU", +"01P", "Vj", "5K6", "4Nv", "4CF", "62m", "2MK", "0xh", "0VD", "uv", "6XI", "4mj", "5NS", "6kQ", "Fn", "11u", "0Kp", "hB", "aoo", "44V", +"49f", "6HM", "er", "1Va", "0hl", "3Mn", "6fa", "4SB", "5Lb", "7yA", "W", "0gm", "0IA", "js", "6GL", "46g", "bEl", "hyW", "gC", "0Dq", +"8cE", "Io", "5T3", "4Qs", "5J1", "4Oq", "00W", "Wm", "yA", "0Zs", "55t", "404", "6YN", "4lm", "0WC", "0y", "2LL", "0yo", "4BA", "63j", +"6xc", "bws", "02f", "2CM", "2ma", "0XB", "4cl", "6VO", "a5e", "bTo", "0Ur", "2H", "Xl", "86N", "5PQ", "5E0", "dh", "0GZ", "5lU", "5y4", +"4G9", "4RX", "0iv", "JD", "Gt", "0dF", "5OI", "6jK", "6Dg", "45L", "81", "iX", "fY", "70", "5nd", "6Kf", "6eJ", "4Pi", "0kG", "Hu", +"EE", "0fw", "5Mx", "4H8", "5v5", "4su", "1Xz", "ki", "0VY", "1c", "5h7", "4mw", "5Sz", "62p", "2MV", "0xu", "01M", "Vw", "7ki", "4Nk", +"54n", "6Ud", "2nJ", "1KH", "0Th", "3R", "6Ze", "4oF", "4Aj", "60A", "Yv", "0zD", "0wt", "TF", "6yy", "4LZ", "4bv", "5g6", "zj", "0YX", +"0Kt", "hF", "6Ey", "44R", "5NW", "6kU", "Fj", "0eX", "0hh", "KZ", "6fe", "4SF", "49b", "6HI", "ev", "0FD", "0IE", "jw", "6GH", "46c", +"5Lf", "6id", "S", "0gi", "0jY", "Ik", "5T7", "4Qw", "5oz", "6Jx", "gG", "0Du", "yE", "0Zw", "4aY", "400", "5J5", "4Ou", "00S", "Wi", +"ZY", "O8", "4BE", "63n", "6YJ", "4li", "0WG", "tu", "2me", "0XF", "4ch", "6VK", "6xg", "4MD", "02b", "UX", "Xh", "3K9", "5PU", "5E4", +"7KZ", "4nX", "0Uv", "2L", "73V", "bho", "0ir", "1l2", "dl", "335", "48x", "5y0", "6Dc", "45H", "85", "3ol", "Gp", "0dB", "5OM", "6jO", +"6eN", "4Pm", "0kC", "Hq", "24D", "74", "bDr", "6Kb", "529", "47y", "8AG", "km", "EA", "0fs", "bgn", "aBl", "774", "62t", "199", "0xq", +"9Od", "1g", "5h3", "4ms", "54j", "7EA", "2nN", "1KL", "01I", "Vs", "7km", "4No", "4An", "60E", "Yr", "1ja", "0Tl", "3V", "6Za", "4oB", +"4br", "5g2", "zn", "8PD", "03x", "TB", "aSo", "785", "49n", "6HE", "ez", "0FH", "0hd", "KV", "6fi", "4SJ", "bdI", "6kY", "Ff", "0eT", +"0Kx", "hJ", "4e7", "4pV", "5ov", "4j6", "gK", "0Dy", "0jU", "Ig", "6dX", "5AZ", "5Lj", "6ih", "DW", "Q6", "0II", "28b", "6GD", "46o", +"6YF", "4le", "0WK", "0q", "ZU", "O4", "4BI", "63b", "5J9", "4Oy", "0tW", "We", "yI", "1JZ", "4aU", "4t4", "7KV", "4nT", "0Uz", "vH", +"Xd", "1kw", "5PY", "5E8", "6xk", "4MH", "02n", "UT", "2mi", "0XJ", "4cd", "6VG", "2Qm", "0dN", "5OA", "6jC", "6Do", "45D", "89", "iP", +"0R3", "0GR", "48t", "acM", "4G1", "4RP", "94O", "JL", "EM", "12V", "5Mp", "4H0", "525", "47u", "0HS", "ka", "fQ", "78", "5nl", "6Kn", +"6eB", "4Pa", "0kO", "3NM", "01E", "3PO", "7ka", "4Nc", "54f", "6Ul", "xS", "m2", "0VQ", "1k", "a6F", "59V", "4CS", "4V2", "195", "85m", +"03t", "TN", "4Y3", "4LR", "56W", "a9G", "zb", "0YP", "b3", "3Z", "6Zm", "4oN", "4Ab", "60I", "2Oo", "0zL", "1xA", "KR", "6fm", "4SN", +"49j", "6HA", "27g", "0FL", "8Bd", "hN", "4e3", "44Z", "bdM", "aAO", "Fb", "0eP", "0jQ", "Ic", "70u", "655", "5or", "4j2", "gO", "8Me", +"0IM", "28f", "7Wa", "46k", "5Ln", "6il", "DS", "Q2", "ZQ", "O0", "4BM", "63f", "6YB", "4la", "0WO", "0u", "yM", "8Sg", "4aQ", "408", +"aPL", "b1F", "0tS", "Wa", "0n3", "1ks", "bzO", "61W", "7KR", "4nP", "214", "2D", "2mm", "0XN", "57I", "6VC", "6xo", "4ML", "02j", "UP", +"6Dk", "4qH", "0Jf", "iT", "Gx", "0dJ", "5OE", "6jG", "4G5", "4RT", "0iz", "JH", "dd", "0GV", "48p", "5y8", "521", "47q", "0HW", "ke", +"EI", "12R", "5Mt", "4H4", "6eF", "4Pe", "0kK", "Hy", "fU", "s4", "5nh", "6Kj", "54b", "6Uh", "xW", "m6", "01A", "3PK", "7ke", "4Ng", +"4CW", "4V6", "191", "0xy", "0VU", "1o", "6XX", "59R", "4bz", "6WY", "zf", "0YT", "03p", "TJ", "4Y7", "4LV", "4Af", "60M", "Yz", "0zH", +"b7", "wV", "6Zi", "4oJ", "5H3", "4Ms", "02U", "Uo", "2mR", "0Xq", "57v", "426", "7Km", "4no", "0UA", "vs", "2NN", "1kL", "5Pb", "61h", +"6za", "4OB", "00d", "2AO", "yr", "1Ja", "4an", "6TM", "a7g", "58w", "0Wp", "0J", "Zn", "84L", "4Br", "5G2", "5LQ", "5Y0", "d", "13w", +"0Ir", "1L2", "amm", "46T", "5oM", "6JO", "gp", "0DB", "0jn", "3Ol", "6dc", "5Aa", "bdr", "6kb", "2PL", "0eo", "0KC", "hq", "6EN", "44e", +"49U", "abl", "eA", "0Fs", "8aG", "Km", "5V1", "4Sq", "1Dz", "3a", "5j5", "4ou", "4AY", "4T8", "YE", "0zw", "03O", "Tu", "6yJ", "4Li", +"4bE", "6Wf", "zY", "o8", "0Vj", "1P", "6Xg", "4mD", "4Ch", "62C", "2Me", "0xF", "0uv", "VD", "7kZ", "4NX", "5pU", "5e4", "xh", "3k9", +"fj", "0EX", "5nW", "6KU", "6ey", "4PZ", "0kt", "HF", "Ev", "0fD", "5MK", "6hI", "6Fe", "47N", "0Hh", "kZ", "26B", "52", "48O", "6Id", +"6gH", "4Rk", "0iE", "Jw", "GG", "0du", "5Oz", "6jx", "5t7", "4qw", "0JY", "ik", "2mV", "0Xu", "57r", "422", "5H7", "4Mw", "02Q", "Uk", +"2NJ", "1kH", "5Pf", "61l", "7Ki", "4nk", "0UE", "vw", "yv", "0ZD", "4aj", "6TI", "6ze", "4OF", "0th", "WZ", "Zj", "0yX", "4Bv", "5G6", +"6Yy", "4lZ", "0Wt", "0N", "0Iv", "jD", "4g9", "46P", "5LU", "5Y4", "Dh", "0gZ", "0jj", "IX", "6dg", "4QD", "5oI", "6JK", "gt", "0DF", +"0KG", "hu", "6EJ", "44a", "5Nd", "6kf", "FY", "S8", "1xz", "Ki", "5V5", "4Su", "49Q", "4h8", "eE", "0Fw", "756", "60v", "YA", "0zs", +"9Mf", "3e", "5j1", "4oq", "4bA", "6Wb", "2lL", "0Yo", "03K", "Tq", "6yN", "4Lm", "4Cl", "62G", "2Ma", "0xB", "0Vn", "1T", "6Xc", "59i", +"54Y", "5e0", "xl", "8RF", "01z", "1p2", "aQm", "b0g", "71T", "bjm", "0kp", "HB", "fn", "317", "5nS", "6KQ", "6Fa", "47J", "0Hl", "29G", +"Er", "12i", "5MO", "6hM", "6gL", "4Ro", "0iA", "Js", "26F", "56", "48K", "7YA", "5t3", "4qs", "8CE", "io", "GC", "0dq", "bel", "hYW", +"7Ke", "4ng", "0UI", "2s", "XW", "M6", "5Pj", "6uh", "6xX", "6m9", "0vU", "Ug", "2mZ", "0Xy", "4cW", "4v6", "4y7", "4lV", "0Wx", "0B", +"Zf", "0yT", "4Bz", "63Q", "6zi", "4OJ", "B7", "WV", "yz", "0ZH", "4af", "6TE", "5oE", "6JG", "gx", "0DJ", "0jf", "IT", "6dk", "4QH", +"5LY", "5Y8", "l", "0gV", "0Iz", "jH", "4g5", "4rT", "5mt", "4h4", "eI", "1VZ", "0hW", "Ke", "5V9", "4Sy", "5Nh", "6kj", "FU", "S4", +"0KK", "hy", "6EF", "44m", "03G", "2Bl", "6yB", "4La", "4bM", "6Wn", "zQ", "o0", "0TS", "3i", "a4D", "bUN", "4AQ", "4T0", "YM", "87o", +"01v", "VL", "7kR", "4NP", "54U", "hft", "0N3", "1Ks", "0Vb", "1X", "6Xo", "4mL", "5SA", "62K", "2Mm", "0xN", "2So", "0fL", "5MC", "6hA", +"6Fm", "47F", "1XA", "kR", "fb", "0EP", "bDM", "aaO", "4E3", "4PR", "8bd", "HN", "GO", "10T", "5Or", "4J2", "507", "45w", "0JQ", "ic", +"dS", "q2", "48G", "6Il", "73i", "4Rc", "0iM", "3LO", "XS", "M2", "5Pn", "61d", "7Ka", "4nc", "0UM", "2w", "39w", "8Qe", "4cS", "4v2", +"aRN", "b3D", "02Y", "Uc", "Zb", "0yP", "bxM", "63U", "4y3", "4lR", "236", "0F", "2oo", "0ZL", "4ab", "6TA", "6zm", "4ON", "B3", "WR", +"0jb", "IP", "6do", "4QL", "5oA", "6JC", "25e", "0DN", "9PG", "jL", "4g1", "46X", "686", "aCM", "h", "0gR", "0hS", "Ka", "72w", "677", +"49Y", "4h0", "eM", "8Og", "0KO", "3nM", "6EB", "44i", "5Nl", "6kn", "FQ", "S0", "4bI", "6Wj", "zU", "o4", "03C", "Ty", "6yF", "4Le", +"4AU", "4T4", "YI", "1jZ", "0TW", "3m", "5j9", "4oy", "54Q", "5e8", "xd", "1Kw", "01r", "VH", "7kV", "4NT", "4Cd", "62O", "2Mi", "0xJ", +"0Vf", "uT", "6Xk", "4mH", "6Fi", "47B", "0Hd", "kV", "Ez", "0fH", "5MG", "6hE", "4E7", "4PV", "0kx", "HJ", "ff", "0ET", "bDI", "6KY", +"503", "45s", "0JU", "ig", "GK", "0dy", "5Ov", "4J6", "6gD", "4Rg", "0iI", "3LK", "dW", "q6", "48C", "6Ih", "4Z2", "4OS", "00u", "WO", +"yc", "0ZQ", "55V", "hgw", "6Yl", "4lO", "a2", "tS", "2Ln", "0yM", "4Bc", "63H", "6xA", "4Mb", "02D", "2Co", "2mC", "n3", "4cN", "6Vm", +"a5G", "bTM", "0UP", "2j", "XN", "86l", "5Ps", "4U3", "5Nq", "4K1", "FL", "11W", "0KR", "3nP", "514", "44t", "49D", "6Ho", "eP", "49", +"0hN", "3ML", "6fC", "5CA", "aV1", "6iB", "u", "0gO", "0Ic", "jQ", "6Gn", "46E", "bEN", "hyu", "ga", "0DS", "8cg", "IM", "4D0", "4QQ", +"1FZ", "1A", "4x4", "4mU", "4Cy", "5F9", "0m6", "0xW", "C4", "VU", "7kK", "4NI", "54L", "6UF", "xy", "1Kj", "0TJ", "3p", "6ZG", "4od", +"4AH", "60c", "YT", "L5", "0wV", "Td", "5I8", "4Lx", "4bT", "4w5", "zH", "0Yz", "dJ", "0Gx", "5lw", "4i7", "6gY", "4Rz", "0iT", "Jf", +"GV", "R7", "5Ok", "6ji", "6DE", "45n", "0JH", "iz", "24b", "0EI", "5nF", "6KD", "6eh", "4PK", "0ke", "HW", "Eg", "0fU", "5MZ", "6hX", +"4f6", "4sW", "0Hy", "kK", "yg", "0ZU", "55R", "6TX", "4Z6", "4OW", "00q", "WK", "2Lj", "0yI", "4Bg", "63L", "6Yh", "4lK", "a6", "tW", +"2mG", "n7", "4cJ", "6Vi", "6xE", "4Mf", "0vH", "Uz", "XJ", "1kY", "5Pw", "4U7", "7Kx", "4nz", "0UT", "2n", "0KV", "hd", "510", "44p", +"5Nu", "4K5", "FH", "0ez", "0hJ", "Kx", "6fG", "4Sd", "5mi", "6Hk", "eT", "p5", "0Ig", "jU", "6Gj", "46A", "5LD", "6iF", "q", "0gK", +"1zZ", "II", "4D4", "4QU", "5oX", "5z9", "ge", "0DW", "byN", "62V", "0m2", "0xS", "225", "1E", "4x0", "4mQ", "54H", "6UB", "2nl", "1Kn", +"C0", "VQ", "7kO", "4NM", "4AL", "60g", "YP", "L1", "0TN", "3t", "6ZC", "ae0", "4bP", "439", "zL", "8Pf", "03Z", "0b3", "aSM", "b2G", +"73t", "664", "0iP", "Jb", "dN", "8Nd", "48Z", "4i3", "6DA", "45j", "0JL", "3oN", "GR", "R3", "5Oo", "6jm", "6el", "4PO", "0ka", "HS", +"24f", "0EM", "5nB", "aaR", "4f2", "4sS", "8Ae", "kO", "Ec", "0fQ", "695", "aBN", "6Yd", "4lG", "0Wi", "0S", "Zw", "0yE", "4Bk", "6wH", +"6zx", "buh", "0tu", "WG", "yk", "0ZY", "4aw", "5d7", "5k6", "4nv", "0UX", "2b", "XF", "1kU", "741", "61q", "6xI", "4Mj", "02L", "Uv", +"2mK", "0Xh", "4cF", "6Ve", "49L", "6Hg", "eX", "41", "0hF", "Kt", "6fK", "4Sh", "5Ny", "4K9", "FD", "0ev", "0KZ", "hh", "5u4", "4pt", +"5oT", "5z5", "gi", "1Tz", "0jw", "IE", "4D8", "4QY", "5LH", "6iJ", "Du", "0gG", "0Ik", "jY", "6Gf", "46M", "01g", "3Pm", "7kC", "4NA", +"54D", "6UN", "xq", "1Kb", "0Vs", "1I", "a6d", "59t", "4Cq", "5F1", "d3G", "85O", "03V", "Tl", "5I0", "4Lp", "56u", "435", "2lQ", "0Yr", +"0TB", "3x", "6ZO", "4ol", "5Qa", "60k", "2OM", "0zn", "2QO", "0dl", "5Oc", "6ja", "6DM", "45f", "1Za", "ir", "dB", "0Gp", "48V", "aco", +"5W2", "4Rr", "94m", "Jn", "Eo", "12t", "5MR", "5X3", "aln", "47W", "0Hq", "kC", "fs", "0EA", "5nN", "6KL", "71I", "4PC", "0km", "3No", +"Zs", "0yA", "4Bo", "63D", "7IA", "4lC", "0Wm", "0W", "yo", "8SE", "4as", "5d3", "aPn", "b1d", "00y", "WC", "XB", "1kQ", "745", "61u", +"5k2", "4nr", "9Le", "2f", "2mO", "0Xl", "4cB", "6Va", "6xM", "4Mn", "02H", "Ur", "0hB", "Kp", "6fO", "4Sl", "49H", "6Hc", "27E", "45", +"8BF", "hl", "518", "44x", "bdo", "aAm", "2PQ", "0er", "0js", "IA", "70W", "bkn", "5oP", "5z1", "gm", "304", "0Io", "28D", "6Gb", "46I", +"5LL", "6iN", "y", "0gC", "5pH", "6UJ", "xu", "1Kf", "C8", "VY", "7kG", "4NE", "4Cu", "5F5", "2Mx", "1hz", "0Vw", "1M", "4x8", "4mY", +"4bX", "431", "zD", "0Yv", "03R", "Th", "5I4", "4Lt", "4AD", "60o", "YX", "L9", "0TF", "wt", "6ZK", "4oh", "6DI", "45b", "0JD", "iv", +"GZ", "0dh", "5Og", "6je", "5W6", "4Rv", "0iX", "Jj", "dF", "0Gt", "48R", "6Iy", "6Fx", "47S", "0Hu", "kG", "Ek", "0fY", "5MV", "5X7", +"6ed", "4PG", "0ki", "3Nk", "fw", "0EE", "5nJ", "6KH", "356", "bo", "6OP", "4zs", "bnl", "75U", "LC", "0oq", "0bA", "As", "6lL", "4Yo", +"43K", "7RA", "2yN", "0Lm", "17", "22G", "6Ma", "4xB", "4Vn", "6cM", "Nr", "19i", "14Y", "CB", "aDo", "bam", "41z", "5p2", "mn", "8GD", +"7d", "8YF", "4kp", "5n0", "64w", "717", "1nS", "2KQ", "Pp", "07J", "4Hl", "69G", "6Sc", "52i", "1MO", "2hM", "5U", "0Ro", "4iA", "7LC", +"66F", "4Gm", "08K", "3YA", "RA", "0qs", "b4f", "aUl", "5a1", "4dq", "8VG", "8e", "6mV", "4Xu", "17r", "022", "nE", "0Mw", "42Q", "4c8", +"6NJ", "5kH", "1Pf", "cu", "MY", "X8", "4UE", "74O", "6og", "4ZD", "W9", "BX", "lt", "0OF", "4th", "6AK", "4l9", "4yX", "0Bv", "aD", +"Oh", "0lZ", "4Wt", "5R4", "4Iv", "5L6", "Qj", "06P", "1LU", "1Y4", "463", "4gZ", "4jj", "7Oh", "rv", "0QD", "1oI", "2JK", "65m", "4DF", +"4KG", "7nE", "2EJ", "04a", "1Nd", "2kf", "6PH", "4ek", "5xz", "492", "4O", "0Su", "09Q", "0h8", "5C7", "4Fw", "5Dz", "6ax", "LG", "0ou", +"0AY", "bk", "6OT", "4zw", "43O", "6Bd", "2yJ", "0Li", "0bE", "Aw", "6lH", "4Yk", "4Vj", "6cI", "Nv", "0mD", "13", "22C", "6Me", "4xF", +"4uv", "5p6", "mj", "0NX", "1pU", "CF", "6ny", "7k9", "4P9", "4EX", "1nW", "2KU", "sh", "0PZ", "4kt", "5n4", "6Sg", "4fD", "k9", "2hI", +"Pt", "07N", "4Hh", "69C", "66B", "4Gi", "08O", "2Id", "5Q", "d8", "4iE", "7LG", "5a5", "4du", "1Oz", "8a", "RE", "0qw", "4JY", "aUh", +"nA", "0Ms", "42U", "ail", "6mR", "4Xq", "17v", "026", "3Km", "0no", "4UA", "74K", "6NN", "5kL", "1Pb", "cq", "lp", "0OB", "40d", "6AO", +"6oc", "5Ja", "0an", "2TM", "Ol", "18w", "4Wp", "5R0", "afm", "bCo", "0Br", "1G2", "1LQ", "1Y0", "467", "53w", "4Ir", "5L2", "Qn", "06T", +"1oM", "2JO", "65i", "4DB", "4jn", "7Ol", "6z", "1Aa", "8WY", "2kb", "6PL", "4eo", "4KC", "7nA", "2EN", "04e", "09U", "d6E", "5C3", "4Fs", +"bRl", "496", "4K", "0Sq", "0bI", "2Wj", "6lD", "4Yg", "43C", "6Bh", "oW", "z6", "0AU", "bg", "6OX", "5jZ", "4TW", "4A6", "LK", "0oy", +"14Q", "CJ", "4N7", "5Kw", "41r", "542", "mf", "0NT", "u7", "22O", "6Mi", "4xJ", "4Vf", "6cE", "Nz", "0mH", "Px", "07B", "4Hd", "69O", +"6Sk", "4fH", "k5", "2hE", "7l", "0PV", "4kx", "5n8", "4P5", "4ET", "83j", "2KY", "RI", "05s", "4JU", "7oW", "5a9", "4dy", "1Ov", "8m", +"qU", "d4", "4iI", "7LK", "66N", "4Ge", "08C", "2Ih", "6NB", "a59", "1Pn", "21d", "MQ", "X0", "4UM", "74G", "79w", "bbN", "0cS", "0v2", +"nM", "8Dg", "42Y", "4c0", "4l1", "4yP", "8Kf", "aL", "0y3", "0lR", "636", "76v", "6oo", "4ZL", "W1", "BP", "2zm", "0ON", "40h", "6AC", +"4jb", "auS", "6v", "0QL", "I3", "2JC", "65e", "4DN", "b7E", "68U", "Qb", "06X", "286", "dSl", "4r3", "4gR", "4hS", "7MQ", "4G", "277", +"09Y", "0h0", "67T", "b8D", "4KO", "7nM", "SS", "F2", "1Nl", "9w", "azR", "4ec", "43G", "6Bl", "oS", "z2", "0bM", "2Wn", "78i", "4Yc", +"4TS", "4A2", "LO", "8fe", "0AQ", "bc", "aeN", "cPm", "41v", "546", "mb", "0NP", "14U", "CN", "4N3", "5Ks", "4Vb", "6cA", "2Xo", "0mL", +"u3", "22K", "6Mm", "4xN", "6So", "4fL", "k1", "2hA", "2Fm", "07F", "5XA", "69K", "4P1", "4EP", "83n", "d5f", "7h", "0PR", "bQO", "a0E", +"hbu", "50T", "1Or", "8i", "RM", "05w", "4JQ", "7oS", "66J", "4Ga", "08G", "2Il", "5Y", "d0", "4iM", "7LO", "MU", "X4", "4UI", "74C", +"6NF", "5kD", "1Pj", "cy", "nI", "2m9", "4vU", "4c4", "6mZ", "4Xy", "0cW", "0v6", "Od", "0lV", "4Wx", "5R8", "4l5", "4yT", "0Bz", "aH", +"lx", "0OJ", "40l", "6AG", "6ok", "4ZH", "W5", "BT", "I7", "2JG", "65a", "4DJ", "4jf", "7Od", "6r", "0QH", "1LY", "1Y8", "4r7", "4gV", +"4Iz", "68Q", "Qf", "0rT", "1mt", "0h4", "67P", "5VZ", "4hW", "7MU", "4C", "0Sy", "1Nh", "9s", "6PD", "4eg", "4KK", "7nI", "SW", "F6", +"8Je", "22V", "4m2", "4xS", "625", "77u", "Nc", "0mQ", "V2", "CS", "6nl", "5Kn", "41k", "7Pa", "3kO", "0NM", "0AL", "20g", "6OA", "4zb", +"4TN", "6am", "LR", "Y3", "0bP", "Ab", "78t", "bcM", "43Z", "4b3", "oN", "8Ed", "5D", "264", "4iP", "489", "66W", "b9G", "08Z", "0i3", +"RP", "G1", "4JL", "7oN", "6QC", "50I", "1Oo", "8t", "7u", "0PO", "4ka", "7Nc", "64f", "4EM", "H0", "963", "Pa", "0sS", "b6F", "69V", +"478", "4fQ", "295", "dRo", "4O4", "4ZU", "15R", "BI", "le", "0OW", "40q", "551", "6Lj", "4yI", "t4", "aU", "Oy", "0lK", "4We", "6bF", +"6mG", "4Xd", "0cJ", "2Vi", "nT", "0Mf", "4vH", "6Ck", "adI", "5kY", "1Pw", "cd", "MH", "0nz", "4UT", "7pV", "4KV", "7nT", "SJ", "04p", +"1Nu", "9n", "6PY", "4ez", "4hJ", "7MH", "pV", "e7", "1mi", "2Hk", "67M", "4Ff", "4Ig", "68L", "2Gj", "06A", "j6", "2iF", "6Rh", "4gK", +"5zZ", "7Oy", "6o", "0QU", "1oX", "1z9", "4Q6", "4DW", "5FZ", "6cX", "Ng", "0mU", "0Cy", "1F9", "4m6", "4xW", "41o", "7Pe", "3kK", "0NI", +"V6", "CW", "6nh", "5Kj", "4TJ", "6ai", "LV", "Y7", "0AH", "bz", "6OE", "4zf", "4wV", "4b7", "oJ", "0Lx", "0bT", "Af", "6lY", "4Yz", +"5B8", "4Gx", "1lw", "0i7", "qH", "0Rz", "4iT", "7LV", "6QG", "4dd", "1Ok", "8p", "RT", "G5", "4JH", "7oJ", "64b", "4EI", "H4", "2KD", +"7q", "0PK", "4ke", "7Ng", "4s4", "4fU", "1MZ", "2hX", "Pe", "0sW", "4Hy", "5M9", "la", "0OS", "40u", "555", "4O0", "4ZQ", "15V", "BM", +"2Yl", "0lO", "4Wa", "6bB", "6Ln", "4yM", "08", "aQ", "nP", "0Mb", "42D", "6Co", "6mC", "5HA", "0cN", "2Vm", "ML", "8gf", "4UP", "74Z", +"adM", "bAO", "1Ps", "0U3", "1Nq", "9j", "azO", "51W", "4KR", "7nP", "SN", "04t", "09D", "2Ho", "67I", "4Fb", "4hN", "7ML", "4Z", "e3", +"j2", "2iB", "6Rl", "4gO", "4Ic", "68H", "2Gn", "06E", "82m", "d4e", "4Q2", "4DS", "bPL", "a1F", "6k", "0QQ", "1pH", "2UJ", "6nd", "5Kf", +"41c", "7Pi", "mw", "0NE", "0Cu", "1F5", "6Mx", "5hz", "4Vw", "5S7", "Nk", "0mY", "0bX", "Aj", "6lU", "4Yv", "43R", "6By", "oF", "0Lt", +"0AD", "bv", "6OI", "4zj", "4TF", "6ae", "LZ", "0oh", "RX", "G9", "4JD", "7oF", "6QK", "4dh", "1Og", "2je", "5L", "0Rv", "4iX", "481", +"5B4", "4Gt", "08R", "2Iy", "Pi", "07S", "4Hu", "5M5", "470", "4fY", "1MV", "1X7", "su", "0PG", "4ki", "7Nk", "64n", "4EE", "H8", "2KH", +"6Lb", "4yA", "04", "23D", "Oq", "0lC", "4Wm", "6bN", "aEl", "c4G", "0as", "BA", "lm", "8FG", "40y", "559", "6NS", "5kQ", "345", "cl", +"1k2", "0nr", "boo", "74V", "6mO", "4Xl", "0cB", "2Va", "2xM", "0Mn", "42H", "6Cc", "4hB", "aws", "4V", "0Sl", "09H", "2Hc", "67E", "4Fn", +"b5e", "aTo", "SB", "04x", "8WD", "9f", "6PQ", "4er", "4js", "5o3", "6g", "8XE", "1oP", "1z1", "65t", "704", "4Io", "68D", "Qs", "06I", +"1LL", "2iN", "7BA", "4gC", "41g", "7Pm", "ms", "0NA", "14D", "2UN", "aDr", "5Kb", "4Vs", "5S3", "No", "19t", "0Cq", "1F1", "agn", "bBl", +"43V", "aho", "oB", "0Lp", "16u", "An", "6lQ", "4Yr", "4TB", "6aa", "2ZO", "0ol", "1Qa", "br", "6OM", "4zn", "6QO", "4dl", "1Oc", "8x", +"2DM", "05f", "5Za", "7oB", "5B0", "4Gp", "08V", "d7F", "5H", "0Rr", "bSo", "485", "474", "52t", "1MR", "1X3", "Pm", "07W", "4Hq", "5M1", +"64j", "4EA", "1nN", "2KL", "7y", "0PC", "4km", "7No", "Ou", "0lG", "4Wi", "6bJ", "6Lf", "4yE", "00", "aY", "li", "8FC", "4tu", "5q5", +"4O8", "4ZY", "0aw", "BE", "MD", "0nv", "4UX", "74R", "6NW", "5kU", "341", "ch", "nX", "0Mj", "42L", "6Cg", "6mK", "4Xh", "0cF", "2Ve", +"09L", "2Hg", "67A", "4Fj", "4hF", "7MD", "4R", "0Sh", "1Ny", "9b", "6PU", "4ev", "4KZ", "7nX", "SF", "0pt", "1oT", "1z5", "65p", "5Tz", +"4jw", "5o7", "6c", "0QY", "1LH", "2iJ", "6Rd", "4gG", "4Ik", "7li", "Qw", "06M", "7F", "246", "4kR", "7NP", "64U", "col", "1nq", "0k1", +"PR", "E3", "4HN", "69e", "6SA", "4fb", "1Mm", "2ho", "5w", "0RM", "4ic", "7La", "66d", "4GO", "J2", "2IB", "Rc", "05Y", "b4D", "aUN", +"4q2", "4dS", "8Ve", "8G", "8Hg", "bM", "4o0", "4zQ", "607", "75w", "La", "0oS", "T0", "AQ", "6ln", "4YM", "43i", "6BB", "2yl", "0LO", +"0CN", "22e", "6MC", "5hA", "4VL", "6co", "NP", "0mb", "1ps", "0u3", "aDM", "baO", "41X", "7PR", "mL", "8Gf", "4IT", "7lV", "QH", "06r", +"1Lw", "0I7", "5b8", "4gx", "4jH", "7OJ", "rT", "g5", "1ok", "2Ji", "65O", "4Dd", "4Ke", "7ng", "Sy", "04C", "h4", "2kD", "6Pj", "4eI", +"4hy", "5m9", "4m", "0SW", "09s", "2HX", "4S4", "4FU", "4M6", "4XW", "0cy", "1f9", "ng", "0MU", "42s", "573", "6Nh", "5kj", "v6", "cW", +"3KK", "0nI", "4Ug", "74m", "6oE", "4Zf", "0aH", "Bz", "lV", "y7", "40B", "6Ai", "582", "4yz", "0BT", "af", "OJ", "0lx", "4WV", "4B7", +"64Q", "4Ez", "1nu", "0k5", "7B", "0Px", "4kV", "7NT", "6SE", "4ff", "1Mi", "2hk", "PV", "E7", "4HJ", "69a", "6rh", "4GK", "J6", "2IF", +"5s", "0RI", "4ig", "7Le", "4q6", "4dW", "1OX", "8C", "Rg", "0qU", "5ZZ", "7oy", "4Ty", "5Q9", "Le", "0oW", "1QZ", "bI", "4o4", "4zU", +"43m", "6BF", "oy", "0LK", "T4", "AU", "6lj", "4YI", "4VH", "6ck", "NT", "0mf", "0CJ", "22a", "6MG", "4xd", "4uT", "7PV", "mH", "0Nz", +"1pw", "Cd", "aDI", "5KY", "1Ls", "0I3", "axM", "53U", "4IP", "7lR", "QL", "06v", "1oo", "2Jm", "65K", "5TA", "4jL", "7ON", "6X", "g1", +"h0", "9Y", "6Pn", "4eM", "4Ka", "7nc", "2El", "04G", "09w", "d6g", "4S0", "4FQ", "bRN", "a3D", "4i", "0SS", "nc", "0MQ", "42w", "577", +"4M2", "4XS", "17T", "dlm", "3KO", "0nM", "4Uc", "74i", "6Nl", "5kn", "v2", "cS", "lR", "y3", "40F", "6Am", "6oA", "4Zb", "0aL", "2To", +"ON", "18U", "4WR", "4B3", "586", "bCM", "0BP", "ab", "PZ", "0sh", "4HF", "69m", "6SI", "4fj", "1Me", "2hg", "7N", "0Pt", "4kZ", "7NX", +"6pU", "4Ev", "1ny", "0k9", "Rk", "05Q", "4Jw", "5O7", "452", "50r", "1OT", "8O", "qw", "0RE", "4ik", "7Li", "66l", "4GG", "08a", "2IJ", +"T8", "AY", "6lf", "4YE", "43a", "6BJ", "ou", "0LG", "0Aw", "bE", "4o8", "4zY", "4Tu", "5Q5", "Li", "8fC", "14s", "Ch", "6nW", "5KU", +"41P", "7PZ", "mD", "0Nv", "0CF", "22m", "6MK", "4xh", "4VD", "6cg", "NX", "0mj", "5za", "7OB", "6T", "0Qn", "1oc", "2Ja", "65G", "4Dl", +"b7g", "68w", "1w2", "06z", "8UF", "dSN", "5b0", "4gp", "4hq", "5m1", "4e", "8ZG", "1mR", "1x3", "67v", "726", "4Km", "7no", "Sq", "04K", +"1NN", "9U", "6Pb", "4eA", "adr", "5kb", "26", "21F", "Ms", "0nA", "4Uo", "74e", "79U", "bbl", "0cq", "1f1", "no", "396", "4vs", "5s3", +"6LQ", "4yr", "367", "an", "OB", "0lp", "bmm", "76T", "6oM", "4Zn", "15i", "Br", "2zO", "0Ol", "40J", "6Aa", "6SM", "4fn", "1Ma", "2hc", +"2FO", "07d", "4HB", "69i", "64Y", "4Er", "83L", "d5D", "7J", "0Pp", "bQm", "a0g", "456", "50v", "1OP", "8K", "Ro", "05U", "4Js", "5O3", +"66h", "4GC", "08e", "2IN", "qs", "0RA", "4io", "7Lm", "43e", "6BN", "oq", "0LC", "0bo", "2WL", "6lb", "4YA", "4Tq", "5Q1", "Lm", "8fG", +"0As", "bA", "ael", "cPO", "41T", "ajm", "1K2", "0Nr", "14w", "Cl", "6nS", "5KQ", "5Fa", "6cc", "2XM", "0mn", "0CB", "22i", "6MO", "4xl", +"1og", "2Je", "65C", "4Dh", "4jD", "7OF", "6P", "g9", "3l9", "2iy", "5b4", "4gt", "4IX", "68s", "QD", "0rv", "1mV", "1x7", "4S8", "4FY", +"4hu", "5m5", "4a", "1Cz", "h8", "9Q", "6Pf", "4eE", "4Ki", "7nk", "Su", "04O", "Mw", "0nE", "4Uk", "74a", "6Nd", "5kf", "22", "21B", +"nk", "0MY", "4vw", "5s7", "6mx", "5Hz", "0cu", "1f5", "OF", "0lt", "4WZ", "6by", "6LU", "4yv", "0BX", "aj", "lZ", "0Oh", "40N", "6Ae", +"6oI", "4Zj", "0aD", "Bv", "5f", "9Ke", "4ir", "5l2", "66u", "735", "08x", "1y0", "Rr", "05H", "4Jn", "7ol", "6Qa", "4dB", "1OM", "8V", +"7W", "0Pm", "4kC", "7NA", "64D", "4Eo", "83Q", "2Kb", "PC", "07y", "b6d", "69t", "5c3", "4fs", "8TE", "dRM", "374", "22t", "599", "4xq", +"bln", "77W", "NA", "0ms", "14j", "Cq", "6nN", "5KL", "41I", "7PC", "3km", "0No", "35", "20E", "6Oc", "5ja", "4Tl", "6aO", "Lp", "0oB", +"0br", "1g2", "78V", "bco", "43x", "568", "ol", "385", "4Kt", "5N4", "Sh", "04R", "1NW", "9L", "441", "4eX", "4hh", "7Mj", "pt", "0SF", +"K9", "2HI", "67o", "4FD", "4IE", "68n", "QY", "0", "1Lf", "2id", "6RJ", "4gi", "4jY", "auh", "6M", "0Qw", "1oz", "2Jx", "5A5", "4Du", +"6oT", "4Zw", "0aY", "Bk", "lG", "0Ou", "40S", "6Ax", "6LH", "4yk", "0BE", "aw", "2YJ", "0li", "4WG", "6bd", "6me", "4XF", "0ch", "2VK", +"nv", "0MD", "42b", "6CI", "6Ny", "7K9", "1PU", "cF", "Mj", "0nX", "4Uv", "5P6", "66q", "4GZ", "1lU", "1y4", "5b", "0RX", "4iv", "5l6", +"6Qe", "4dF", "1OI", "8R", "Rv", "05L", "4Jj", "7oh", "6pH", "4Ek", "1nd", "2Kf", "7S", "0Pi", "4kG", "7NE", "5c7", "4fw", "1Mx", "0H8", +"PG", "0su", "5Xz", "69p", "4VY", "4C8", "NE", "0mw", "1Sz", "22p", "6MV", "4xu", "41M", "7PG", "mY", "x8", "14n", "Cu", "6nJ", "5KH", +"4Th", "6aK", "Lt", "0oF", "31", "bX", "6Og", "4zD", "4wt", "5r4", "oh", "0LZ", "0bv", "AD", "4L9", "4YX", "1NS", "9H", "445", "51u", +"4Kp", "5N0", "Sl", "04V", "09f", "2HM", "67k", "5Va", "4hl", "7Mn", "4x", "0SB", "1Lb", "3yA", "6RN", "4gm", "4IA", "68j", "2GL", "4", +"82O", "d4G", "5A1", "4Dq", "bPn", "a1d", "6I", "0Qs", "lC", "0Oq", "40W", "akn", "6oP", "4Zs", "15t", "Bo", "2YN", "0lm", "4WC", "76I", +"6LL", "4yo", "0BA", "as", "nr", "8DX", "42f", "6CM", "6ma", "4XB", "0cl", "2VO", "Mn", "8gD", "4Ur", "5P2", "ado", "bAm", "1PQ", "cB", +"Rz", "0qH", "4Jf", "7od", "6Qi", "4dJ", "i7", "2jG", "5n", "0RT", "4iz", "7Lx", "4R7", "4GV", "08p", "1y8", "PK", "07q", "4HW", "7mU", +"6SX", "52R", "1Mt", "0H4", "sW", "f6", "4kK", "7NI", "64L", "4Eg", "1nh", "2Kj", "14b", "Cy", "6nF", "5KD", "41A", "7PK", "mU", "x4", +"0CW", "0V6", "591", "4xy", "4VU", "4C4", "NI", "19R", "0bz", "AH", "4L5", "4YT", "43p", "560", "od", "0LV", "w5", "bT", "6Ok", "4zH", +"4Td", "6aG", "Lx", "0oJ", "5xA", "7Mb", "4t", "0SN", "K1", "2HA", "67g", "4FL", "b5G", "aTM", "0e3", "04Z", "8Wf", "9D", "449", "4eP", +"4jQ", "7OS", "6E", "255", "1or", "0j2", "65V", "cno", "4IM", "68f", "QQ", "8", "1Ln", "2il", "6RB", "4ga", "afR", "4yc", "0BM", "23f", +"OS", "Z2", "4WO", "6bl", "aEN", "c4e", "0aQ", "Bc", "lO", "8Fe", "4tS", "4a2", "4n3", "5ks", "8Id", "cN", "Mb", "0nP", "614", "74t", +"6mm", "4XN", "U3", "2VC", "2xo", "0ML", "42j", "6CA", "6Qm", "4dN", "i3", "8Z", "2Do", "05D", "4Jb", "aUS", "4R3", "4GR", "08t", "d7d", +"5j", "0RP", "bSM", "a2G", "ayN", "52V", "1Mp", "0H0", "PO", "07u", "4HS", "69x", "64H", "4Ec", "1nl", "2Kn", "sS", "f2", "4kO", "7NM", +"41E", "7PO", "mQ", "x0", "14f", "2Ul", "6nB", "aQ1", "4VQ", "4C0", "NM", "19V", "0CS", "0V2", "595", "bBN", "43t", "564", "0Y3", "0LR", +"16W", "AL", "4L1", "4YP", "5DA", "6aC", "2Zm", "0oN", "39", "bP", "6Oo", "4zL", "K5", "2HE", "67c", "4FH", "4hd", "7Mf", "4p", "0SJ", +"8Wb", "2kY", "4p5", "4eT", "4Kx", "5N8", "Sd", "0pV", "1ov", "0j6", "5A9", "4Dy", "4jU", "7OW", "6A", "1AZ", "1Lj", "2ih", "6RF", "4ge", +"4II", "68b", "QU", "D4", "OW", "Z6", "4WK", "6bh", "6LD", "4yg", "0BI", "23b", "lK", "0Oy", "4tW", "4a6", "6oX", "5JZ", "0aU", "Bg", +"Mf", "0nT", "4Uz", "74p", "4n7", "5kw", "1PY", "cJ", "nz", "0MH", "42n", "6CE", "6mi", "4XJ", "U7", "2VG", "4MP", "4X1", "UL", "02v", +"0XR", "0M3", "a8E", "57U", "4nL", "7KN", "2X", "c1", "1ko", "2Nm", "61K", "5PA", "4Oa", "6zB", "2Al", "00G", "l0", "yQ", "6Tn", "4aM", +"58T", "a7D", "0i", "0WS", "84o", "ZM", "4W0", "4BQ", "4I2", "5Lr", "13T", "G", "jc", "0IQ", "46w", "537", "6Jl", "5on", "r2", "gS", +"3OO", "0jM", "4Qc", "70i", "6kA", "5NC", "0eL", "2Po", "hR", "8Bx", "44F", "6Em", "abO", "49v", "0FP", "eb", "KN", "8ad", "4SR", "4F3", +"3B", "0Tx", "4oV", "4z7", "60Q", "4Az", "0zT", "Yf", "TV", "A7", "4LJ", "6yi", "6WE", "4bf", "0YH", "zz", "1s", "0VI", "4mg", "6XD", +"6vh", "4CK", "N6", "2MF", "Vg", "0uU", "6n9", "7ky", "4u6", "5pv", "1KX", "xK", "1UZ", "fI", "4k4", "5nt", "4Py", "5U9", "He", "0kW", +"P4", "EU", "6hj", "5Mh", "47m", "6FF", "ky", "0HK", "0GJ", "dx", "6IG", "48l", "4RH", "6gk", "JT", "0if", "0dV", "Gd", "5Z8", "5OY", +"4qT", "4d5", "iH", "0Jz", "0XV", "0M7", "5f8", "4cx", "4MT", "4X5", "UH", "02r", "1kk", "Xx", "61O", "5PE", "4nH", "7KJ", "vT", "c5", +"l4", "yU", "6Tj", "4aI", "4Oe", "6zF", "Wy", "00C", "1iZ", "ZI", "4W4", "4BU", "4ly", "5i9", "0m", "0WW", "jg", "0IU", "46s", "533", +"4I6", "5Lv", "0gy", "C", "3OK", "0jI", "4Qg", "6dD", "6Jh", "5oj", "r6", "gW", "hV", "0Kd", "44B", "6Ei", "6kE", "5NG", "0eH", "Fz", +"KJ", "0hx", "4SV", "4F7", "6HY", "49r", "0FT", "ef", "60U", "ckl", "0zP", "Yb", "3F", "206", "4oR", "4z3", "6WA", "4bb", "0YL", "2lo", +"TR", "A3", "4LN", "6ym", "62d", "4CO", "N2", "2MB", "1w", "0VM", "4mc", "7Ha", "4u2", "54z", "8Re", "xO", "Vc", "01Y", "b0D", "aQN", +"647", "71w", "Ha", "0kS", "8Lg", "fM", "4k0", "5np", "47i", "6FB", "29d", "0HO", "P0", "EQ", "6hn", "5Ml", "4RL", "6go", "JP", "0ib", +"0GN", "26e", "6IC", "48h", "45X", "4d1", "iL", "8Cf", "0dR", "0q3", "hYt", "beO", "4nD", "7KF", "2P", "c9", "1kg", "Xt", "61C", "5PI", +"4MX", "4X9", "UD", "0vv", "0XZ", "2my", "5f4", "4ct", "4lu", "5i5", "0a", "1Gz", "0yw", "ZE", "4W8", "4BY", "4Oi", "6zJ", "Wu", "00O", +"l8", "yY", "6Tf", "4aE", "6Jd", "5of", "62", "25B", "Iw", "0jE", "4Qk", "6dH", "6ix", "5Lz", "0gu", "O", "jk", "0IY", "4rw", "5w7", +"5x6", "5mW", "0FX", "ej", "KF", "0ht", "4SZ", "6fy", "6kI", "5NK", "0eD", "Fv", "hZ", "93", "44N", "6Ee", "2BO", "03d", "4LB", "6ya", +"6WM", "4bn", "1Ia", "zr", "3J", "0Tp", "bUm", "a4g", "5D2", "4Ar", "87L", "Yn", "Vo", "01U", "4Ns", "5K3", "416", "54v", "1KP", "xC", +"us", "0VA", "4mo", "6XL", "62h", "4CC", "0xm", "2MN", "0fo", "2SL", "6hb", "bgr", "47e", "6FN", "kq", "0HC", "0Es", "fA", "aal", "bDn", +"4Pq", "5U1", "Hm", "8bG", "10w", "Gl", "5Z0", "5OQ", "45T", "anm", "1O2", "0Jr", "0GB", "dp", "6IO", "48d", "5Ba", "6gc", "3Ll", "0in", +"1kc", "Xp", "61G", "5PM", "bTs", "7KB", "2T", "0Un", "8QF", "39T", "5f0", "4cp", "797", "aRm", "1s2", "02z", "0ys", "ZA", "63v", "766", +"4lq", "5i1", "0e", "9Nf", "0Zo", "2oL", "6Tb", "4aA", "4Om", "6zN", "Wq", "00K", "Is", "0jA", "4Qo", "6dL", "7ZA", "5ob", "66", "25F", +"jo", "9Pd", "4rs", "5w3", "aCn", "bfl", "0gq", "K", "KB", "0hp", "bim", "72T", "5x2", "49z", "327", "en", "3nn", "97", "44J", "6Ea", +"6kM", "5NO", "11i", "Fr", "6WI", "4bj", "0YD", "zv", "TZ", "0wh", "4LF", "6ye", "5D6", "4Av", "0zX", "Yj", "3N", "0Tt", "4oZ", "6Zy", +"412", "54r", "1KT", "xG", "Vk", "01Q", "4Nw", "5K7", "62l", "4CG", "0xi", "2MJ", "uw", "0VE", "4mk", "6XH", "47a", "6FJ", "ku", "0HG", +"P8", "EY", "6hf", "5Md", "4Pu", "5U5", "Hi", "8bC", "0Ew", "fE", "4k8", "5nx", "45P", "4d9", "iD", "0Jv", "0dZ", "Gh", "5Z4", "5OU", +"4RD", "6gg", "JX", "0ij", "0GF", "dt", "6IK", "5lI", "4Op", "5J0", "Wl", "00V", "0Zr", "2oQ", "405", "55u", "4ll", "6YO", "0x", "0WB", +"0yn", "2LM", "63k", "5Ra", "4MA", "6xb", "2CL", "02g", "0XC", "39I", "6VN", "4cm", "bTn", "a5d", "2I", "0Us", "86O", "Xm", "5E1", "5PP", +"6kP", "5NR", "11t", "Fo", "hC", "0Kq", "44W", "aon", "6HL", "49g", "0FA", "es", "3Mo", "0hm", "4SC", "72I", "6ia", "5Lc", "0gl", "V", +"jr", "1Ya", "46f", "6GM", "hyV", "bEm", "0Dp", "gB", "In", "8cD", "4Qr", "5T2", "1b", "0VX", "4mv", "5h6", "62q", "4CZ", "0xt", "2MW", +"Vv", "01L", "4Nj", "7kh", "6Ue", "54o", "1KI", "xZ", "3S", "0Ti", "4oG", "6Zd", "6tH", "4Ak", "0zE", "Yw", "TG", "0wu", "780", "6yx", +"5g7", "4bw", "0YY", "zk", "1Wz", "di", "5y5", "5lT", "4RY", "4G8", "JE", "0iw", "0dG", "Gu", "6jJ", "5OH", "45M", "6Df", "iY", "80", +"71", "fX", "6Kg", "5ne", "4Ph", "6eK", "Ht", "0kF", "0fv", "ED", "4H9", "5My", "4st", "5v4", "kh", "0HZ", "0Zv", "yD", "401", "4aX", +"4Ot", "5J4", "Wh", "00R", "O9", "ZX", "63o", "4BD", "4lh", "6YK", "tt", "0WF", "0XG", "2md", "6VJ", "4ci", "4ME", "6xf", "UY", "02c", +"1kz", "Xi", "5E5", "5PT", "4nY", "aqh", "2M", "0Uw", "hG", "0Ku", "44S", "6Ex", "6kT", "5NV", "0eY", "Fk", "3Mk", "0hi", "4SG", "6fd", +"6HH", "49c", "0FE", "ew", "jv", "0ID", "46b", "6GI", "6ie", "5Lg", "0gh", "R", "Ij", "0jX", "4Qv", "5T6", "6Jy", "7O9", "0Dt", "gF", +"62u", "775", "0xp", "198", "1f", "9Oe", "4mr", "5h2", "6Ua", "54k", "1KM", "2nO", "Vr", "01H", "4Nn", "7kl", "60D", "4Ao", "0zA", "Ys", +"3W", "0Tm", "4oC", "7JA", "5g3", "4bs", "8PE", "zo", "TC", "03y", "784", "aSn", "bhn", "73W", "JA", "0is", "334", "dm", "5y1", "48y", +"45I", "6Db", "3om", "84", "0dC", "Gq", "6jN", "5OL", "4Pl", "6eO", "Hp", "0kB", "75", "24E", "6Kc", "5na", "47x", "528", "kl", "8AF", +"0fr", "1c2", "aBm", "bgo", "4ld", "6YG", "0p", "0WJ", "O5", "ZT", "63c", "4BH", "4Ox", "5J8", "Wd", "0tV", "0Zz", "yH", "4t5", "4aT", +"4nU", "7KW", "2A", "1EZ", "1kv", "Xe", "5E9", "5PX", "4MI", "6xj", "UU", "02o", "0XK", "2mh", "6VF", "4ce", "6HD", "49o", "0FI", "27b", +"KW", "0he", "4SK", "6fh", "6kX", "5NZ", "0eU", "Fg", "hK", "0Ky", "4pW", "4e6", "4j7", "5ow", "0Dx", "gJ", "If", "0jT", "4Qz", "6dY", +"6ii", "5Lk", "Q7", "DV", "jz", "0IH", "46n", "6GE", "3PN", "01D", "4Nb", "aQS", "6Um", "54g", "m3", "xR", "1j", "0VP", "59W", "a6G", +"4V3", "4CR", "85l", "194", "TO", "03u", "4LS", "4Y2", "a9F", "56V", "0YQ", "zc", "wS", "b2", "4oO", "6Zl", "60H", "4Ac", "0zM", "2On", +"0dO", "2Ql", "6jB", "aU1", "45E", "6Dn", "iQ", "88", "0GS", "da", "acL", "48u", "4RQ", "4G0", "JM", "94N", "12W", "EL", "4H1", "5Mq", +"47t", "524", "29y", "0HR", "79", "fP", "6Ko", "5nm", "aZ0", "6eC", "3NL", "0kN", "O1", "ZP", "63g", "4BL", "58I", "6YC", "0t", "0WN", +"8Sf", "yL", "409", "4aP", "b1G", "aPM", "0a3", "00Z", "1kr", "Xa", "61V", "bzN", "4nQ", "7KS", "2E", "215", "0XO", "2ml", "6VB", "4ca", +"4MM", "6xn", "UQ", "02k", "KS", "0ha", "4SO", "6fl", "7Xa", "49k", "0FM", "27f", "hO", "8Be", "4pS", "4e2", "aAN", "bdL", "0eQ", "Fc", +"Ib", "0jP", "654", "70t", "4j3", "5os", "8Md", "gN", "28g", "0IL", "46j", "6GA", "6im", "5Lo", "Q3", "Z", "6Ui", "54c", "m7", "xV", +"Vz", "0uH", "4Nf", "7kd", "4V7", "4CV", "0xx", "190", "1n", "0VT", "4mz", "6XY", "6WX", "56R", "0YU", "zg", "TK", "03q", "4LW", "4Y6", +"60L", "4Ag", "0zI", "2Oj", "wW", "b6", "4oK", "6Zh", "45A", "6Dj", "iU", "0Jg", "0dK", "Gy", "6jF", "5OD", "4RU", "4G4", "JI", "1yZ", +"0GW", "de", "5y9", "48q", "47p", "520", "kd", "0HV", "0fz", "EH", "4H5", "5Mu", "4Pd", "6eG", "Hx", "0kJ", "s5", "fT", "6Kk", "5ni", +"5Y1", "5LP", "13v", "e", "jA", "0Is", "46U", "aml", "6JN", "5oL", "0DC", "gq", "3Om", "0jo", "4QA", "6db", "6kc", "5Na", "0en", "2PM", +"hp", "0KB", "44d", "6EO", "abm", "49T", "0Fr", "1C2", "Kl", "8aF", "4Sp", "5V0", "4Mr", "5H2", "Un", "02T", "0Xp", "2mS", "427", "57w", +"4nn", "7Kl", "2z", "1Ea", "1kM", "2NO", "61i", "5Pc", "4OC", "7jA", "2AN", "00e", "0ZA", "ys", "6TL", "4ao", "58v", "a7f", "0K", "0Wq", +"84M", "Zo", "5G3", "4Bs", "0EY", "fk", "6KT", "5nV", "bjh", "6ex", "HG", "0ku", "0fE", "Ew", "6hH", "5MJ", "47O", "6Fd", "29B", "0Hi", +"53", "dZ", "6Ie", "48N", "4Rj", "6gI", "Jv", "0iD", "0dt", "GF", "6jy", "7o9", "4qv", "5t6", "ij", "0JX", "wh", "0TZ", "4ot", "5j4", +"4T9", "4AX", "0zv", "YD", "Tt", "03N", "4Lh", "6yK", "6Wg", "4bD", "o9", "zX", "1Q", "0Vk", "4mE", "6Xf", "62B", "4Ci", "0xG", "2Md", +"VE", "0uw", "4NY", "aQh", "5e5", "5pT", "1Kz", "xi", "jE", "0Iw", "46Q", "4g8", "5Y5", "5LT", "13r", "a", "IY", "0jk", "4QE", "6df", +"6JJ", "5oH", "0DG", "gu", "ht", "0KF", "4ph", "6EK", "6kg", "5Ne", "S9", "FX", "Kh", "0hZ", "4St", "5V4", "4h9", "49P", "0Fv", "eD", +"0Xt", "2mW", "423", "4cZ", "4Mv", "5H6", "Uj", "02P", "1kI", "XZ", "61m", "5Pg", "4nj", "7Kh", "vv", "0UD", "0ZE", "yw", "6TH", "4ak", +"4OG", "6zd", "2AJ", "00a", "0yY", "Zk", "5G7", "4Bw", "58r", "6Yx", "0O", "0Wu", "bjl", "71U", "HC", "0kq", "316", "fo", "6KP", "5nR", +"47K", "7VA", "29F", "0Hm", "0fA", "Es", "6hL", "5MN", "4Rn", "6gM", "Jr", "1ya", "57", "26G", "6Ia", "48J", "45z", "5t2", "in", "8CD", +"0dp", "GB", "hYV", "bem", "60w", "757", "0zr", "2OQ", "3d", "9Mg", "4op", "5j0", "6Wc", "56i", "0Yn", "2lM", "Tp", "03J", "4Ll", "6yO", +"62F", "4Cm", "0xC", "dwS", "1U", "0Vo", "4mA", "6Xb", "5e1", "54X", "8RG", "xm", "VA", "0us", "b0f", "aQl", "6JF", "5oD", "0DK", "gy", +"IU", "0jg", "4QI", "6dj", "5Y9", "5LX", "0gW", "m", "jI", "1YZ", "4rU", "4g4", "4h5", "5mu", "0Fz", "eH", "Kd", "0hV", "4Sx", "5V8", +"6kk", "5Ni", "S5", "FT", "hx", "0KJ", "44l", "6EG", "4nf", "7Kd", "2r", "0UH", "M7", "XV", "61a", "5Pk", "4Mz", "6xY", "Uf", "0vT", +"0Xx", "39r", "4v7", "4cV", "4lW", "4y6", "0C", "0Wy", "0yU", "Zg", "63P", "5RZ", "4OK", "6zh", "WW", "B6", "0ZI", "2oj", "6TD", "4ag", +"0fM", "2Sn", "7xa", "5MB", "47G", "6Fl", "kS", "0Ha", "0EQ", "fc", "aaN", "bDL", "4PS", "4E2", "HO", "8be", "10U", "GN", "4J3", "5Os", +"45v", "506", "ib", "0JP", "q3", "dR", "6Im", "48F", "4Rb", "6gA", "3LN", "0iL", "2Bm", "03F", "aF0", "6yC", "6Wo", "4bL", "o1", "zP", +"3h", "0TR", "bUO", "a4E", "4T1", "4AP", "87n", "YL", "VM", "01w", "4NQ", "7kS", "hfu", "54T", "1Kr", "xa", "1Y", "0Vc", "4mM", "6Xn", +"62J", "4Ca", "0xO", "2Ml", "IQ", "0jc", "4QM", "6dn", "6JB", "a19", "0DO", "25d", "jM", "9PF", "46Y", "4g0", "aCL", "687", "0gS", "i", +"3MP", "0hR", "676", "72v", "4h1", "49X", "8Of", "eL", "3nL", "0KN", "44h", "6EC", "6ko", "5Nm", "S1", "FP", "M3", "XR", "61e", "5Po", +"4nb", "aqS", "2v", "0UL", "8Qd", "39v", "4v3", "4cR", "b3E", "aRO", "Ub", "02X", "0yQ", "Zc", "63T", "bxL", "4lS", "4y2", "0G", "237", +"0ZM", "2on", "7Da", "4ac", "4OO", "6zl", "WS", "B2", "47C", "6Fh", "kW", "0He", "0fI", "2Sj", "6hD", "5MF", "4PW", "4E6", "HK", "0ky", +"0EU", "fg", "6KX", "5nZ", "45r", "502", "if", "0JT", "0dx", "GJ", "4J7", "5Ow", "4Rf", "6gE", "Jz", "0iH", "q7", "dV", "6Ii", "48B", +"6Wk", "4bH", "o5", "zT", "Tx", "03B", "4Ld", "6yG", "4T5", "4AT", "0zz", "YH", "3l", "0TV", "4ox", "5j8", "5e9", "54P", "1Kv", "xe", +"VI", "01s", "4NU", "7kW", "62N", "4Ce", "0xK", "2Mh", "uU", "0Vg", "4mI", "6Xj", "4K0", "5Np", "11V", "FM", "ha", "0KS", "44u", "515", +"6Hn", "49E", "48", "eQ", "3MM", "0hO", "4Sa", "6fB", "6iC", "5LA", "0gN", "t", "jP", "0Ib", "46D", "6Go", "hyt", "bEO", "0DR", "0Q3", +"IL", "8cf", "4QP", "4D1", "4OR", "4Z3", "WN", "00t", "0ZP", "yb", "hgv", "55W", "4lN", "6Ym", "0Z", "a3", "0yL", "2Lo", "63I", "4Bb", +"4Mc", "7ha", "2Cn", "02E", "n2", "2mB", "6Vl", "4cO", "bTL", "a5F", "2k", "0UQ", "86m", "XO", "4U2", "5Pr", "0Gy", "dK", "4i6", "5lv", +"5BZ", "6gX", "Jg", "0iU", "R6", "GW", "6jh", "5Oj", "45o", "6DD", "3oK", "0JI", "0EH", "fz", "6KE", "5nG", "4PJ", "6ei", "HV", "0kd", +"0fT", "Ef", "6hY", "690", "4sV", "4f7", "kJ", "0Hx", "uH", "0Vz", "4mT", "4x5", "5F8", "4Cx", "0xV", "0m7", "VT", "C5", "4NH", "7kJ", +"6UG", "54M", "1Kk", "xx", "3q", "0TK", "4oe", "6ZF", "60b", "4AI", "L4", "YU", "Te", "0wW", "4Ly", "5I9", "4w4", "4bU", "1IZ", "zI", +"he", "0KW", "44q", "511", "4K4", "5Nt", "11R", "FI", "Ky", "0hK", "4Se", "6fF", "6Hj", "49A", "p4", "eU", "jT", "0If", "4rH", "6Gk", +"6iG", "5LE", "0gJ", "p", "IH", "0jz", "4QT", "4D5", "5z8", "5oY", "0DV", "gd", "0ZT", "yf", "6TY", "4az", "4OV", "4Z7", "WJ", "00p", +"0yH", "Zz", "63M", "4Bf", "4lJ", "6Yi", "tV", "a7", "n6", "2mF", "6Vh", "4cK", "4Mg", "6xD", "2Cj", "02A", "1kX", "XK", "4U6", "5Pv", +"6N9", "7Ky", "2o", "0UU", "665", "73u", "Jc", "0iQ", "8Ne", "dO", "4i2", "5lr", "45k", "7Ta", "3oO", "0JM", "R2", "GS", "6jl", "5On", +"4PN", "6em", "HR", "8bx", "0EL", "24g", "6KA", "5nC", "47Z", "4f3", "kN", "8Ad", "0fP", "Eb", "aBO", "694", "62W", "byO", "0xR", "0m3", +"1D", "224", "4mP", "4x1", "6UC", "54I", "1Ko", "2nm", "VP", "C1", "4NL", "7kN", "60f", "4AM", "L0", "YQ", "3u", "0TO", "4oa", "6ZB", +"438", "4bQ", "8Pg", "zM", "Ta", "0wS", "b2F", "aSL", "6Hf", "49M", "40", "eY", "Ku", "0hG", "4Si", "6fJ", "4K8", "5Nx", "0ew", "FE", +"hi", "8BC", "4pu", "5u5", "5z4", "5oU", "0DZ", "gh", "ID", "0jv", "4QX", "4D9", "6iK", "5LI", "0gF", "Dt", "jX", "0Ij", "46L", "6Gg", +"4lF", "6Ye", "0R", "0Wh", "0yD", "Zv", "63A", "4Bj", "4OZ", "6zy", "WF", "0tt", "0ZX", "yj", "5d6", "4av", "4nw", "5k7", "2c", "0UY", +"1kT", "XG", "61p", "5Pz", "4Mk", "6xH", "Uw", "02M", "0Xi", "2mJ", "6Vd", "4cG", "0dm", "2QN", "7zA", "5Ob", "45g", "6DL", "is", "0JA", +"0Gq", "dC", "acn", "48W", "4Rs", "5W3", "Jo", "94l", "12u", "En", "5X2", "5MS", "47V", "alo", "kB", "0Hp", "1Ua", "fr", "6KM", "5nO", +"4PB", "6ea", "3Nn", "0kl", "3Pl", "01f", "bts", "7kB", "6UO", "54E", "1Kc", "xp", "1H", "0Vr", "59u", "a6e", "5F0", "4Cp", "85N", "d3F", +"Tm", "03W", "4Lq", "5I1", "434", "56t", "0Ys", "zA", "3y", "0TC", "4om", "6ZN", "60j", "4AA", "0zo", "2OL", "Kq", "0hC", "4Sm", "6fN", +"6Hb", "49I", "44", "27D", "hm", "8BG", "44y", "519", "aAl", "bdn", "0es", "FA", "1o2", "0jr", "bko", "70V", "5z0", "5oQ", "305", "gl", +"28E", "0In", "46H", "6Gc", "6iO", "5LM", "0gB", "x", "1ia", "Zr", "63E", "4Bn", "4lB", "6Ya", "0V", "0Wl", "8SD", "yn", "5d2", "4ar", +"b1e", "aPo", "WB", "00x", "1kP", "XC", "61t", "744", "4ns", "5k3", "2g", "9Ld", "0Xm", "2mN", "7FA", "4cC", "4Mo", "6xL", "Us", "02I", +"45c", "6DH", "iw", "0JE", "0di", "2QJ", "6jd", "5Of", "4Rw", "5W7", "Jk", "0iY", "0Gu", "dG", "6Ix", "48S", "47R", "6Fy", "kF", "0Ht", +"0fX", "Ej", "5X6", "5MW", "4PF", "6ee", "HZ", "0kh", "0ED", "fv", "6KI", "5nK", "6UK", "54A", "1Kg", "xt", "VX", "C9", "4ND", "7kF", +"5F4", "4Ct", "0xZ", "2My", "1L", "0Vv", "4mX", "4x9", "430", "4bY", "0Yw", "zE", "Ti", "03S", "4Lu", "5I5", "60n", "4AE", "L8", "YY", +"wu", "0TG", "4oi", "6ZJ" }; + + +#endif + diff --git a/redis.submodule/src/crc64.c b/redis.submodule/src/crc64.c index f1f7649..d4db415 100644 --- a/redis.submodule/src/crc64.c +++ b/redis.submodule/src/crc64.c @@ -1,16 +1,5 @@ -/* Redis uses the CRC64 variant with "Jones" coefficients and init value of 0. - * - * Specification of this CRC64 variant follows: - * Name: crc-64-jones - * Width: 64 bites - * Poly: 0xad93d23594c935a9 - * Reflected In: True - * Xor_In: 0xffffffffffffffff - * Reflected_Out: True - * Xor_Out: 0x0 - * Check("123456789"): 0xe9c6d914c4b8d9ca - * - * Copyright (c) 2012, Salvatore Sanfilippo +/* Copyright (c) 2014, Matt Stancliff + * Copyright (c) 2020, Amazon Web Services * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -37,147 +26,100 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ -#include - -static const uint64_t crc64_tab[256] = { - UINT64_C(0x0000000000000000), UINT64_C(0x7ad870c830358979), - UINT64_C(0xf5b0e190606b12f2), UINT64_C(0x8f689158505e9b8b), - UINT64_C(0xc038e5739841b68f), UINT64_C(0xbae095bba8743ff6), - UINT64_C(0x358804e3f82aa47d), UINT64_C(0x4f50742bc81f2d04), - UINT64_C(0xab28ecb46814fe75), UINT64_C(0xd1f09c7c5821770c), - UINT64_C(0x5e980d24087fec87), UINT64_C(0x24407dec384a65fe), - UINT64_C(0x6b1009c7f05548fa), UINT64_C(0x11c8790fc060c183), - UINT64_C(0x9ea0e857903e5a08), UINT64_C(0xe478989fa00bd371), - UINT64_C(0x7d08ff3b88be6f81), UINT64_C(0x07d08ff3b88be6f8), - UINT64_C(0x88b81eabe8d57d73), UINT64_C(0xf2606e63d8e0f40a), - UINT64_C(0xbd301a4810ffd90e), UINT64_C(0xc7e86a8020ca5077), - UINT64_C(0x4880fbd87094cbfc), UINT64_C(0x32588b1040a14285), - UINT64_C(0xd620138fe0aa91f4), UINT64_C(0xacf86347d09f188d), - UINT64_C(0x2390f21f80c18306), UINT64_C(0x594882d7b0f40a7f), - UINT64_C(0x1618f6fc78eb277b), UINT64_C(0x6cc0863448deae02), - UINT64_C(0xe3a8176c18803589), UINT64_C(0x997067a428b5bcf0), - UINT64_C(0xfa11fe77117cdf02), UINT64_C(0x80c98ebf2149567b), - UINT64_C(0x0fa11fe77117cdf0), UINT64_C(0x75796f2f41224489), - UINT64_C(0x3a291b04893d698d), UINT64_C(0x40f16bccb908e0f4), - UINT64_C(0xcf99fa94e9567b7f), UINT64_C(0xb5418a5cd963f206), - UINT64_C(0x513912c379682177), UINT64_C(0x2be1620b495da80e), - UINT64_C(0xa489f35319033385), UINT64_C(0xde51839b2936bafc), - UINT64_C(0x9101f7b0e12997f8), UINT64_C(0xebd98778d11c1e81), - UINT64_C(0x64b116208142850a), UINT64_C(0x1e6966e8b1770c73), - UINT64_C(0x8719014c99c2b083), UINT64_C(0xfdc17184a9f739fa), - UINT64_C(0x72a9e0dcf9a9a271), UINT64_C(0x08719014c99c2b08), - UINT64_C(0x4721e43f0183060c), UINT64_C(0x3df994f731b68f75), - UINT64_C(0xb29105af61e814fe), UINT64_C(0xc849756751dd9d87), - UINT64_C(0x2c31edf8f1d64ef6), UINT64_C(0x56e99d30c1e3c78f), - UINT64_C(0xd9810c6891bd5c04), UINT64_C(0xa3597ca0a188d57d), - UINT64_C(0xec09088b6997f879), UINT64_C(0x96d1784359a27100), - UINT64_C(0x19b9e91b09fcea8b), UINT64_C(0x636199d339c963f2), - UINT64_C(0xdf7adabd7a6e2d6f), UINT64_C(0xa5a2aa754a5ba416), - UINT64_C(0x2aca3b2d1a053f9d), UINT64_C(0x50124be52a30b6e4), - UINT64_C(0x1f423fcee22f9be0), UINT64_C(0x659a4f06d21a1299), - UINT64_C(0xeaf2de5e82448912), UINT64_C(0x902aae96b271006b), - UINT64_C(0x74523609127ad31a), UINT64_C(0x0e8a46c1224f5a63), - UINT64_C(0x81e2d7997211c1e8), UINT64_C(0xfb3aa75142244891), - UINT64_C(0xb46ad37a8a3b6595), UINT64_C(0xceb2a3b2ba0eecec), - UINT64_C(0x41da32eaea507767), UINT64_C(0x3b024222da65fe1e), - UINT64_C(0xa2722586f2d042ee), UINT64_C(0xd8aa554ec2e5cb97), - UINT64_C(0x57c2c41692bb501c), UINT64_C(0x2d1ab4dea28ed965), - UINT64_C(0x624ac0f56a91f461), UINT64_C(0x1892b03d5aa47d18), - UINT64_C(0x97fa21650afae693), UINT64_C(0xed2251ad3acf6fea), - UINT64_C(0x095ac9329ac4bc9b), UINT64_C(0x7382b9faaaf135e2), - UINT64_C(0xfcea28a2faafae69), UINT64_C(0x8632586aca9a2710), - UINT64_C(0xc9622c4102850a14), UINT64_C(0xb3ba5c8932b0836d), - UINT64_C(0x3cd2cdd162ee18e6), UINT64_C(0x460abd1952db919f), - UINT64_C(0x256b24ca6b12f26d), UINT64_C(0x5fb354025b277b14), - UINT64_C(0xd0dbc55a0b79e09f), UINT64_C(0xaa03b5923b4c69e6), - UINT64_C(0xe553c1b9f35344e2), UINT64_C(0x9f8bb171c366cd9b), - UINT64_C(0x10e3202993385610), UINT64_C(0x6a3b50e1a30ddf69), - UINT64_C(0x8e43c87e03060c18), UINT64_C(0xf49bb8b633338561), - UINT64_C(0x7bf329ee636d1eea), UINT64_C(0x012b592653589793), - UINT64_C(0x4e7b2d0d9b47ba97), UINT64_C(0x34a35dc5ab7233ee), - UINT64_C(0xbbcbcc9dfb2ca865), UINT64_C(0xc113bc55cb19211c), - UINT64_C(0x5863dbf1e3ac9dec), UINT64_C(0x22bbab39d3991495), - UINT64_C(0xadd33a6183c78f1e), UINT64_C(0xd70b4aa9b3f20667), - UINT64_C(0x985b3e827bed2b63), UINT64_C(0xe2834e4a4bd8a21a), - UINT64_C(0x6debdf121b863991), UINT64_C(0x1733afda2bb3b0e8), - UINT64_C(0xf34b37458bb86399), UINT64_C(0x8993478dbb8deae0), - UINT64_C(0x06fbd6d5ebd3716b), UINT64_C(0x7c23a61ddbe6f812), - UINT64_C(0x3373d23613f9d516), UINT64_C(0x49aba2fe23cc5c6f), - UINT64_C(0xc6c333a67392c7e4), UINT64_C(0xbc1b436e43a74e9d), - UINT64_C(0x95ac9329ac4bc9b5), UINT64_C(0xef74e3e19c7e40cc), - UINT64_C(0x601c72b9cc20db47), UINT64_C(0x1ac40271fc15523e), - UINT64_C(0x5594765a340a7f3a), UINT64_C(0x2f4c0692043ff643), - UINT64_C(0xa02497ca54616dc8), UINT64_C(0xdafce7026454e4b1), - UINT64_C(0x3e847f9dc45f37c0), UINT64_C(0x445c0f55f46abeb9), - UINT64_C(0xcb349e0da4342532), UINT64_C(0xb1eceec59401ac4b), - UINT64_C(0xfebc9aee5c1e814f), UINT64_C(0x8464ea266c2b0836), - UINT64_C(0x0b0c7b7e3c7593bd), UINT64_C(0x71d40bb60c401ac4), - UINT64_C(0xe8a46c1224f5a634), UINT64_C(0x927c1cda14c02f4d), - UINT64_C(0x1d148d82449eb4c6), UINT64_C(0x67ccfd4a74ab3dbf), - UINT64_C(0x289c8961bcb410bb), UINT64_C(0x5244f9a98c8199c2), - UINT64_C(0xdd2c68f1dcdf0249), UINT64_C(0xa7f41839ecea8b30), - UINT64_C(0x438c80a64ce15841), UINT64_C(0x3954f06e7cd4d138), - UINT64_C(0xb63c61362c8a4ab3), UINT64_C(0xcce411fe1cbfc3ca), - UINT64_C(0x83b465d5d4a0eece), UINT64_C(0xf96c151de49567b7), - UINT64_C(0x76048445b4cbfc3c), UINT64_C(0x0cdcf48d84fe7545), - UINT64_C(0x6fbd6d5ebd3716b7), UINT64_C(0x15651d968d029fce), - UINT64_C(0x9a0d8ccedd5c0445), UINT64_C(0xe0d5fc06ed698d3c), - UINT64_C(0xaf85882d2576a038), UINT64_C(0xd55df8e515432941), - UINT64_C(0x5a3569bd451db2ca), UINT64_C(0x20ed197575283bb3), - UINT64_C(0xc49581ead523e8c2), UINT64_C(0xbe4df122e51661bb), - UINT64_C(0x3125607ab548fa30), UINT64_C(0x4bfd10b2857d7349), - UINT64_C(0x04ad64994d625e4d), UINT64_C(0x7e7514517d57d734), - UINT64_C(0xf11d85092d094cbf), UINT64_C(0x8bc5f5c11d3cc5c6), - UINT64_C(0x12b5926535897936), UINT64_C(0x686de2ad05bcf04f), - UINT64_C(0xe70573f555e26bc4), UINT64_C(0x9ddd033d65d7e2bd), - UINT64_C(0xd28d7716adc8cfb9), UINT64_C(0xa85507de9dfd46c0), - UINT64_C(0x273d9686cda3dd4b), UINT64_C(0x5de5e64efd965432), - UINT64_C(0xb99d7ed15d9d8743), UINT64_C(0xc3450e196da80e3a), - UINT64_C(0x4c2d9f413df695b1), UINT64_C(0x36f5ef890dc31cc8), - UINT64_C(0x79a59ba2c5dc31cc), UINT64_C(0x037deb6af5e9b8b5), - UINT64_C(0x8c157a32a5b7233e), UINT64_C(0xf6cd0afa9582aa47), - UINT64_C(0x4ad64994d625e4da), UINT64_C(0x300e395ce6106da3), - UINT64_C(0xbf66a804b64ef628), UINT64_C(0xc5bed8cc867b7f51), - UINT64_C(0x8aeeace74e645255), UINT64_C(0xf036dc2f7e51db2c), - UINT64_C(0x7f5e4d772e0f40a7), UINT64_C(0x05863dbf1e3ac9de), - UINT64_C(0xe1fea520be311aaf), UINT64_C(0x9b26d5e88e0493d6), - UINT64_C(0x144e44b0de5a085d), UINT64_C(0x6e963478ee6f8124), - UINT64_C(0x21c640532670ac20), UINT64_C(0x5b1e309b16452559), - UINT64_C(0xd476a1c3461bbed2), UINT64_C(0xaeaed10b762e37ab), - UINT64_C(0x37deb6af5e9b8b5b), UINT64_C(0x4d06c6676eae0222), - UINT64_C(0xc26e573f3ef099a9), UINT64_C(0xb8b627f70ec510d0), - UINT64_C(0xf7e653dcc6da3dd4), UINT64_C(0x8d3e2314f6efb4ad), - UINT64_C(0x0256b24ca6b12f26), UINT64_C(0x788ec2849684a65f), - UINT64_C(0x9cf65a1b368f752e), UINT64_C(0xe62e2ad306bafc57), - UINT64_C(0x6946bb8b56e467dc), UINT64_C(0x139ecb4366d1eea5), - UINT64_C(0x5ccebf68aecec3a1), UINT64_C(0x2616cfa09efb4ad8), - UINT64_C(0xa97e5ef8cea5d153), UINT64_C(0xd3a62e30fe90582a), - UINT64_C(0xb0c7b7e3c7593bd8), UINT64_C(0xca1fc72bf76cb2a1), - UINT64_C(0x45775673a732292a), UINT64_C(0x3faf26bb9707a053), - UINT64_C(0x70ff52905f188d57), UINT64_C(0x0a2722586f2d042e), - UINT64_C(0x854fb3003f739fa5), UINT64_C(0xff97c3c80f4616dc), - UINT64_C(0x1bef5b57af4dc5ad), UINT64_C(0x61372b9f9f784cd4), - UINT64_C(0xee5fbac7cf26d75f), UINT64_C(0x9487ca0fff135e26), - UINT64_C(0xdbd7be24370c7322), UINT64_C(0xa10fceec0739fa5b), - UINT64_C(0x2e675fb4576761d0), UINT64_C(0x54bf2f7c6752e8a9), - UINT64_C(0xcdcf48d84fe75459), UINT64_C(0xb71738107fd2dd20), - UINT64_C(0x387fa9482f8c46ab), UINT64_C(0x42a7d9801fb9cfd2), - UINT64_C(0x0df7adabd7a6e2d6), UINT64_C(0x772fdd63e7936baf), - UINT64_C(0xf8474c3bb7cdf024), UINT64_C(0x829f3cf387f8795d), - UINT64_C(0x66e7a46c27f3aa2c), UINT64_C(0x1c3fd4a417c62355), - UINT64_C(0x935745fc4798b8de), UINT64_C(0xe98f353477ad31a7), - UINT64_C(0xa6df411fbfb21ca3), UINT64_C(0xdc0731d78f8795da), - UINT64_C(0x536fa08fdfd90e51), UINT64_C(0x29b7d047efec8728), -}; +#include "crc64.h" +#include "crcspeed.h" +static uint64_t crc64_table[8][256] = {{0}}; -uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) { - uint64_t j; +#define POLY UINT64_C(0xad93d23594c935a9) +/******************** BEGIN GENERATED PYCRC FUNCTIONS ********************/ +/** + * Generated on Sun Dec 21 14:14:07 2014, + * by pycrc v0.8.2, https://www.tty1.net/pycrc/ + * + * LICENSE ON GENERATED CODE: + * ========================== + * As of version 0.6, pycrc is released under the terms of the MIT licence. + * The code generated by pycrc is not considered a substantial portion of the + * software, therefore the author of pycrc will not claim any copyright on + * the generated code. + * ========================== + * + * CRC configuration: + * Width = 64 + * Poly = 0xad93d23594c935a9 + * XorIn = 0xffffffffffffffff + * ReflectIn = True + * XorOut = 0x0000000000000000 + * ReflectOut = True + * Algorithm = bit-by-bit-fast + * + * Modifications after generation (by matt): + * - included finalize step in-line with update for single-call generation + * - re-worked some inner variable architectures + * - adjusted function parameters to match expected prototypes. + *****************************************************************************/ + +/** + * Reflect all bits of a \a data word of \a data_len bytes. + * + * \param data The data word to be reflected. + * \param data_len The width of \a data expressed in number of bits. + * \return The reflected data. + *****************************************************************************/ +static inline uint_fast64_t crc_reflect(uint_fast64_t data, size_t data_len) { + uint_fast64_t ret = data & 0x01; + + for (size_t i = 1; i < data_len; i++) { + data >>= 1; + ret = (ret << 1) | (data & 0x01); + } + + return ret; +} + +/** + * Update the crc value with new data. + * + * \param crc The current crc value. + * \param data Pointer to a buffer of \a data_len bytes. + * \param data_len Number of bytes in the \a data buffer. + * \return The updated crc value. + ******************************************************************************/ +uint64_t _crc64(uint_fast64_t crc, const void *in_data, const uint64_t len) { + const uint8_t *data = in_data; + unsigned long long bit; - for (j = 0; j < l; j++) { - uint8_t byte = s[j]; - crc = crc64_tab[(uint8_t)crc ^ byte] ^ (crc >> 8); + for (uint64_t offset = 0; offset < len; offset++) { + uint8_t c = data[offset]; + for (uint_fast8_t i = 0x01; i & 0xff; i <<= 1) { + bit = crc & 0x8000000000000000; + if (c & i) { + bit = !bit; + } + + crc <<= 1; + if (bit) { + crc ^= POLY; + } + } + + crc &= 0xffffffffffffffff; } - return crc; + + crc = crc & 0xffffffffffffffff; + return crc_reflect(crc, 64) ^ 0x0000000000000000; +} + +/******************** END GENERATED PYCRC FUNCTIONS ********************/ + +/* Initializes the 16KB lookup tables. */ +void crc64_init(void) { + crcspeed64native_init(_crc64, crc64_table); +} + +/* Compute crc64 */ +uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) { + return crcspeed64native(crc64_table, crc, (void *) s, l); } /* Test main */ @@ -185,11 +127,35 @@ uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) { #include #define UNUSED(x) (void)(x) -int crc64Test(int argc, char *argv[]) { +int crc64Test(int argc, char *argv[], int accurate) { UNUSED(argc); UNUSED(argv); - printf("e9c6d914c4b8d9ca == %016llx\n", - (unsigned long long) crc64(0,(unsigned char*)"123456789",9)); + UNUSED(accurate); + crc64_init(); + printf("[calcula]: e9c6d914c4b8d9ca == %016" PRIx64 "\n", + (uint64_t)_crc64(0, "123456789", 9)); + printf("[64speed]: e9c6d914c4b8d9ca == %016" PRIx64 "\n", + (uint64_t)crc64(0, (unsigned char*)"123456789", 9)); + char li[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed " + "do eiusmod tempor incididunt ut labore et dolore magna " + "aliqua. Ut enim ad minim veniam, quis nostrud exercitation " + "ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis " + "aute irure dolor in reprehenderit in voluptate velit esse " + "cillum dolore eu fugiat nulla pariatur. Excepteur sint " + "occaecat cupidatat non proident, sunt in culpa qui officia " + "deserunt mollit anim id est laborum."; + printf("[calcula]: c7794709e69683b3 == %016" PRIx64 "\n", + (uint64_t)_crc64(0, li, sizeof(li))); + printf("[64speed]: c7794709e69683b3 == %016" PRIx64 "\n", + (uint64_t)crc64(0, (unsigned char*)li, sizeof(li))); return 0; } + +#endif + +#ifdef REDIS_TEST_MAIN +int main(int argc, char *argv[]) { + return crc64Test(argc, argv); +} + #endif diff --git a/redis.submodule/src/crc64.h b/redis.submodule/src/crc64.h index c9fca51..38b0b63 100644 --- a/redis.submodule/src/crc64.h +++ b/redis.submodule/src/crc64.h @@ -3,10 +3,11 @@ #include +void crc64_init(void); uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); #ifdef REDIS_TEST -int crc64Test(int argc, char *argv[]); +int crc64Test(int argc, char *argv[], int accurate); #endif #endif diff --git a/redis.submodule/src/crcspeed.c b/redis.submodule/src/crcspeed.c new file mode 100644 index 0000000..67cb8fd --- /dev/null +++ b/redis.submodule/src/crcspeed.c @@ -0,0 +1,282 @@ +/* + * Copyright (C) 2013 Mark Adler + * Originally by: crc64.c Version 1.4 16 Dec 2013 Mark Adler + * Modifications by Matt Stancliff : + * - removed CRC64-specific behavior + * - added generation of lookup tables by parameters + * - removed inversion of CRC input/result + * - removed automatic initialization in favor of explicit initialization + + This software is provided 'as-is', without any express or implied + warranty. In no event will the author be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Mark Adler + madler@alumni.caltech.edu + */ + +#include "crcspeed.h" + +/* Fill in a CRC constants table. */ +void crcspeed64little_init(crcfn64 crcfn, uint64_t table[8][256]) { + uint64_t crc; + + /* generate CRCs for all single byte sequences */ + for (int n = 0; n < 256; n++) { + unsigned char v = n; + table[0][n] = crcfn(0, &v, 1); + } + + /* generate nested CRC table for future slice-by-8 lookup */ + for (int n = 0; n < 256; n++) { + crc = table[0][n]; + for (int k = 1; k < 8; k++) { + crc = table[0][crc & 0xff] ^ (crc >> 8); + table[k][n] = crc; + } + } +} + +void crcspeed16little_init(crcfn16 crcfn, uint16_t table[8][256]) { + uint16_t crc; + + /* generate CRCs for all single byte sequences */ + for (int n = 0; n < 256; n++) { + table[0][n] = crcfn(0, &n, 1); + } + + /* generate nested CRC table for future slice-by-8 lookup */ + for (int n = 0; n < 256; n++) { + crc = table[0][n]; + for (int k = 1; k < 8; k++) { + crc = table[0][(crc >> 8) & 0xff] ^ (crc << 8); + table[k][n] = crc; + } + } +} + +/* Reverse the bytes in a 64-bit word. */ +static inline uint64_t rev8(uint64_t a) { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap64(a); +#else + uint64_t m; + + m = UINT64_C(0xff00ff00ff00ff); + a = ((a >> 8) & m) | (a & m) << 8; + m = UINT64_C(0xffff0000ffff); + a = ((a >> 16) & m) | (a & m) << 16; + return a >> 32 | a << 32; +#endif +} + +/* This function is called once to initialize the CRC table for use on a + big-endian architecture. */ +void crcspeed64big_init(crcfn64 fn, uint64_t big_table[8][256]) { + /* Create the little endian table then reverse all the entries. */ + crcspeed64little_init(fn, big_table); + for (int k = 0; k < 8; k++) { + for (int n = 0; n < 256; n++) { + big_table[k][n] = rev8(big_table[k][n]); + } + } +} + +void crcspeed16big_init(crcfn16 fn, uint16_t big_table[8][256]) { + /* Create the little endian table then reverse all the entries. */ + crcspeed16little_init(fn, big_table); + for (int k = 0; k < 8; k++) { + for (int n = 0; n < 256; n++) { + big_table[k][n] = rev8(big_table[k][n]); + } + } +} + +/* Calculate a non-inverted CRC multiple bytes at a time on a little-endian + * architecture. If you need inverted CRC, invert *before* calling and invert + * *after* calling. + * 64 bit crc = process 8 bytes at once; + */ +uint64_t crcspeed64little(uint64_t little_table[8][256], uint64_t crc, + void *buf, size_t len) { + unsigned char *next = buf; + + /* process individual bytes until we reach an 8-byte aligned pointer */ + while (len && ((uintptr_t)next & 7) != 0) { + crc = little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8); + len--; + } + + /* fast middle processing, 8 bytes (aligned!) per loop */ + while (len >= 8) { + crc ^= *(uint64_t *)next; + crc = little_table[7][crc & 0xff] ^ + little_table[6][(crc >> 8) & 0xff] ^ + little_table[5][(crc >> 16) & 0xff] ^ + little_table[4][(crc >> 24) & 0xff] ^ + little_table[3][(crc >> 32) & 0xff] ^ + little_table[2][(crc >> 40) & 0xff] ^ + little_table[1][(crc >> 48) & 0xff] ^ + little_table[0][crc >> 56]; + next += 8; + len -= 8; + } + + /* process remaining bytes (can't be larger than 8) */ + while (len) { + crc = little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8); + len--; + } + + return crc; +} + +uint16_t crcspeed16little(uint16_t little_table[8][256], uint16_t crc, + void *buf, size_t len) { + unsigned char *next = buf; + + /* process individual bytes until we reach an 8-byte aligned pointer */ + while (len && ((uintptr_t)next & 7) != 0) { + crc = little_table[0][((crc >> 8) ^ *next++) & 0xff] ^ (crc << 8); + len--; + } + + /* fast middle processing, 8 bytes (aligned!) per loop */ + while (len >= 8) { + uint64_t n = *(uint64_t *)next; + crc = little_table[7][(n & 0xff) ^ ((crc >> 8) & 0xff)] ^ + little_table[6][((n >> 8) & 0xff) ^ (crc & 0xff)] ^ + little_table[5][(n >> 16) & 0xff] ^ + little_table[4][(n >> 24) & 0xff] ^ + little_table[3][(n >> 32) & 0xff] ^ + little_table[2][(n >> 40) & 0xff] ^ + little_table[1][(n >> 48) & 0xff] ^ + little_table[0][n >> 56]; + next += 8; + len -= 8; + } + + /* process remaining bytes (can't be larger than 8) */ + while (len) { + crc = little_table[0][((crc >> 8) ^ *next++) & 0xff] ^ (crc << 8); + len--; + } + + return crc; +} + +/* Calculate a non-inverted CRC eight bytes at a time on a big-endian + * architecture. + */ +uint64_t crcspeed64big(uint64_t big_table[8][256], uint64_t crc, void *buf, + size_t len) { + unsigned char *next = buf; + + crc = rev8(crc); + while (len && ((uintptr_t)next & 7) != 0) { + crc = big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8); + len--; + } + + while (len >= 8) { + crc ^= *(uint64_t *)next; + crc = big_table[0][crc & 0xff] ^ + big_table[1][(crc >> 8) & 0xff] ^ + big_table[2][(crc >> 16) & 0xff] ^ + big_table[3][(crc >> 24) & 0xff] ^ + big_table[4][(crc >> 32) & 0xff] ^ + big_table[5][(crc >> 40) & 0xff] ^ + big_table[6][(crc >> 48) & 0xff] ^ + big_table[7][crc >> 56]; + next += 8; + len -= 8; + } + + while (len) { + crc = big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8); + len--; + } + + return rev8(crc); +} + +/* WARNING: Completely untested on big endian architecture. Possibly broken. */ +uint16_t crcspeed16big(uint16_t big_table[8][256], uint16_t crc_in, void *buf, + size_t len) { + unsigned char *next = buf; + uint64_t crc = crc_in; + + crc = rev8(crc); + while (len && ((uintptr_t)next & 7) != 0) { + crc = big_table[0][((crc >> (56 - 8)) ^ *next++) & 0xff] ^ (crc >> 8); + len--; + } + + while (len >= 8) { + uint64_t n = *(uint64_t *)next; + crc = big_table[0][(n & 0xff) ^ ((crc >> (56 - 8)) & 0xff)] ^ + big_table[1][((n >> 8) & 0xff) ^ (crc & 0xff)] ^ + big_table[2][(n >> 16) & 0xff] ^ + big_table[3][(n >> 24) & 0xff] ^ + big_table[4][(n >> 32) & 0xff] ^ + big_table[5][(n >> 40) & 0xff] ^ + big_table[6][(n >> 48) & 0xff] ^ + big_table[7][n >> 56]; + next += 8; + len -= 8; + } + + while (len) { + crc = big_table[0][((crc >> (56 - 8)) ^ *next++) & 0xff] ^ (crc >> 8); + len--; + } + + return rev8(crc); +} + +/* Return the CRC of buf[0..len-1] with initial crc, processing eight bytes + at a time using passed-in lookup table. + This selects one of two routines depending on the endianess of + the architecture. */ +uint64_t crcspeed64native(uint64_t table[8][256], uint64_t crc, void *buf, + size_t len) { + uint64_t n = 1; + + return *(char *)&n ? crcspeed64little(table, crc, buf, len) + : crcspeed64big(table, crc, buf, len); +} + +uint16_t crcspeed16native(uint16_t table[8][256], uint16_t crc, void *buf, + size_t len) { + uint64_t n = 1; + + return *(char *)&n ? crcspeed16little(table, crc, buf, len) + : crcspeed16big(table, crc, buf, len); +} + +/* Initialize CRC lookup table in architecture-dependent manner. */ +void crcspeed64native_init(crcfn64 fn, uint64_t table[8][256]) { + uint64_t n = 1; + + *(char *)&n ? crcspeed64little_init(fn, table) + : crcspeed64big_init(fn, table); +} + +void crcspeed16native_init(crcfn16 fn, uint16_t table[8][256]) { + uint64_t n = 1; + + *(char *)&n ? crcspeed16little_init(fn, table) + : crcspeed16big_init(fn, table); +} diff --git a/redis.submodule/src/crcspeed.h b/redis.submodule/src/crcspeed.h new file mode 100644 index 0000000..d7ee95e --- /dev/null +++ b/redis.submodule/src/crcspeed.h @@ -0,0 +1,60 @@ +/* Copyright (c) 2014, Matt Stancliff + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. */ + +#ifndef CRCSPEED_H +#define CRCSPEED_H + +#include +#include + +typedef uint64_t (*crcfn64)(uint64_t, const void *, const uint64_t); +typedef uint16_t (*crcfn16)(uint16_t, const void *, const uint64_t); + +/* CRC-64 */ +void crcspeed64little_init(crcfn64 fn, uint64_t table[8][256]); +void crcspeed64big_init(crcfn64 fn, uint64_t table[8][256]); +void crcspeed64native_init(crcfn64 fn, uint64_t table[8][256]); + +uint64_t crcspeed64little(uint64_t table[8][256], uint64_t crc, void *buf, + size_t len); +uint64_t crcspeed64big(uint64_t table[8][256], uint64_t crc, void *buf, + size_t len); +uint64_t crcspeed64native(uint64_t table[8][256], uint64_t crc, void *buf, + size_t len); + +/* CRC-16 */ +void crcspeed16little_init(crcfn16 fn, uint16_t table[8][256]); +void crcspeed16big_init(crcfn16 fn, uint16_t table[8][256]); +void crcspeed16native_init(crcfn16 fn, uint16_t table[8][256]); + +uint16_t crcspeed16little(uint16_t table[8][256], uint16_t crc, void *buf, + size_t len); +uint16_t crcspeed16big(uint16_t table[8][256], uint16_t crc, void *buf, + size_t len); +uint16_t crcspeed16native(uint16_t table[8][256], uint16_t crc, void *buf, + size_t len); +#endif diff --git a/redis.submodule/src/db.c b/redis.submodule/src/db.c index 62c8aa1..c8d6199 100644 --- a/redis.submodule/src/db.c +++ b/redis.submodule/src/db.c @@ -30,10 +30,18 @@ #include "server.h" #include "cluster.h" #include "atomicvar.h" +#include "latency.h" #include #include +/* Database backup. */ +struct dbBackup { + redisDb *dbarray; + rax *slots_to_keys; + uint64_t slots_keys_count[CLUSTER_SLOTS]; +}; + /*----------------------------------------------------------------------------- * C-level DB API *----------------------------------------------------------------------------*/ @@ -60,10 +68,7 @@ robj *lookupKey(redisDb *db, robj *key, int flags) { /* Update the access time for the ageing algorithm. * Don't do it if we have a saving child, as this will trigger * a copy on write madness. */ - if (server.rdb_child_pid == -1 && - server.aof_child_pid == -1 && - !(flags & LOOKUP_NOTOUCH)) - { + if (!hasActiveChildProcess() && !(flags & LOOKUP_NOTOUCH)){ if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { updateLFU(val); } else { @@ -83,6 +88,7 @@ robj *lookupKey(redisDb *db, robj *key, int flags) { * 1. A key gets expired if it reached it's TTL. * 2. The key last access time is updated. * 3. The global keys hits/misses stats are updated (reported in INFO). + * 4. If keyspace notifications are enabled, a "keymiss" notification is fired. * * This API should not be used when we write to the key after obtaining * the object linked to the key, but only for read only operations. @@ -101,13 +107,10 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) { robj *val; if (expireIfNeeded(db,key) == 1) { - /* Key expired. If we are in the context of a master, expireIfNeeded() - * returns 0 only when the key does not exist at all, so it's safe - * to return NULL ASAP. */ - if (server.masterhost == NULL) { - server.stat_keyspace_misses++; - return NULL; - } + /* If we are in the context of a master, expireIfNeeded() returns 1 + * when the key is no longer valid, so we can return NULL ASAP. */ + if (server.masterhost == NULL) + goto keymiss; /* However if we are in the context of a slave, expireIfNeeded() will * not really try to expire the key, it only returns information @@ -117,7 +120,7 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) { * However, if the command caller is not the master, and as additional * safety measure, the command invoked is a read-only command, we can * safely return NULL here, and provide a more consistent behavior - * to clients accessign expired values in a read-only fashion, that + * to clients accessing expired values in a read-only fashion, that * will say the key as non existing. * * Notably this covers GETs when slaves are used to scale reads. */ @@ -126,16 +129,21 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) { server.current_client->cmd && server.current_client->cmd->flags & CMD_READONLY) { - server.stat_keyspace_misses++; - return NULL; + goto keymiss; } } val = lookupKey(db,key,flags); if (val == NULL) - server.stat_keyspace_misses++; - else - server.stat_keyspace_hits++; + goto keymiss; + server.stat_keyspace_hits++; return val; + +keymiss: + if (!(flags & LOOKUP_NONOTIFY)) { + notifyKeyspaceEvent(NOTIFY_KEY_MISS, "keymiss", key, db->id); + } + server.stat_keyspace_misses++; + return NULL; } /* Like lookupKeyReadWithFlags(), but does not use any flag, which is the @@ -149,20 +157,32 @@ robj *lookupKeyRead(redisDb *db, robj *key) { * * Returns the linked value object if the key exists or NULL if the key * does not exist in the specified DB. */ -robj *lookupKeyWrite(redisDb *db, robj *key) { +robj *lookupKeyWriteWithFlags(redisDb *db, robj *key, int flags) { expireIfNeeded(db,key); - return lookupKey(db,key,LOOKUP_NONE); + return lookupKey(db,key,flags); } +robj *lookupKeyWrite(redisDb *db, robj *key) { + return lookupKeyWriteWithFlags(db, key, LOOKUP_NONE); +} +void SentReplyOnKeyMiss(client *c, robj *reply){ + serverAssert(sdsEncodedObject(reply)); + sds rep = reply->ptr; + if (sdslen(rep) > 1 && rep[0] == '-'){ + addReplyErrorObject(c, reply); + } else { + addReply(c,reply); + } +} robj *lookupKeyReadOrReply(client *c, robj *key, robj *reply) { robj *o = lookupKeyRead(c->db, key); - if (!o) addReply(c,reply); + if (!o) SentReplyOnKeyMiss(c, reply); return o; } robj *lookupKeyWriteOrReply(client *c, robj *key, robj *reply) { robj *o = lookupKeyWrite(c->db, key); - if (!o) addReply(c,reply); + if (!o) SentReplyOnKeyMiss(c, reply); return o; } @@ -175,10 +195,26 @@ void dbAdd(redisDb *db, robj *key, robj *val) { int retval = dictAdd(db->dict, copy, val); serverAssertWithInfo(NULL,key,retval == DICT_OK); - if (val->type == OBJ_LIST || - val->type == OBJ_ZSET) - signalKeyAsReady(db, key); + signalKeyAsReady(db, key, val->type); + if (server.cluster_enabled) slotToKeyAdd(key->ptr); +} + +/* This is a special version of dbAdd() that is used only when loading + * keys from the RDB file: the key is passed as an SDS string that is + * retained by the function (and not freed by the caller). + * + * Moreover this function will not abort if the key is already busy, to + * give more control to the caller, nor will signal the key as ready + * since it is not useful in this context. + * + * The function returns 1 if the key was added to the database, taking + * ownership of the SDS string, otherwise 0 is returned, and is up to the + * caller to free the SDS string. */ +int dbAddRDBLoad(redisDb *db, sds key, robj *val) { + int retval = dictAdd(db->dict, key, val); + if (retval != DICT_OK) return 0; if (server.cluster_enabled) slotToKeyAdd(key); + return 1; } /* Overwrite an existing key with a new value. Incrementing the reference @@ -195,10 +231,14 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) { if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { val->lru = old->lru; } + /* Although the key is not really deleted from the database, we regard + overwrite as two steps of unlink+add, so we still need to call the unlink + callback of the module. */ + moduleNotifyKeyUnlink(key,old); dictSetVal(db->dict, de, val); if (server.lazyfree_lazy_server_del) { - freeObjAsync(old); + freeObjAsync(key,old); dictSetVal(db->dict, &auxentry, NULL); } @@ -210,22 +250,26 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) { * * 1) The ref count of the value object is incremented. * 2) clients WATCHing for the destination key notified. - * 3) The expire time of the key is reset (the key is made persistent). + * 3) The expire time of the key is reset (the key is made persistent), + * unless 'keepttl' is true. * - * All the new keys in the database should be created via this interface. */ -void setKey(redisDb *db, robj *key, robj *val) { + * All the new keys in the database should be created via this interface. + * The client 'c' argument may be set to NULL if the operation is performed + * in a context where there is no clear client performing the operation. */ +void genericSetKey(client *c, redisDb *db, robj *key, robj *val, int keepttl, int signal) { if (lookupKeyWrite(db,key) == NULL) { dbAdd(db,key,val); } else { dbOverwrite(db,key,val); } incrRefCount(val); - removeExpire(db,key); - signalModifiedKey(db,key); + if (!keepttl) removeExpire(db,key); + if (signal) signalModifiedKey(c,db,key); } -int dbExists(redisDb *db, robj *key) { - return dictFind(db->dict,key->ptr) != NULL; +/* Common case for genericSetKey() where the TTL is not retained. */ +void setKey(client *c, redisDb *db, robj *key, robj *val) { + genericSetKey(c,db,key,val,0,1); } /* Return a random key, in form of a Redis object. @@ -241,7 +285,7 @@ robj *dbRandomKey(redisDb *db) { sds key; robj *keyobj; - de = dictGetRandomKey(db->dict); + de = dictGetFairRandomKey(db->dict); if (de == NULL) return NULL; key = dictGetKey(de); @@ -272,8 +316,13 @@ int dbSyncDelete(redisDb *db, robj *key) { /* Deleting an entry from the expires dict will not free the sds of * the key, because it is shared with the main dictionary. */ if (dictSize(db->expires) > 0) dictDelete(db->expires,key->ptr); - if (dictDelete(db->dict,key->ptr) == DICT_OK) { - if (server.cluster_enabled) slotToKeyDel(key); + dictEntry *de = dictUnlink(db->dict,key->ptr); + if (de) { + robj *val = dictGetVal(de); + /* Tells the module that the key has been unlinked from the database. */ + moduleNotifyKeyUnlink(key,val); + dictFreeUnlinkedEntry(db->dict,de); + if (server.cluster_enabled) slotToKeyDel(key->ptr); return 1; } else { return 0; @@ -325,6 +374,41 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) { return o; } +/* Remove all keys from the database(s) structure. The dbarray argument + * may not be the server main DBs (could be a backup). + * + * The dbnum can be -1 if all the DBs should be emptied, or the specified + * DB index if we want to empty only a single database. + * The function returns the number of keys removed from the database(s). */ +long long emptyDbStructure(redisDb *dbarray, int dbnum, int async, + void(callback)(void*)) +{ + long long removed = 0; + int startdb, enddb; + + if (dbnum == -1) { + startdb = 0; + enddb = server.dbnum-1; + } else { + startdb = enddb = dbnum; + } + + for (int j = startdb; j <= enddb; j++) { + removed += dictSize(dbarray[j].dict); + if (async) { + emptyDbAsync(&dbarray[j]); + } else { + dictEmpty(dbarray[j].dict,callback); + dictEmpty(dbarray[j].expires,callback); + } + /* Because all keys of database are removed, reset average ttl. */ + dbarray[j].avg_ttl = 0; + dbarray[j].expires_cursor = 0; + } + + return removed; +} + /* Remove all keys from all the databases in a Redis server. * If callback is given the function is called from time to time to * signal that work is in progress. @@ -336,11 +420,12 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) { * EMPTYDB_ASYNC if we want the memory to be freed in a different thread * and the function to return ASAP. * - * On success the fuction returns the number of keys removed from the + * On success the function returns the number of keys removed from the * database(s). Otherwise -1 is returned in the specific case the * DB number is out of range, and errno is set to EINVAL. */ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { int async = (flags & EMPTYDB_ASYNC); + RedisModuleFlushInfoV1 fi = {REDISMODULE_FLUSHINFO_VERSION,!async,dbnum}; long long removed = 0; if (dbnum < -1 || dbnum >= server.dbnum) { @@ -348,32 +433,119 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { return -1; } - int startdb, enddb; - if (dbnum == -1) { - startdb = 0; - enddb = server.dbnum-1; - } else { - startdb = enddb = dbnum; + /* Fire the flushdb modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, + REDISMODULE_SUBEVENT_FLUSHDB_START, + &fi); + + /* Make sure the WATCHed keys are affected by the FLUSH* commands. + * Note that we need to call the function while the keys are still + * there. */ + signalFlushedDb(dbnum, async); + + /* Empty redis database structure. */ + removed = emptyDbStructure(server.db, dbnum, async, callback); + + /* Flush slots to keys map if enable cluster, we can flush entire + * slots to keys map whatever dbnum because only support one DB + * in cluster mode. */ + if (server.cluster_enabled) slotToKeyFlush(async); + + if (dbnum == -1) flushSlaveKeysWithExpireList(); + + /* Also fire the end event. Note that this event will fire almost + * immediately after the start event if the flush is asynchronous. */ + moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, + REDISMODULE_SUBEVENT_FLUSHDB_END, + &fi); + + return removed; +} + +/* Store a backup of the database for later use, and put an empty one + * instead of it. */ +dbBackup *backupDb(void) { + dbBackup *backup = zmalloc(sizeof(dbBackup)); + + /* Backup main DBs. */ + backup->dbarray = zmalloc(sizeof(redisDb)*server.dbnum); + for (int i=0; idbarray[i] = server.db[i]; + server.db[i].dict = dictCreate(&dbDictType,NULL); + server.db[i].expires = dictCreate(&dbExpiresDictType,NULL); } - for (int j = startdb; j <= enddb; j++) { - removed += dictSize(server.db[j].dict); - if (async) { - emptyDbAsync(&server.db[j]); - } else { - dictEmpty(server.db[j].dict,callback); - dictEmpty(server.db[j].expires,callback); - } + /* Backup cluster slots to keys map if enable cluster. */ + if (server.cluster_enabled) { + backup->slots_to_keys = server.cluster->slots_to_keys; + memcpy(backup->slots_keys_count, server.cluster->slots_keys_count, + sizeof(server.cluster->slots_keys_count)); + server.cluster->slots_to_keys = raxNew(); + memset(server.cluster->slots_keys_count, 0, + sizeof(server.cluster->slots_keys_count)); + } + + moduleFireServerEvent(REDISMODULE_EVENT_REPL_BACKUP, + REDISMODULE_SUBEVENT_REPL_BACKUP_CREATE, + NULL); + + return backup; +} + +/* Discard a previously created backup, this can be slow (similar to FLUSHALL) + * Arguments are similar to the ones of emptyDb, see EMPTYDB_ flags. */ +void discardDbBackup(dbBackup *buckup, int flags, void(callback)(void*)) { + int async = (flags & EMPTYDB_ASYNC); + + /* Release main DBs backup . */ + emptyDbStructure(buckup->dbarray, -1, async, callback); + for (int i=0; idbarray[i].dict); + dictRelease(buckup->dbarray[i].expires); + } + + /* Release slots to keys map backup if enable cluster. */ + if (server.cluster_enabled) freeSlotsToKeysMap(buckup->slots_to_keys, async); + + /* Release buckup. */ + zfree(buckup->dbarray); + zfree(buckup); + + moduleFireServerEvent(REDISMODULE_EVENT_REPL_BACKUP, + REDISMODULE_SUBEVENT_REPL_BACKUP_DISCARD, + NULL); +} + +/* Restore the previously created backup (discarding what currently resides + * in the db). + * This function should be called after the current contents of the database + * was emptied with a previous call to emptyDb (possibly using the async mode). */ +void restoreDbBackup(dbBackup *buckup) { + /* Restore main DBs. */ + for (int i=0; idbarray[i]; } + + /* Restore slots to keys map backup if enable cluster. */ if (server.cluster_enabled) { - if (async) { - slotToKeyFlushAsync(); - } else { - slotToKeyFlush(); - } + serverAssert(server.cluster->slots_to_keys->numele == 0); + raxFree(server.cluster->slots_to_keys); + server.cluster->slots_to_keys = buckup->slots_to_keys; + memcpy(server.cluster->slots_keys_count, buckup->slots_keys_count, + sizeof(server.cluster->slots_keys_count)); } - if (dbnum == -1) flushSlaveKeysWithExpireList(); - return removed; + + /* Release buckup. */ + zfree(buckup->dbarray); + zfree(buckup); + + moduleFireServerEvent(REDISMODULE_EVENT_REPL_BACKUP, + REDISMODULE_SUBEVENT_REPL_BACKUP_RESTORE, + NULL); } int selectDb(client *c, int id) { @@ -383,6 +555,15 @@ int selectDb(client *c, int id) { return C_OK; } +long long dbTotalServerKeyCount() { + long long total = 0; + int j; + for (j = 0; j < server.dbnum; j++) { + total += dictSize(server.db[j].dict); + } + return total; +} + /*----------------------------------------------------------------------------- * Hooks for key space changes. * @@ -392,12 +573,27 @@ int selectDb(client *c, int id) { * Every time a DB is flushed the function signalFlushDb() is called. *----------------------------------------------------------------------------*/ -void signalModifiedKey(redisDb *db, robj *key) { +/* Note that the 'c' argument may be NULL if the key was modified out of + * a context of a client. */ +void signalModifiedKey(client *c, redisDb *db, robj *key) { touchWatchedKey(db,key); + trackingInvalidateKey(c,key); } -void signalFlushedDb(int dbid) { - touchWatchedKeysOnFlush(dbid); +void signalFlushedDb(int dbid, int async) { + int startdb, enddb; + if (dbid == -1) { + startdb = 0; + enddb = server.dbnum-1; + } else { + startdb = enddb = dbid; + } + + for (int j = startdb; j <= enddb; j++) { + touchAllWatchedKeysInDb(&server.db[j], NULL); + } + + trackingInvalidateKeysOnFlush(async); } /*----------------------------------------------------------------------------- @@ -407,25 +603,53 @@ void signalFlushedDb(int dbid) { /* Return the set of flags to use for the emptyDb() call for FLUSHALL * and FLUSHDB commands. * - * Currently the command just attempts to parse the "ASYNC" option. It - * also checks if the command arity is wrong. + * sync: flushes the database in an sync manner. + * async: flushes the database in an async manner. + * no option: determine sync or async according to the value of lazyfree-lazy-user-flush. * * On success C_OK is returned and the flags are stored in *flags, otherwise * C_ERR is returned and the function sends an error to the client. */ int getFlushCommandFlags(client *c, int *flags) { /* Parse the optional ASYNC option. */ - if (c->argc > 1) { - if (c->argc > 2 || strcasecmp(c->argv[1]->ptr,"async")) { - addReply(c,shared.syntaxerr); - return C_ERR; - } + if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"sync")) { + *flags = EMPTYDB_NO_FLAGS; + } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"async")) { *flags = EMPTYDB_ASYNC; + } else if (c->argc == 1) { + *flags = server.lazyfree_lazy_user_flush ? EMPTYDB_ASYNC : EMPTYDB_NO_FLAGS; } else { - *flags = EMPTYDB_NO_FLAGS; + addReplyErrorObject(c,shared.syntaxerr); + return C_ERR; } return C_OK; } +/* Flushes the whole server data set. */ +void flushAllDataAndResetRDB(int flags) { + server.dirty += emptyDb(-1,flags,NULL); + if (server.child_type == CHILD_TYPE_RDB) killRDBChild(); + if (server.saveparamslen > 0) { + /* Normally rdbSave() will reset dirty, but we don't want this here + * as otherwise FLUSHALL will not be replicated nor put into the AOF. */ + int saved_dirty = server.dirty; + rdbSaveInfo rsi, *rsiptr; + rsiptr = rdbPopulateSaveInfo(&rsi); + rdbSave(server.rdb_filename,rsiptr); + server.dirty = saved_dirty; + } + + /* Without that extra dirty++, when db was already empty, FLUSHALL will + * not be replicated nor put into the AOF. */ + server.dirty++; +#if defined(USE_JEMALLOC) + /* jemalloc 5 doesn't release pages back to the OS when there's no traffic. + * for large databases, flushdb blocks for long anyway, so a bit more won't + * harm and this way the flush and purge will be synchroneus. */ + if (!(flags & EMPTYDB_ASYNC)) + jemalloc_purge(); +#endif +} + /* FLUSHDB [ASYNC] * * Flushes the currently SELECTed Redis DB. */ @@ -433,9 +657,15 @@ void flushdbCommand(client *c) { int flags; if (getFlushCommandFlags(c,&flags) == C_ERR) return; - signalFlushedDb(c->db->id); server.dirty += emptyDb(c->db->id,flags,NULL); addReply(c,shared.ok); +#if defined(USE_JEMALLOC) + /* jemalloc 5 doesn't release pages back to the OS when there's no traffic. + * for large databases, flushdb blocks for long anyway, so a bit more won't + * harm and this way the flush and purge will be synchroneus. */ + if (!(flags & EMPTYDB_ASYNC)) + jemalloc_purge(); +#endif } /* FLUSHALL [ASYNC] @@ -443,25 +673,9 @@ void flushdbCommand(client *c) { * Flushes the whole server data set. */ void flushallCommand(client *c) { int flags; - if (getFlushCommandFlags(c,&flags) == C_ERR) return; - signalFlushedDb(-1); - server.dirty += emptyDb(-1,flags,NULL); + flushAllDataAndResetRDB(flags); addReply(c,shared.ok); - if (server.rdb_child_pid != -1) { - kill(server.rdb_child_pid,SIGUSR1); - rdbRemoveTempFile(server.rdb_child_pid); - } - if (server.saveparamslen > 0) { - /* Normally rdbSave() will reset dirty, but we don't want this here - * as otherwise FLUSHALL will not be replicated nor put into the AOF. */ - int saved_dirty = server.dirty; - rdbSaveInfo rsi, *rsiptr; - rsiptr = rdbPopulateSaveInfo(&rsi); - rdbSave(server.rdb_filename,rsiptr); - server.dirty = saved_dirty; - } - server.dirty++; } /* This command implements DEL and LAZYDEL. */ @@ -473,7 +687,7 @@ void delGenericCommand(client *c, int lazy) { int deleted = lazy ? dbAsyncDelete(c->db,c->argv[j]) : dbSyncDelete(c->db,c->argv[j]); if (deleted) { - signalModifiedKey(c->db,c->argv[j]); + signalModifiedKey(c,c->db,c->argv[j]); notifyKeyspaceEvent(NOTIFY_GENERIC, "del",c->argv[j],c->db->id); server.dirty++; @@ -484,7 +698,7 @@ void delGenericCommand(client *c, int lazy) { } void delCommand(client *c) { - delGenericCommand(c,0); + delGenericCommand(c,server.lazyfree_lazy_user_del); } void unlinkCommand(client *c) { @@ -498,16 +712,15 @@ void existsCommand(client *c) { int j; for (j = 1; j < c->argc; j++) { - if (lookupKeyRead(c->db,c->argv[j])) count++; + if (lookupKeyReadWithFlags(c->db,c->argv[j],LOOKUP_NOTOUCH)) count++; } addReplyLongLong(c,count); } void selectCommand(client *c) { - long id; + int id; - if (getLongFromObjectOrReply(c, c->argv[1], &id, - "invalid DB index") != C_OK) + if (getIntFromObjectOrReply(c, c->argv[1], &id, NULL) != C_OK) return; if (server.cluster_enabled && id != 0) { @@ -525,7 +738,7 @@ void randomkeyCommand(client *c) { robj *key; if ((key = dbRandomKey(c->db)) == NULL) { - addReply(c,shared.nullbulk); + addReplyNull(c); return; } @@ -539,10 +752,10 @@ void keysCommand(client *c) { sds pattern = c->argv[1]->ptr; int plen = sdslen(pattern), allkeys; unsigned long numkeys = 0; - void *replylen = addDeferredMultiBulkLength(c); + void *replylen = addReplyDeferredLen(c); di = dictGetSafeIterator(c->db->dict); - allkeys = (pattern[0] == '*' && pattern[1] == '\0'); + allkeys = (pattern[0] == '*' && plen == 1); while((de = dictNext(di)) != NULL) { sds key = dictGetKey(de); robj *keyobj; @@ -557,7 +770,7 @@ void keysCommand(client *c) { } } dictReleaseIterator(di); - setDeferredMultiBulkLength(c,replylen,numkeys); + setDeferredArrayLen(c,replylen,numkeys); } /* This callback is used by scanGenericCommand in order to collect elements @@ -611,7 +824,7 @@ int parseScanCursorOrReply(client *c, robj *o, unsigned long *cursor) { } /* This command implements SCAN, HSCAN and SSCAN commands. - * If object 'o' is passed, then it must be a Hash or Set object, otherwise + * If object 'o' is passed, then it must be a Hash, Set or Zset object, otherwise * if 'o' is NULL the command will operate on the dictionary associated with * the current database. * @@ -627,6 +840,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { listNode *node, *nextnode; long count = 10; sds pat = NULL; + sds typename = NULL; int patlen = 0, use_pattern = 0; dict *ht; @@ -649,7 +863,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { } if (count < 1) { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); goto cleanup; } @@ -663,8 +877,12 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { use_pattern = !(pat[0] == '*' && patlen == 1); i += 2; + } else if (!strcasecmp(c->argv[i]->ptr, "type") && o == NULL && j >= 2) { + /* SCAN for a particular type only applies to the db dict */ + typename = c->argv[i+1]->ptr; + i+= 2; } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); goto cleanup; } } @@ -743,7 +961,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { int filter = 0; /* Filter element if it does not match the pattern. */ - if (!filter && use_pattern) { + if (use_pattern) { if (sdsEncodedObject(kobj)) { if (!stringmatchlen(pat, patlen, kobj->ptr, sdslen(kobj->ptr), 0)) filter = 1; @@ -757,10 +975,17 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { } } + /* Filter an element if it isn't the type we want. */ + if (!filter && o == NULL && typename){ + robj* typecheck = lookupKeyReadWithFlags(c->db, kobj, LOOKUP_NOTOUCH); + char* type = getObjectTypeName(typecheck); + if (strcasecmp((char*) typename, type)) filter = 1; + } + /* Filter element if it is an expired key. */ if (!filter && o == NULL && expireIfNeeded(c->db, kobj)) filter = 1; - /* Remove the element and its associted value if needed. */ + /* Remove the element and its associated value if needed. */ if (filter) { decrRefCount(kobj); listDelNode(keys, node); @@ -771,6 +996,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { * value, or skip it if it was not filtered: we only match keys. */ if (o && (o->type == OBJ_ZSET || o->type == OBJ_HASH)) { node = nextnode; + serverAssert(node); /* assertion for valgrind (avoid NPD) */ nextnode = listNextNode(node); if (filter) { kobj = listNodeValue(node); @@ -782,10 +1008,10 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { } /* Step 4: Reply to the client. */ - addReplyMultiBulkLen(c, 2); + addReplyArrayLen(c, 2); addReplyBulkLongLong(c,cursor); - addReplyMultiBulkLen(c, listLength(keys)); + addReplyArrayLen(c, listLength(keys)); while ((node = listFirst(keys)) != NULL) { robj *kobj = listNodeValue(node); addReplyBulk(c, kobj); @@ -813,11 +1039,8 @@ void lastsaveCommand(client *c) { addReplyLongLong(c,server.lastsave); } -void typeCommand(client *c) { - robj *o; - char *type; - - o = lookupKeyReadWithFlags(c->db,c->argv[1],LOOKUP_NOTOUCH); +char* getObjectTypeName(robj *o) { + char* type; if (o == NULL) { type = "none"; } else { @@ -835,14 +1058,20 @@ void typeCommand(client *c) { default: type = "unknown"; break; } } - addReplyStatus(c,type); + return type; +} + +void typeCommand(client *c) { + robj *o; + o = lookupKeyReadWithFlags(c->db,c->argv[1],LOOKUP_NOTOUCH); + addReplyStatus(c, getObjectTypeName(o)); } void shutdownCommand(client *c) { int flags = 0; if (c->argc > 2) { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } else if (c->argc == 2) { if (!strcasecmp(c->argv[1]->ptr,"nosave")) { @@ -850,18 +1079,10 @@ void shutdownCommand(client *c) { } else if (!strcasecmp(c->argv[1]->ptr,"save")) { flags |= SHUTDOWN_SAVE; } else { - addReply(c,shared.syntaxerr); + addReplyErrorObject(c,shared.syntaxerr); return; } } - /* When SHUTDOWN is called while the server is loading a dataset in - * memory we need to make sure no attempt is performed to save - * the dataset on shutdown (otherwise it could overwrite the current DB - * with half-read data). - * - * Also when in Sentinel mode clear the SAVE flag and force NOSAVE. */ - if (server.loading || server.sentinel_mode) - flags = (flags & ~SHUTDOWN_SAVE) | SHUTDOWN_NOSAVE; if (prepareForShutdown(flags) == C_OK) exit(0); addReplyError(c,"Errors trying to SHUTDOWN. Check logs."); } @@ -898,8 +1119,8 @@ void renameGenericCommand(client *c, int nx) { dbAdd(c->db,c->argv[2],o); if (expire != -1) setExpire(c,c->db,c->argv[2],expire); dbDelete(c->db,c->argv[1]); - signalModifiedKey(c->db,c->argv[1]); - signalModifiedKey(c->db,c->argv[2]); + signalModifiedKey(c,c->db,c->argv[1]); + signalModifiedKey(c,c->db,c->argv[2]); notifyKeyspaceEvent(NOTIFY_GENERIC,"rename_from", c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_GENERIC,"rename_to", @@ -919,8 +1140,8 @@ void renamenxCommand(client *c) { void moveCommand(client *c) { robj *o; redisDb *src, *dst; - int srcid; - long long dbid, expire; + int srcid, dbid; + long long expire; if (server.cluster_enabled) { addReplyError(c,"MOVE is not allowed in cluster mode"); @@ -931,11 +1152,11 @@ void moveCommand(client *c) { src = c->db; srcid = c->db->id; - if (getLongLongFromObject(c->argv[2],&dbid) == C_ERR || - dbid < INT_MIN || dbid > INT_MAX || - selectDb(c,dbid) == C_ERR) - { - addReply(c,shared.outofrangeerr); + if (getIntFromObjectOrReply(c, c->argv[2], &dbid, NULL) != C_OK) + return; + + if (selectDb(c,dbid) == C_ERR) { + addReplyError(c,"DB index is out of range"); return; } dst = c->db; @@ -944,7 +1165,7 @@ void moveCommand(client *c) { /* If the user is moving using as target the same * DB as the source DB it is probably an error. */ if (src == dst) { - addReply(c,shared.sameobjecterr); + addReplyErrorObject(c,shared.sameobjecterr); return; } @@ -967,6 +1188,115 @@ void moveCommand(client *c) { /* OK! key moved, free the entry in the source DB */ dbDelete(src,c->argv[1]); + signalModifiedKey(c,src,c->argv[1]); + signalModifiedKey(c,dst,c->argv[1]); + notifyKeyspaceEvent(NOTIFY_GENERIC, + "move_from",c->argv[1],src->id); + notifyKeyspaceEvent(NOTIFY_GENERIC, + "move_to",c->argv[1],dst->id); + + server.dirty++; + addReply(c,shared.cone); +} + +void copyCommand(client *c) { + robj *o; + redisDb *src, *dst; + int srcid, dbid; + long long expire; + int j, replace = 0, delete = 0; + + /* Obtain source and target DB pointers + * Default target DB is the same as the source DB + * Parse the REPLACE option and targetDB option. */ + src = c->db; + dst = c->db; + srcid = c->db->id; + dbid = c->db->id; + for (j = 3; j < c->argc; j++) { + int additional = c->argc - j - 1; + if (!strcasecmp(c->argv[j]->ptr,"replace")) { + replace = 1; + } else if (!strcasecmp(c->argv[j]->ptr, "db") && additional >= 1) { + if (getIntFromObjectOrReply(c, c->argv[j+1], &dbid, NULL) != C_OK) + return; + + if (selectDb(c, dbid) == C_ERR) { + addReplyError(c,"DB index is out of range"); + return; + } + dst = c->db; + selectDb(c,srcid); /* Back to the source DB */ + j++; /* Consume additional arg. */ + } else { + addReplyErrorObject(c,shared.syntaxerr); + return; + } + } + + if ((server.cluster_enabled == 1) && (srcid != 0 || dbid != 0)) { + addReplyError(c,"Copying to another database is not allowed in cluster mode"); + return; + } + + /* If the user select the same DB as + * the source DB and using newkey as the same key + * it is probably an error. */ + robj *key = c->argv[1]; + robj *newkey = c->argv[2]; + if (src == dst && (sdscmp(key->ptr, newkey->ptr) == 0)) { + addReplyErrorObject(c,shared.sameobjecterr); + return; + } + + /* Check if the element exists and get a reference */ + o = lookupKeyWrite(c->db, key); + if (!o) { + addReply(c,shared.czero); + return; + } + expire = getExpire(c->db,key); + + /* Return zero if the key already exists in the target DB. + * If REPLACE option is selected, delete newkey from targetDB. */ + if (lookupKeyWrite(dst,newkey) != NULL) { + if (replace) { + delete = 1; + } else { + addReply(c,shared.czero); + return; + } + } + + /* Duplicate object according to object's type. */ + robj *newobj; + switch(o->type) { + case OBJ_STRING: newobj = dupStringObject(o); break; + case OBJ_LIST: newobj = listTypeDup(o); break; + case OBJ_SET: newobj = setTypeDup(o); break; + case OBJ_ZSET: newobj = zsetDup(o); break; + case OBJ_HASH: newobj = hashTypeDup(o); break; + case OBJ_STREAM: newobj = streamDup(o); break; + case OBJ_MODULE: + newobj = moduleTypeDupOrReply(c, key, newkey, o); + if (!newobj) return; + break; + default: + addReplyError(c, "unknown type object"); + return; + } + + if (delete) { + dbDelete(dst,newkey); + } + + dbAdd(dst,newkey,newobj); + if (expire != -1) setExpire(c, dst, newkey, expire); + + /* OK! key copied */ + signalModifiedKey(c,dst,c->argv[2]); + notifyKeyspaceEvent(NOTIFY_GENERIC,"copy_to",c->argv[2],dst->id); + server.dirty++; addReply(c,shared.cone); } @@ -981,10 +1311,7 @@ void scanDatabaseForReadyLists(redisDb *db) { while((de = dictNext(di)) != NULL) { robj *key = dictGetKey(de); robj *value = lookupKey(db,key,LOOKUP_NOTOUCH); - if (value && (value->type == OBJ_LIST || - value->type == OBJ_STREAM || - value->type == OBJ_ZSET)) - signalKeyAsReady(db, key); + if (value) signalKeyAsReady(db, key, value->type); } dictReleaseIterator(di); } @@ -1010,10 +1337,12 @@ int dbSwapDatabases(int id1, int id2) { db1->dict = db2->dict; db1->expires = db2->expires; db1->avg_ttl = db2->avg_ttl; + db1->expires_cursor = db2->expires_cursor; db2->dict = aux.dict; db2->expires = aux.expires; db2->avg_ttl = aux.avg_ttl; + db2->expires_cursor = aux.expires_cursor; /* Now we need to handle clients blocked on lists: as an effect * of swapping the two DBs, a client that was waiting for list @@ -1023,15 +1352,20 @@ int dbSwapDatabases(int id1, int id2) { * However normally we only do this check for efficiency reasons * in dbAdd() when a list is created. So here we need to rescan * the list of clients blocked on lists and signal lists as ready - * if needed. */ + * if needed. + * + * Also the swapdb should make transaction fail if there is any + * client watching keys */ scanDatabaseForReadyLists(db1); + touchAllWatchedKeysInDb(db1, db2); scanDatabaseForReadyLists(db2); + touchAllWatchedKeysInDb(db2, db1); return C_OK; } /* SWAPDB db1 db2 */ void swapdbCommand(client *c) { - long id1, id2; + int id1, id2; /* Not allowed in cluster mode: we have just DB 0 there. */ if (server.cluster_enabled) { @@ -1040,11 +1374,11 @@ void swapdbCommand(client *c) { } /* Get the two DBs indexes. */ - if (getLongFromObjectOrReply(c, c->argv[1], &id1, + if (getIntFromObjectOrReply(c, c->argv[1], &id1, "invalid first DB index") != C_OK) return; - if (getLongFromObjectOrReply(c, c->argv[2], &id2, + if (getIntFromObjectOrReply(c, c->argv[2], &id2, "invalid second DB index") != C_OK) return; @@ -1053,6 +1387,8 @@ void swapdbCommand(client *c) { addReplyError(c,"DB index is out of range"); return; } else { + RedisModuleSwapDbInfo si = {REDISMODULE_SWAPDBINFO_VERSION,id1,id2}; + moduleFireServerEvent(REDISMODULE_EVENT_SWAPDB,0,&si); server.dirty++; addReply(c,shared.ok); } @@ -1102,6 +1438,22 @@ long long getExpire(redisDb *db, robj *key) { return dictGetSignedIntegerVal(de); } +/* Delete the specified expired key and propagate expire. */ +void deleteExpiredKeyAndPropagate(redisDb *db, robj *keyobj) { + mstime_t expire_latency; + latencyStartMonitor(expire_latency); + if (server.lazyfree_lazy_expire) + dbAsyncDelete(db,keyobj); + else + dbSyncDelete(db,keyobj); + latencyEndMonitor(expire_latency); + latencyAddSampleIfNeeded("expire-del",expire_latency); + notifyKeyspaceEvent(NOTIFY_EXPIRED,"expired",keyobj,db->id); + signalModifiedKey(NULL, db, keyobj); + propagateExpire(db,keyobj,server.lazyfree_lazy_expire); + server.stat_expiredkeys++; +} + /* Propagate expires into slaves and the AOF file. * When a key expires in the master, a DEL operation for this key is sent * to all the slaves and the AOF file if enabled. @@ -1118,9 +1470,12 @@ void propagateExpire(redisDb *db, robj *key, int lazy) { incrRefCount(argv[0]); incrRefCount(argv[1]); - if (server.aof_state != AOF_OFF) - feedAppendOnlyFile(server.delCommand,db->id,argv,2); - replicationFeedSlaves(server.slaves,db->id,argv,2); + /* If the master decided to expire a key we must propagate it to replicas no matter what.. + * Even if module executed a command without asking for propagation. */ + int prev_replication_allowed = server.replication_allowed; + server.replication_allowed = 1; + propagate(server.delCommand,db->id,argv,2,PROPAGATE_AOF|PROPAGATE_REPL); + server.replication_allowed = prev_replication_allowed; decrRefCount(argv[0]); decrRefCount(argv[1]); @@ -1129,6 +1484,7 @@ void propagateExpire(redisDb *db, robj *key, int lazy) { /* Check if the key is expired. */ int keyIsExpired(redisDb *db, robj *key) { mstime_t when = getExpire(db,key); + mstime_t now; if (when < 0) return 0; /* No expire for this key */ @@ -1140,8 +1496,26 @@ int keyIsExpired(redisDb *db, robj *key) { * only the first time it is accessed and not in the middle of the * script execution, making propagation to slaves / AOF consistent. * See issue #1525 on Github for more information. */ - mstime_t now = server.lua_caller ? server.lua_time_start : mstime(); + if (server.lua_caller) { + now = server.lua_time_snapshot; + } + /* If we are in the middle of a command execution, we still want to use + * a reference time that does not change: in that case we just use the + * cached time, that we update before each call in the call() function. + * This way we avoid that commands such as RPOPLPUSH or similar, that + * may re-open the same key multiple times, can invalidate an already + * open object in a next call, if the next call will see the key expired, + * while the first did not. */ + else if (server.fixed_time_expire > 0) { + now = server.mstime; + } + /* For the other cases, we want to use the most fresh time we have. */ + else { + now = mstime(); + } + /* The key expired if the current (virtual or real) time is greater + * than the expire time of the key. */ return now > when; } @@ -1177,33 +1551,69 @@ int expireIfNeeded(redisDb *db, robj *key) { * we think the key is expired at this time. */ if (server.masterhost != NULL) return 1; + /* If clients are paused, we keep the current dataset constant, + * but return to the client what we believe is the right state. Typically, + * at the end of the pause we will properly expire the key OR we will + * have failed over and the new primary will send us the expire. */ + if (checkClientPauseTimeoutAndReturnIfPaused()) return 1; + /* Delete the key */ - server.stat_expiredkeys++; - propagateExpire(db,key,server.lazyfree_lazy_expire); - notifyKeyspaceEvent(NOTIFY_EXPIRED, - "expired",key,db->id); - return server.lazyfree_lazy_expire ? dbAsyncDelete(db,key) : - dbSyncDelete(db,key); + deleteExpiredKeyAndPropagate(db,key); + return 1; } /* ----------------------------------------------------------------------------- * API to get key arguments from commands * ---------------------------------------------------------------------------*/ +/* Prepare the getKeysResult struct to hold numkeys, either by using the + * pre-allocated keysbuf or by allocating a new array on the heap. + * + * This function must be called at least once before starting to populate + * the result, and can be called repeatedly to enlarge the result array. + */ +int *getKeysPrepareResult(getKeysResult *result, int numkeys) { + /* GETKEYS_RESULT_INIT initializes keys to NULL, point it to the pre-allocated stack + * buffer here. */ + if (!result->keys) { + serverAssert(!result->numkeys); + result->keys = result->keysbuf; + } + + /* Resize if necessary */ + if (numkeys > result->size) { + if (result->keys != result->keysbuf) { + /* We're not using a static buffer, just (re)alloc */ + result->keys = zrealloc(result->keys, numkeys * sizeof(int)); + } else { + /* We are using a static buffer, copy its contents */ + result->keys = zmalloc(numkeys * sizeof(int)); + if (result->numkeys) + memcpy(result->keys, result->keysbuf, result->numkeys * sizeof(int)); + } + result->size = numkeys; + } + + return result->keys; +} + /* The base case is to use the keys position as given in the command table * (firstkey, lastkey, step). */ -int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, int *numkeys) { +int getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, getKeysResult *result) { int j, i = 0, last, *keys; UNUSED(argv); if (cmd->firstkey == 0) { - *numkeys = 0; - return NULL; + result->numkeys = 0; + return 0; } last = cmd->lastkey; if (last < 0) last = argc+last; - keys = zmalloc(sizeof(int)*((last - cmd->firstkey)+1)); + + int count = ((last - cmd->firstkey)+1); + keys = getKeysPrepareResult(result, count); + for (j = cmd->firstkey; j <= last; j += cmd->keystep) { if (j >= argc) { /* Modules commands, and standard commands with a not fixed number @@ -1213,23 +1623,22 @@ int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, in * return no keys and expect the command implementation to report * an arity or syntax error. */ if (cmd->flags & CMD_MODULE || cmd->arity < 0) { - zfree(keys); - *numkeys = 0; - return NULL; + result->numkeys = 0; + return 0; } else { serverPanic("Redis built-in command declared keys positions not matching the arity requirements."); } } keys[i++] = j; } - *numkeys = i; - return keys; + result->numkeys = i; + return i; } /* Return all the arguments that are keys in the command passed via argc / argv. * * The command returns the positions of all the key arguments inside the array, - * so the actual return value is an heap allocated array of integers. The + * so the actual return value is a heap allocated array of integers. The * length of the array is returned by reference into *numkeys. * * 'cmd' must be point to the corresponding entry into the redisCommand @@ -1237,72 +1646,70 @@ int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, in * * This function uses the command table if a command-specific helper function * is not required, otherwise it calls the command-specific function. */ -int *getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) { +int getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result) { if (cmd->flags & CMD_MODULE_GETKEYS) { - return moduleGetCommandKeysViaAPI(cmd,argv,argc,numkeys); + return moduleGetCommandKeysViaAPI(cmd,argv,argc,result); } else if (!(cmd->flags & CMD_MODULE) && cmd->getkeys_proc) { - return cmd->getkeys_proc(cmd,argv,argc,numkeys); + return cmd->getkeys_proc(cmd,argv,argc,result); } else { - return getKeysUsingCommandTable(cmd,argv,argc,numkeys); + return getKeysUsingCommandTable(cmd,argv,argc,result); } } /* Free the result of getKeysFromCommand. */ -void getKeysFreeResult(int *result) { - zfree(result); +void getKeysFreeResult(getKeysResult *result) { + if (result && result->keys != result->keysbuf) + zfree(result->keys); } /* Helper function to extract keys from following commands: + * COMMAND [destkey] [...] [...] ... + * + * eg: + * ZUNION ... * ZUNIONSTORE ... - * ZINTERSTORE ... */ -int *zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) { + * + * 'storeKeyOfs': destkey index, 0 means destkey not exists. + * 'keyCountOfs': num-keys index. + * 'firstKeyOfs': firstkey index. + * 'keyStep': the interval of each key, usually this value is 1. + * */ +int genericGetKeys(int storeKeyOfs, int keyCountOfs, int firstKeyOfs, int keyStep, + robj **argv, int argc, getKeysResult *result) { int i, num, *keys; - UNUSED(cmd); - num = atoi(argv[2]->ptr); + num = atoi(argv[keyCountOfs]->ptr); /* Sanity check. Don't return any key if the command is going to - * reply with syntax error. */ - if (num < 1 || num > (argc-3)) { - *numkeys = 0; - return NULL; + * reply with syntax error. (no input keys). */ + if (num < 1 || num > (argc - firstKeyOfs)/keyStep) { + result->numkeys = 0; + return 0; } - /* Keys in z{union,inter}store come from two places: - * argv[1] = storage key, - * argv[3...n] = keys to intersect */ - keys = zmalloc(sizeof(int)*(num+1)); + int numkeys = storeKeyOfs ? num + 1 : num; + keys = getKeysPrepareResult(result, numkeys); + result->numkeys = numkeys; - /* Add all key positions for argv[3...n] to keys[] */ - for (i = 0; i < num; i++) keys[i] = 3+i; + /* Add all key positions for argv[firstKeyOfs...n] to keys[] */ + for (i = 0; i < num; i++) keys[i] = firstKeyOfs+(i*keyStep); - /* Finally add the argv[1] key position (the storage key target). */ - keys[num] = 1; - *numkeys = num+1; /* Total keys = {union,inter} keys + storage key */ - return keys; + if (storeKeyOfs) keys[num] = storeKeyOfs; + return result->numkeys; } -/* Helper function to extract keys from the following commands: - * EVAL