From f20faf61721b2e704b0d9e16997eef01d19d5905 Mon Sep 17 00:00:00 2001 From: Dwight Hubbard Date: Wed, 17 May 2017 16:27:07 -0700 Subject: [PATCH] Update to redis version 3.2.9 --- redis.submodule/00-RELEASENOTES | 170 ++++++++++++++++++ redis.submodule/src/anet.c | 3 +- redis.submodule/src/aof.c | 2 + redis.submodule/src/cluster.c | 35 +++- redis.submodule/src/debug.c | 2 +- redis.submodule/src/geo.c | 2 +- redis.submodule/src/networking.c | 26 ++- redis.submodule/src/object.c | 6 +- redis.submodule/src/redis-cli.c | 23 ++- redis.submodule/src/replication.c | 3 +- redis.submodule/src/scripting.c | 1 - redis.submodule/src/server.c | 3 + redis.submodule/src/server.h | 1 + redis.submodule/src/t_zset.c | 2 +- redis.submodule/src/version.h | 2 +- redis.submodule/src/ziplist.c | 38 ++-- redis.submodule/src/zmalloc.c | 5 +- .../tests/integration/replication-psync.tcl | 2 +- 18 files changed, 279 insertions(+), 47 deletions(-) diff --git a/redis.submodule/00-RELEASENOTES b/redis.submodule/00-RELEASENOTES index badc38a..72cfd62 100644 --- a/redis.submodule/00-RELEASENOTES +++ b/redis.submodule/00-RELEASENOTES @@ -10,6 +10,176 @@ HIGH: There is a critical bug that may affect a subset of users. Upgrade! CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. -------------------------------------------------------------------------------- +================================================================================ +Redis 3.2.9 Released Mon May 17 17:35:38 CEST 2017 +================================================================================ + +Upgrade urgency LOW: A few rarely harmful bugs were fixed. + +This release just fixes bugs that are unlikely to cause serious problems +so there is no need to update ASAP. Please, see the list of commits +for the details on the bugs fixed and credits: + +antirez in commit 3b46cf97: + redis-cli --bigkeys: show error when TYPE fails. + 1 file changed, 7 insertions(+), 2 deletions(-) + +antirez in commit f59b4b93: + Fix preprocessor if/else chain broken in order to fix #3927. + 1 file changed, 3 insertions(+) + +antirez in commit dd80fedf: + Fix zmalloc_get_memory_size() ifdefs to actually use the else branch. + 1 file changed, 2 deletions(-) + +antirez in commit 697d3abe: + Set lua-time-limit default value at safe place. + 2 files changed, 1 insertion(+), 1 deletion(-) + +antirez in commit c9c04b11: + Fix #3848 by closing the descriptor on error. + 1 file changed, 2 insertions(+), 1 deletion(-) + +张文康 in commit d3b49924: + update block->free after some diff data are written to the child process + 1 file changed, 1 insertion(+) + +antirez in commit 6a33952b: + Test: fix, hopefully, false PSYNC failure like in issue #2715. + 1 file changed, 1 insertion(+), 1 deletion(-) + +John.Koepi in commit b83f9fea: + fix #2883, #2857 pipe fds leak when fork() failed on bg aof rw + 1 file changed, 1 insertion(+) + +antirez in commit 10dbb5cd: + Don't leak file descriptor on syncWithMaster(). + 1 file changed, 2 insertions(+), 1 deletion(-) + +================================================================================ +Redis 3.2.8 Released Sun Feb 12 16:11:18 CET 2017 +================================================================================ + +Upgrade urgency CRITICAL: This release reverts back the Jemalloc upgrade + that is believed to potentially cause a server + deadlock. A MIGRATE crash is also fixed. + +Two important bug fixes, the first of one is critical: + +1. Apparently Jemalloc 4.4.0 may contain a deadlock under particular + conditions. See https://github.com/antirez/redis/issues/3799. + We reverted back to the previously used Jemalloc versions and plan + to upgrade Jemalloc again after having more info about the + cause of the bug. + +2. MIGRATE could crash the server after a socket error. See for reference: + https://github.com/antirez/redis/issues/3796. + +List of commits: + +antirez in commit 7178cac: + Revert "Jemalloc updated to 4.4.0." + 150 files changed, 6330 insertions(+), 17245 deletions(-) + +antirez in commit 33fad43: + Fix MIGRATE closing of cached socket on error. + 1 file changed, 23 insertions(+), 6 deletions(-) + +================================================================================ +Redis 3.2.7 Released Tue Jan 31 16:21:41 CET 2017 +================================================================================ + +Upgrade urgency HIGH: This release fixes important security and correctness + issues. It is especially important to upgrade for Redis + Cluster users and for users running Redis in their laptop + since a cross-scripting attack is fixed in this release. + +Main bugs fixes and improvements in this release: + +1. MIGRATE could incorrectly move keys between Redis Cluster nodes by turning + keys with an expire set into persisting keys. This bug was introduced with + the multiple-keys migration recently. It is now fixed. Only applies to + Redis Cluster users that use the resharding features of Redis Cluster. + +2. As Redis 4.0 beta and the unstable branch already did (for some months at + this point), Redis 3.2.7 also aliases the Host: and POST commands to QUIT + avoiding to process the remaining pipeline if there are pending commands. + This is a security protection against a "Cross Scripting" attack, that + usually involves trying to feed Redis with HTTP in order to execute commands. + Example: a developer is running a local copy of Redis for development + purposes. She also runs a web browser in the same computer. The web browser + could send an HTTP request to http://127.0.0.1:6379 in order to access the + Redis instance, since a specially crafted HTTP requesta may also be partially + valid Redis protocol. However if POST and Host: break the connection, this + problem should be avoided. IMPORTANT: It is important to realize that it + is not impossible that another way will be found to talk with a localhost + Redis using a Cross Protocol attack not involving sending POST or Host: so + this is only a layer of protection but not a definitive fix for this class + of issues. + +3. A ziplist bug that could cause data corruption, could crash the server and + MAY ALSO HAVE SECURITY IMPLICATIONS was fixed. The bug looks complex to + exploit, but attacks always get worse, never better (cit). The bug is very + very hard to catch in practice, it required manual analysis of the ziplist + code in order to be found. However it is also possible that rarely it + happened in the wild. Upgrading is required if you use LINSERT and other + in-the-middle list manipulation commands. + +4. We upgraded to Jemalloc 4.4.0 since the version we used to ship with Redis + was an early 4.0 release of Jemalloc. This version may have several + improvements including the ability to better reclaim/use the memory of + system. + +The following is the list of commits: + +antirez in commit 3876d98: + Ziplist: insertion bug under particular conditions fixed. + 1 file changed, 9 insertions(+), 1 deletion(-) + +antirez in commit 153f2f0: + Jemalloc updated to 4.4.0. + 150 files changed, 17271 insertions(+), 6356 deletions(-) + +miter in commit ca532c9: + Change switch statment to if statment + 1 file changed, 2 insertions(+), 4 deletions(-) + +oranagra in commit a735035: + fix rare assertion in DEBUG DIGEST + 1 file changed, 1 insertion(+), 1 deletion(-) + +Itamar Haber in commit b917e3f: + Verify pairs are provided after subcommands + 1 file changed, 1 insertion(+), 1 deletion(-) + +antirez in commit 1177cf6: + Avoid geo.c warning in initialization. + 1 file changed, 1 insertion(+), 1 deletion(-) + +antirez in commit 874804d: + Security: Cross Protocol Scripting protection. + 3 files changed, 27 insertions(+), 2 deletions(-) + +antirez in commit 273cd7f: + Ziplist: remove static from functions, they prevent good crash reports. + 1 file changed, 14 insertions(+), 14 deletions(-) + +Jan-Erik Rediger in commit 389b9f5: + Initialize help only in repl mode + 1 file changed, 5 insertions(+), 5 deletions(-) + +Yossi Gottlieb in commit 1370a88: + Fix redis-cli rare crash. + 1 file changed, 4 insertions(+) + +antirez in commit 68aab8e: + MIGRATE: Remove upfront ttl initialization. + 1 file changed, 3 insertions(+), 4 deletions(-) + +Jan-Erik Rediger in commit 788e892: + Reset the ttl for additional keys + 1 file changed, 1 insertion(+) + ================================================================================ Redis 3.2.6 Released Tue Dec 06 09:33:29 CET 2016 ================================================================================ diff --git a/redis.submodule/src/anet.c b/redis.submodule/src/anet.c index ef1711d..e16ce13 100644 --- a/redis.submodule/src/anet.c +++ b/redis.submodule/src/anet.c @@ -462,7 +462,7 @@ static int anetV6Only(char *err, int s) { static int _anetTcpServer(char *err, int port, char *bindaddr, int af, int backlog) { - int s, rv; + int s = -1, rv; char _port[6]; /* strlen("65535") */ struct addrinfo hints, *servinfo, *p; @@ -491,6 +491,7 @@ static int _anetTcpServer(char *err, int port, char *bindaddr, int af, int backl } error: + if (s != -1) close(s); s = ANET_ERR; end: freeaddrinfo(servinfo); diff --git a/redis.submodule/src/aof.c b/redis.submodule/src/aof.c index c6fbbb8..cbd906c 100644 --- a/redis.submodule/src/aof.c +++ b/redis.submodule/src/aof.c @@ -115,6 +115,7 @@ void aofChildWriteDiffData(aeEventLoop *el, int fd, void *privdata, int mask) { if (nwritten <= 0) return; memmove(block->buf,block->buf+nwritten,block->used-nwritten); block->used -= nwritten; + block->free += nwritten; } if (block->used == 0) listDelNode(server.aof_rewrite_buf_blocks,ln); } @@ -1295,6 +1296,7 @@ int rewriteAppendOnlyFileBackground(void) { serverLog(LL_WARNING, "Can't rewrite append only file in background: fork: %s", strerror(errno)); + aofClosePipes(); return C_ERR; } serverLog(LL_NOTICE, diff --git a/redis.submodule/src/cluster.c b/redis.submodule/src/cluster.c index 2dd3344..c245090 100644 --- a/redis.submodule/src/cluster.c +++ b/redis.submodule/src/cluster.c @@ -4652,13 +4652,13 @@ void migrateCommand(client *c) { int copy, replace, j; long timeout; long dbid; - long long ttl, expireat; robj **ov = NULL; /* Objects to migrate. */ robj **kv = NULL; /* Key names. */ robj **newargv = NULL; /* Used to rewrite the command as DEL ... keys ... */ rio cmd, payload; int may_retry = 1; int write_error = 0; + int argv_rewritten = 0; /* To support the KEYS option we need the following additional state. */ int first_key = 3; /* Argument index of the first key. */ @@ -4667,7 +4667,6 @@ void migrateCommand(client *c) { /* Initialization */ copy = 0; replace = 0; - ttl = 0; /* Parse additional options */ for (j = 6; j < c->argc; j++) { @@ -4743,7 +4742,9 @@ void migrateCommand(client *c) { /* Create RESTORE payload and generate the protocol to call the command. */ for (j = 0; j < num_keys; j++) { - expireat = getExpire(c->db,kv[j]); + long long ttl = 0; + long long expireat = getExpire(c->db,kv[j]); + if (expireat != -1) { ttl = expireat-mstime(); if (ttl < 1) ttl = 1; @@ -4841,12 +4842,20 @@ void migrateCommand(client *c) { goto socket_err; /* A retry is guaranteed because of tested conditions.*/ } + /* On socket errors, close the migration socket now that we still have + * the original host/port in the ARGV. Later the original command may be + * rewritten to DEL and will be too later. */ + if (socket_error) migrateCloseSocket(c->argv[1],c->argv[2]); + if (!copy) { - /* Translate MIGRATE as DEL for replication/AOF. */ + /* Translate MIGRATE as DEL for replication/AOF. Note that we do + * this only for the keys for which we received an acknowledgement + * from the receiving Redis server, by using the del_idx index. */ if (del_idx > 1) { newargv[0] = createStringObject("DEL",3); /* Note that the following call takes ownership of newargv. */ replaceClientCommandVector(c,del_idx,newargv); + argv_rewritten = 1; } else { /* No key transfer acknowledged, no need to rewrite as DEL. */ zfree(newargv); @@ -4855,8 +4864,8 @@ void migrateCommand(client *c) { } /* If we are here and a socket error happened, we don't want to retry. - * Just signal the problem to the client, but only do it if we don't - * already queued a different error reported by the destination server. */ + * Just signal the problem to the client, but only do it if we did not + * already queue a different error reported by the destination server. */ if (!error_from_target && socket_error) { may_retry = 0; goto socket_err; @@ -4864,7 +4873,11 @@ void migrateCommand(client *c) { if (!error_from_target) { /* Success! Update the last_dbid in migrateCachedSocket, so that we can - * avoid SELECT the next time if the target DB is the same. Reply +OK. */ + * avoid SELECT the next time if the target DB is the same. Reply +OK. + * + * Note: If we reached this point, even if socket_error is true + * still the SELECT command succeeded (otherwise the code jumps to + * socket_err label. */ cs->last_dbid = dbid; addReply(c,shared.ok); } else { @@ -4874,7 +4887,6 @@ void migrateCommand(client *c) { sdsfree(cmd.io.buffer.ptr); zfree(ov); zfree(kv); zfree(newargv); - if (socket_error) migrateCloseSocket(c->argv[1],c->argv[2]); return; /* On socket errors we try to close the cached socket and try again. @@ -4884,7 +4896,12 @@ void migrateCommand(client *c) { /* Cleanup we want to perform in both the retry and no retry case. * Note: Closing the migrate socket will also force SELECT next time. */ sdsfree(cmd.io.buffer.ptr); - migrateCloseSocket(c->argv[1],c->argv[2]); + + /* If the command was rewritten as DEL and there was a socket error, + * we already closed the socket earlier. While migrateCloseSocket() + * is idempotent, the host/port arguments are now gone, so don't do it + * again. */ + if (!argv_rewritten) migrateCloseSocket(c->argv[1],c->argv[2]); zfree(newargv); newargv = NULL; /* This will get reallocated on retry. */ diff --git a/redis.submodule/src/debug.c b/redis.submodule/src/debug.c index a97400e..1ecb251 100644 --- a/redis.submodule/src/debug.c +++ b/redis.submodule/src/debug.c @@ -126,7 +126,7 @@ void computeDatasetDigest(unsigned char *final) { redisDb *db = server.db+j; if (dictSize(db->dict) == 0) continue; - di = dictGetIterator(db->dict); + di = dictGetSafeIterator(db->dict); /* hash the DB id, so the same dataset moved in a different * DB will lead to a different digest */ diff --git a/redis.submodule/src/geo.c b/redis.submodule/src/geo.c index f2ae8c8..e428fff 100644 --- a/redis.submodule/src/geo.c +++ b/redis.submodule/src/geo.c @@ -352,7 +352,7 @@ int membersOfAllNeighbors(robj *zobj, GeoHashRadius n, double lon, double lat, d if (debugmsg) { GeoHashRange long_range, lat_range; geohashGetCoordRange(&long_range,&lat_range); - GeoHashArea myarea = {{0}}; + GeoHashArea myarea = {{0,0},{0,0},{0,0}}; geohashDecode(long_range, lat_range, neighbors[i], &myarea); /* Dump center square. */ diff --git a/redis.submodule/src/networking.c b/redis.submodule/src/networking.c index 90e64cb..fb5341e 100644 --- a/redis.submodule/src/networking.c +++ b/redis.submodule/src/networking.c @@ -1269,8 +1269,10 @@ void processInputBuffer(client *c) { /* CLIENT_CLOSE_AFTER_REPLY closes the connection once the reply is * written to the client. Make sure to not let the reply grow after - * this flag has been set (i.e. don't process more commands). */ - if (c->flags & CLIENT_CLOSE_AFTER_REPLY) break; + * this flag has been set (i.e. don't process more commands). + * + * The same applies for clients we want to terminate ASAP. */ + if (c->flags & (CLIENT_CLOSE_AFTER_REPLY|CLIENT_CLOSE_ASAP)) break; /* Determine request type when unknown. */ if (!c->reqtype) { @@ -1637,6 +1639,26 @@ void clientCommand(client *c) { } } +/* This callback is bound to POST and "Host:" command names. Those are not + * really commands, but are used in security attacks in order to talk to + * Redis instances via HTTP, with a technique called "cross protocol scripting" + * which exploits the fact that services like Redis will discard invalid + * HTTP headers and will process what follows. + * + * As a protection against this attack, Redis will terminate the connection + * when a POST or "Host:" header is seen, and will log the event from + * time to time (to avoid creating a DOS as a result of too many logs). */ +void securityWarningCommand(client *c) { + static time_t logged_time; + time_t now = time(NULL); + + if (labs(now-logged_time) > 60) { + serverLog(LL_WARNING,"Possible SECURITY ATTACK detected. It looks like somebody is sending POST or Host: commands to Redis. This is likely due to an attacker attempting to use Cross Protocol Scripting to compromise your Redis instance. Connection aborted."); + logged_time = now; + } + freeClientAsync(c); +} + /* Rewrite the command vector of the client. All the new objects ref count * is incremented. The old command vector is freed, and the old objects * ref count is decremented. */ diff --git a/redis.submodule/src/object.c b/redis.submodule/src/object.c index 22d2222..db5ec3a 100644 --- a/redis.submodule/src/object.c +++ b/redis.submodule/src/object.c @@ -241,11 +241,9 @@ void freeStringObject(robj *o) { } void freeListObject(robj *o) { - switch (o->encoding) { - case OBJ_ENCODING_QUICKLIST: + if (o->encoding == OBJ_ENCODING_QUICKLIST) { quicklistRelease(o->ptr); - break; - default: + } else { serverPanic("Unknown list encoding type"); } } diff --git a/redis.submodule/src/redis-cli.c b/redis.submodule/src/redis-cli.c index 9043915..40f9176 100644 --- a/redis.submodule/src/redis-cli.c +++ b/redis.submodule/src/redis-cli.c @@ -275,6 +275,10 @@ static void cliIntegrateHelp(void) { * don't already match what we have. */ for (size_t j = 0; j < reply->elements; j++) { redisReply *entry = reply->element[j]; + if (entry->type != REDIS_REPLY_ARRAY || entry->elements < 4 || + entry->element[0]->type != REDIS_REPLY_STRING || + entry->element[1]->type != REDIS_REPLY_INTEGER || + entry->element[3]->type != REDIS_REPLY_INTEGER) return; char *cmdname = entry->element[0]->str; int i; @@ -1267,6 +1271,11 @@ static void repl(void) { int argc; sds *argv; + /* Initialize the help and, if possible, use the COMMAND command in order + * to retrieve missing entries. */ + cliInitHelp(); + cliIntegrateHelp(); + config.interactive = 1; linenoiseSetMultiLine(1); linenoiseSetCompletionCallback(completionCallback); @@ -2013,8 +2022,13 @@ static void getKeyTypes(redisReply *keys, int *types) { keys->element[i]->str, context->err, context->errstr); exit(1); } else if(reply->type != REDIS_REPLY_STATUS) { - fprintf(stderr, "Invalid reply type (%d) for TYPE on key '%s'!\n", - reply->type, keys->element[i]->str); + if(reply->type == REDIS_REPLY_ERROR) { + fprintf(stderr, "TYPE returned an error: %s\n", reply->str); + } else { + fprintf(stderr, + "Invalid reply type (%d) for TYPE on key '%s'!\n", + reply->type, keys->element[i]->str); + } exit(1); } @@ -2597,11 +2611,6 @@ int main(int argc, char **argv) { argc -= firstarg; argv += firstarg; - /* Initialize the help and, if possible, use the COMMAND command in order - * to retrieve missing entries. */ - cliInitHelp(); - cliIntegrateHelp(); - /* Latency mode */ if (config.latency_mode) { if (cliConnect(0) == REDIS_ERR) exit(1); diff --git a/redis.submodule/src/replication.c b/redis.submodule/src/replication.c index e7194e5..8d9bcfb 100644 --- a/redis.submodule/src/replication.c +++ b/redis.submodule/src/replication.c @@ -1385,7 +1385,7 @@ int slaveTryPartialResynchronization(int fd, int read_reply) { void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { char tmpfile[256], *err = NULL; - int dfd, maxtries = 5; + int dfd = -1, maxtries = 5; int sockerr = 0, psync_result; socklen_t errlen = sizeof(sockerr); UNUSED(el); @@ -1640,6 +1640,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { error: aeDeleteFileEvent(server.el,fd,AE_READABLE|AE_WRITABLE); + if (dfd != -1) close(dfd); close(fd); server.repl_transfer_s = -1; server.repl_state = REPL_STATE_CONNECT; diff --git a/redis.submodule/src/scripting.c b/redis.submodule/src/scripting.c index 0a2ce1f..216dfbf 100644 --- a/redis.submodule/src/scripting.c +++ b/redis.submodule/src/scripting.c @@ -900,7 +900,6 @@ void scriptingInit(int setup) { server.lua_caller = NULL; server.lua_timedout = 0; server.lua_always_replicate_commands = 0; /* Only DEBUG can change it.*/ - server.lua_time_limit = LUA_SCRIPT_TIME_LIMIT; ldbInit(); } diff --git a/redis.submodule/src/server.c b/redis.submodule/src/server.c index 408d03f..609f396 100644 --- a/redis.submodule/src/server.c +++ b/redis.submodule/src/server.c @@ -294,6 +294,8 @@ struct redisCommand redisCommandTable[] = { {"pfcount",pfcountCommand,-2,"r",0,NULL,1,-1,1,0,0}, {"pfmerge",pfmergeCommand,-2,"wm",0,NULL,1,-1,1,0,0}, {"pfdebug",pfdebugCommand,-3,"w",0,NULL,0,0,0,0,0}, + {"post",securityWarningCommand,-1,"lt",0,NULL,0,0,0,0,0}, + {"host:",securityWarningCommand,-1,"lt",0,NULL,0,0,0,0,0}, {"latency",latencyCommand,-2,"aslt",0,NULL,0,0,0,0,0} }; @@ -1531,6 +1533,7 @@ void initServerConfig(void) { server.migrate_cached_sockets = dictCreate(&migrateCacheDictType,NULL); server.next_client_id = 1; /* Client IDs, start from 1 .*/ server.loading_process_events_interval_bytes = (1024*1024*2); + server.lua_time_limit = LUA_SCRIPT_TIME_LIMIT; server.lruclock = getLRUClock(); resetServerSaveParams(); diff --git a/redis.submodule/src/server.h b/redis.submodule/src/server.h index 1a26ec4..3fa7c3a 100644 --- a/redis.submodule/src/server.h +++ b/redis.submodule/src/server.h @@ -1645,6 +1645,7 @@ void pfcountCommand(client *c); void pfmergeCommand(client *c); void pfdebugCommand(client *c); void latencyCommand(client *c); +void securityWarningCommand(client *c); #if defined(__GNUC__) void *calloc(size_t count, size_t size) __attribute__ ((deprecated)); diff --git a/redis.submodule/src/t_zset.c b/redis.submodule/src/t_zset.c index 64901ef..372e0d9 100644 --- a/redis.submodule/src/t_zset.c +++ b/redis.submodule/src/t_zset.c @@ -1247,7 +1247,7 @@ void zaddGenericCommand(client *c, int flags) { /* After the options, we expect to have an even number of args, since * we expect any number of score-element pairs. */ elements = c->argc-scoreidx; - if (elements % 2) { + if (elements % 2 || !elements) { addReply(c,shared.syntaxerr); return; } diff --git a/redis.submodule/src/version.h b/redis.submodule/src/version.h index eb14a3d..4d7dbc2 100644 --- a/redis.submodule/src/version.h +++ b/redis.submodule/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "3.2.6" +#define REDIS_VERSION "3.2.9" diff --git a/redis.submodule/src/ziplist.c b/redis.submodule/src/ziplist.c index 7428d30..110b1a8 100644 --- a/redis.submodule/src/ziplist.c +++ b/redis.submodule/src/ziplist.c @@ -180,7 +180,7 @@ typedef struct zlentry { void ziplistRepr(unsigned char *zl); /* Return bytes needed to store integer encoded by 'encoding' */ -static unsigned int zipIntSize(unsigned char encoding) { +unsigned int zipIntSize(unsigned char encoding) { switch(encoding) { case ZIP_INT_8B: return 1; case ZIP_INT_16B: return 2; @@ -195,7 +195,7 @@ static unsigned int zipIntSize(unsigned char encoding) { /* Encode the length 'rawlen' writing it in 'p'. If p is NULL it just returns * the amount of bytes required to encode such a length. */ -static unsigned int zipEncodeLength(unsigned char *p, unsigned char encoding, unsigned int rawlen) { +unsigned int zipEncodeLength(unsigned char *p, unsigned char encoding, unsigned int rawlen) { unsigned char len = 1, buf[5]; if (ZIP_IS_STR(encoding)) { @@ -259,7 +259,7 @@ static unsigned int zipEncodeLength(unsigned char *p, unsigned char encoding, un /* Encode the length of the previous entry and write it to "p". Return the * number of bytes needed to encode this length if "p" is NULL. */ -static unsigned int zipPrevEncodeLength(unsigned char *p, unsigned int len) { +unsigned int zipPrevEncodeLength(unsigned char *p, unsigned int len) { if (p == NULL) { return (len < ZIP_BIGLEN) ? 1 : sizeof(len)+1; } else { @@ -277,7 +277,7 @@ static unsigned int zipPrevEncodeLength(unsigned char *p, unsigned int len) { /* Encode the length of the previous entry and write it to "p". This only * uses the larger encoding (required in __ziplistCascadeUpdate). */ -static void zipPrevEncodeLengthForceLarge(unsigned char *p, unsigned int len) { +void zipPrevEncodeLengthForceLarge(unsigned char *p, unsigned int len) { if (p == NULL) return; p[0] = ZIP_BIGLEN; memcpy(p+1,&len,sizeof(len)); @@ -309,14 +309,14 @@ static void zipPrevEncodeLengthForceLarge(unsigned char *p, unsigned int len) { /* Return the difference in number of bytes needed to store the length of the * previous element 'len', in the entry pointed to by 'p'. */ -static int zipPrevLenByteDiff(unsigned char *p, unsigned int len) { +int zipPrevLenByteDiff(unsigned char *p, unsigned int len) { unsigned int prevlensize; ZIP_DECODE_PREVLENSIZE(p, prevlensize); return zipPrevEncodeLength(NULL, len) - prevlensize; } /* Return the total number of bytes used by the entry pointed to by 'p'. */ -static unsigned int zipRawEntryLength(unsigned char *p) { +unsigned int zipRawEntryLength(unsigned char *p) { unsigned int prevlensize, encoding, lensize, len; ZIP_DECODE_PREVLENSIZE(p, prevlensize); ZIP_DECODE_LENGTH(p + prevlensize, encoding, lensize, len); @@ -325,7 +325,7 @@ static unsigned int zipRawEntryLength(unsigned char *p) { /* Check if string pointed to by 'entry' can be encoded as an integer. * Stores the integer value in 'v' and its encoding in 'encoding'. */ -static int zipTryEncoding(unsigned char *entry, unsigned int entrylen, long long *v, unsigned char *encoding) { +int zipTryEncoding(unsigned char *entry, unsigned int entrylen, long long *v, unsigned char *encoding) { long long value; if (entrylen >= 32 || entrylen == 0) return 0; @@ -352,7 +352,7 @@ static int zipTryEncoding(unsigned char *entry, unsigned int entrylen, long long } /* Store integer 'value' at 'p', encoded as 'encoding' */ -static void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encoding) { +void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encoding) { int16_t i16; int32_t i32; int64_t i64; @@ -382,7 +382,7 @@ static void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encodi } /* Read integer encoded as 'encoding' from 'p' */ -static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) { +int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) { int16_t i16; int32_t i32; int64_t i64, ret = 0; @@ -414,7 +414,7 @@ static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) { } /* Return a struct with all information about an entry. */ -static void zipEntry(unsigned char *p, zlentry *e) { +void zipEntry(unsigned char *p, zlentry *e) { ZIP_DECODE_PREVLEN(p, e->prevrawlensize, e->prevrawlen); ZIP_DECODE_LENGTH(p + e->prevrawlensize, e->encoding, e->lensize, e->len); @@ -434,7 +434,7 @@ unsigned char *ziplistNew(void) { } /* Resize the ziplist. */ -static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) { +unsigned char *ziplistResize(unsigned char *zl, unsigned int len) { zl = zrealloc(zl,len); ZIPLIST_BYTES(zl) = intrev32ifbe(len); zl[len-1] = ZIP_END; @@ -461,7 +461,7 @@ static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) { * * The pointer "p" points to the first entry that does NOT need to be * updated, i.e. consecutive fields MAY need an update. */ -static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p) { +unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p) { size_t curlen = intrev32ifbe(ZIPLIST_BYTES(zl)), rawlen, rawlensize; size_t offset, noffset, extra; unsigned char *np; @@ -523,7 +523,7 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p } /* Delete "num" entries, starting at "p". Returns pointer to the ziplist. */ -static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsigned int num) { +unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsigned int num) { unsigned int i, totlen, deleted = 0; size_t offset; int nextdiff = 0; @@ -583,7 +583,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig } /* Insert item at "p". */ -static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) { +unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) { size_t curlen = intrev32ifbe(ZIPLIST_BYTES(zl)), reqlen; unsigned int prevlensize, prevlen = 0; size_t offset; @@ -621,7 +621,12 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig /* When the insert position is not equal to the tail, we need to * make sure that the next entry can hold this entry's length in * its prevlen field. */ + int forcelarge = 0; nextdiff = (p[0] != ZIP_END) ? zipPrevLenByteDiff(p,reqlen) : 0; + if (nextdiff == -4 && reqlen < 4) { + nextdiff = 0; + forcelarge = 1; + } /* Store offset because a realloc may change the address of zl. */ offset = p-zl; @@ -634,7 +639,10 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig memmove(p+reqlen,p-nextdiff,curlen-offset-1+nextdiff); /* Encode this entry's raw length in the next entry. */ - zipPrevEncodeLength(p+reqlen,reqlen); + if (forcelarge) + zipPrevEncodeLengthForceLarge(p+reqlen,reqlen); + else + zipPrevEncodeLength(p+reqlen,reqlen); /* Update offset for tail */ ZIPLIST_TAIL_OFFSET(zl) = diff --git a/redis.submodule/src/zmalloc.c b/redis.submodule/src/zmalloc.c index 640ee19..edfbd57 100644 --- a/redis.submodule/src/zmalloc.c +++ b/redis.submodule/src/zmalloc.c @@ -413,8 +413,9 @@ size_t zmalloc_get_memory_size(void) { if (sysctl(mib, 2, &size, &len, NULL, 0) == 0) return (size_t)size; return 0L; /* Failed? */ -#endif /* sysctl and sysconf variants */ - +#else + return 0L; /* Unknown method to get the data. */ +#endif #else return 0L; /* Unknown OS. */ #endif diff --git a/redis.submodule/tests/integration/replication-psync.tcl b/redis.submodule/tests/integration/replication-psync.tcl index 3a41ceb..308d63d 100644 --- a/redis.submodule/tests/integration/replication-psync.tcl +++ b/redis.submodule/tests/integration/replication-psync.tcl @@ -110,7 +110,7 @@ foreach diskless {no yes} { test_psync {no reconnection, just sync} 6 1000000 3600 0 { } $diskless 0 - test_psync {ok psync} 6 1000000 3600 0 { + test_psync {ok psync} 6 100000000 3600 0 { assert {[s -1 sync_partial_ok] > 0} } $diskless 1