Merge branch 'unstable' into conduct

* TLS: Session caching configuration support.
* TLS: Remove redundant config initialization.

.github/workflows/ci.yml

@@ -31,7 +31,7 @@ jobs:
     - name: make
       run: make
 
-  biuld-32bit:
+  build-32bit:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v1

.github/workflows/daily.yml

@@ -47,7 +47,9 @@ jobs:
         sudo apt-get install tcl8.5
         ./runtest --accurate --verbose
     - name: module api test
-      run: ./runtest-moduleapi --verbose
+      run: |
+        make -C tests/modules 32bit # the script below doesn't have an argument, we must build manually ahead of time
+        ./runtest-moduleapi --verbose
 
   test-tls:
     runs-on: ubuntu-latest

BUGS

@@ -1 +1 @@
-Please check https://github.com/antirez/redis/issues
+Please check https://github.com/redis/redis/issues

CONDUCT

@@ -0,0 +1,96 @@
+Contributor Covenant Code of Conduct
+Our Pledge
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+Our Standards
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others’ private information, such as a physical or email
+address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+professional setting
+
+Enforcement Responsibilities
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+Scope
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+Enforcement
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+this email address: redis@redis.io.
+All complaints will be reviewed and investigated promptly and fairly.
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+Enforcement Guidelines
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+1. Correction
+Community Impact: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+Consequence: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+2. Warning
+Community Impact: A violation through a single incident or series
+of actions.
+Consequence: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+3. Temporary Ban
+Community Impact: A serious violation of community standards, including
+sustained inappropriate behavior.
+Consequence: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+4. Permanent Ban
+Community Impact: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+Consequence: A permanent ban from any sort of public interaction within
+the community.
+Attribution
+This Code of Conduct is adapted from the Contributor Covenant,
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+Community Impact Guidelines were inspired by Mozilla’s code of conduct
+enforcement ladder.
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

COPYING

@@ -1,4 +1,4 @@
-Copyright (c) 2006-2015, Salvatore Sanfilippo
+Copyright (c) 2006-2020, Salvatore Sanfilippo
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

README.md

@@ -8,7 +8,7 @@ Redis is often referred as a *data structures* server. What this means is that R
 Data structures implemented into Redis have a few special properties:
 
 * Redis cares to store them on disk, even if they are always served and modified into the server memory. This means that Redis is fast, but that is also non-volatile.
-* Implementation of data structures stress on memory efficiency, so data structures inside Redis will likely use less memory compared to the same data structure modeled using an high level programming language.
+* Implementation of data structures stress on memory efficiency, so data structures inside Redis will likely use less memory compared to the same data structure modeled using a high-level programming language.
 * Redis offers a number of features that are natural to find in a database, like replication, tunable levels of durability, cluster, high availability.
 
 Another good example is to think of Redis as a more complex version of memcached, where the operations are not just SETs and GETs, but operations to work with complex data types like Lists, Sets, ordered data structures, and so forth.
@@ -75,10 +75,10 @@ optimizations (for debugging purposes), and other similar build time options,
 those options are cached indefinitely until you issue a `make distclean`
 command.
 
-Fixing problems building 32 bit binaries
+Fixing problems building 32-bit binaries
 ---------
 
-If after building Redis with a 32 bit target you need to rebuild it
+If after building Redis with a 32-bit target you need to rebuild it
 with a 64 bit target, or the other way around, you need to perform a
 `make distclean` in the root directory of the Redis distribution.
 
@@ -193,7 +193,7 @@ system reboots.
 You'll be able to stop and start Redis using the script named
 `/etc/init.d/redis_<portnumber>`, for instance `/etc/init.d/redis_6379`.
 
-Code contributions
+Code contributions and conduct
 -----------------
 
 Note: by contributing code to the Redis project in any form, including sending
@@ -202,17 +202,18 @@ public discussion groups, you agree to release your code under the terms
 of the BSD license that you can find in the [COPYING][1] file included in the Redis
 source distribution.
 
-Please see the [CONTRIBUTING][2] file in this source distribution for more
-information.
+Please see the [CONTRIBUTING][2] and [CONDUCT][3] files in this source
+distribution for more information.
 
-[1]: https://github.com/antirez/redis/blob/unstable/COPYING
-[2]: https://github.com/antirez/redis/blob/unstable/CONTRIBUTING
+[1]: https://github.com/redis/redis/blob/unstable/COPYING
+[2]: https://github.com/redis/redis/blob/unstable/CONTRIBUTING
+[3]: https://github.com/redis/redis/blob/unstable/CONDUCT
 
 Redis internals
 ===
 
 If you are reading this README you are likely in front of a Github page
-or you just untarred the Redis distribution tar ball. In both the cases
+or you just untarred the Redis distribution tar ball. In both cases
 you are basically one step away from the source code, so here we explain
 the Redis source code layout, what is in each file as a general idea, the
 most important functions and structures inside the Redis server and so forth.
@@ -236,7 +237,7 @@ Inside the root are the following important directories:
 
 * `src`: contains the Redis implementation, written in C.
 * `tests`: contains the unit tests, implemented in Tcl.
-* `deps`: contains libraries Redis uses. Everything needed to compile Redis is inside this directory; your system just needs to provide `libc`, a POSIX compatible interface and a C compiler. Notably `deps` contains a copy of `jemalloc`, which is the default allocator of Redis under Linux. Note that under `deps` there are also things which started with the Redis project, but for which the main repository is not `antirez/redis`.
+* `deps`: contains libraries Redis uses. Everything needed to compile Redis is inside this directory; your system just needs to provide `libc`, a POSIX compatible interface and a C compiler. Notably `deps` contains a copy of `jemalloc`, which is the default allocator of Redis under Linux. Note that under `deps` there are also things which started with the Redis project, but for which the main repository is not `redis/redis`.
 
 There are a few more directories but they are not very important for our goals
 here. We'll focus mostly on `src`, where the Redis implementation is contained,
@@ -336,7 +337,7 @@ There are two special functions called periodically by the event loop:
 Inside server.c you can find code that handles other vital things of the Redis server:
 
 * `call()` is used in order to call a given command in the context of a given client.
-* `activeExpireCycle()` handles eviciton of keys with a time to live set via the `EXPIRE` command.
+* `activeExpireCycle()` handles eviction of keys with a time to live set via the `EXPIRE` command.
 * `freeMemoryIfNeeded()` is called when a new write command should be performed but Redis is out of memory according to the `maxmemory` directive.
 * The global variable `redisCommandTable` defines all the Redis commands, specifying the name of the command, the function implementing the command, the number of arguments required, and other properties of each command.
 
@@ -401,7 +402,7 @@ a basic level, like functions to allocate new objects, handle the reference
 counting and so forth. Notable functions inside this file:
 
 * `incrRefcount()` and `decrRefCount()` are used in order to increment or decrement an object reference count. When it drops to 0 the object is finally freed.
-* `createObject()` allocates a new object. There are also specialized functions to allocate string objects having a specific content, like `createStringObjectFromLongLong()` and similar functions.
+* `createObject()` allocates a new object. There are also specialized functions to allocate string objects having specific contents, like `createStringObjectFromLongLong()` and similar functions.
 
 This file also implements the `OBJECT` command.
 

TLS.md

@@ -68,8 +68,6 @@ but there are probably other good reasons to improve that part anyway.
 To-Do List
 ----------
 
-- [ ] Add session caching support. Check if/how it's handled by clients to
-  assess how useful/important it is.
 - [ ] redis-benchmark support. The current implementation is a mix of using
   hiredis for parsing and basic networking (establishing connections), but
   directly manipulating sockets for most actions. This will need to be cleaned

deps/README.md

@@ -47,7 +47,7 @@ Hiredis
 Hiredis uses the SDS string library, that must be the same version used inside Redis itself. Hiredis is also very critical for Sentinel. Historically Redis often used forked versions of hiredis in a way or the other. In order to upgrade it is advised to take a lot of care:
 
 1. Check with diff if hiredis API changed and what impact it could have in Redis.
-2. Make sure thet the SDS library inside Hiredis and inside Redis are compatible.
+2. Make sure that the SDS library inside Hiredis and inside Redis are compatible.
 3. After the upgrade, run the Redis Sentinel test.
 4. Check manually that redis-cli and redis-benchmark behave as expecteed, since we have no tests for CLI utilities currently.
 

redis.conf

@@ -199,6 +199,22 @@ tcp-keepalive 300
 #
 # tls-prefer-server-ciphers yes
 
+# By default, TLS session caching is enabled to allow faster and less expensive
+# reconnections by clients that support it. Use the following directive to disable
+# caching.
+#
+# tls-session-caching no
+
+# Change the default number of TLS sessions cached. A zero value sets the cache
+# to unlimited size. The default size is 20480.
+#
+# tls-session-cache-size 5000
+
+# Change the default timeout of cached TLS sessions. The default timeout is 300
+# seconds.
+#
+# tls-session-cache-timeout 60
+
 ################################# GENERAL #####################################
 
 # By default Redis does not run as a daemon. Use 'yes' if you need it.
@@ -805,6 +821,11 @@ acllog-max-len 128
 # Once the limit is reached Redis will close all the new connections sending
 # an error 'max number of clients reached'.
 #
+# IMPORTANT: When Redis Cluster is used, the max number of connections is also
+# shared with the cluster bus: every node in the cluster will use two
+# connections, one incoming and another outgoing. It is important to size the
+# limit accordingly in case of very large clusters.
+#
 # maxclients 10000
 
 ############################## MEMORY MANAGEMENT ################################

src/Makefile

@@ -192,9 +192,21 @@ ifeq ($(MALLOC),jemalloc)
 endif
 
 ifeq ($(BUILD_TLS),yes)
-    FINAL_CFLAGS+=-DUSE_OPENSSL $(OPENSSL_CFLAGS)
-    FINAL_LDFLAGS+=$(OPENSSL_LDFLAGS)
-    FINAL_LIBS += ../deps/hiredis/libhiredis_ssl.a -lssl -lcrypto
+	FINAL_CFLAGS+=-DUSE_OPENSSL $(OPENSSL_CFLAGS)
+	FINAL_LDFLAGS+=$(OPENSSL_LDFLAGS)
+	LIBSSL_PKGCONFIG := $(shell $(PKG_CONFIG) --exists libssl && echo $$?)
+ifeq ($(LIBSSL_PKGCONFIG),0)
+	LIBSSL_LIBS=$(shell $(PKG_CONFIG) --libs libssl)
+else
+	LIBSSL_LIBS=-lssl
+endif
+	LIBCRYPTO_PKGCONFIG := $(shell $(PKG_CONFIG) --exists libcrypto && echo $$?)
+ifeq ($(LIBCRYPTO_PKGCONFIG),0)
+	LIBCRYPTO_LIBS=$(shell $(PKG_CONFIG) --libs libcrypto)
+else
+	LIBCRYPTO_LIBS=-lcrypto
+endif
+	FINAL_LIBS += ../deps/hiredis/libhiredis_ssl.a $(LIBSSL_LIBS) $(LIBCRYPTO_LIBS)
 endif
 
 REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS)

src/acl.c

@@ -732,10 +732,11 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) {
  * EEXIST: You are adding a key pattern after "*" was already added. This is
  *         almost surely an error on the user side.
  * ENODEV: The password you are trying to remove from the user does not exist.
- * EBADMSG: The hash you are trying to add is not a valid hash. 
+ * EBADMSG: The hash you are trying to add is not a valid hash.
  */
 int ACLSetUser(user *u, const char *op, ssize_t oplen) {
     if (oplen == -1) oplen = strlen(op);
+    if (oplen == 0) return C_OK; /* Empty string is a no-operation. */
     if (!strcasecmp(op,"on")) {
         u->flags |= USER_FLAG_ENABLED;
         u->flags &= ~USER_FLAG_DISABLED;
@@ -1297,7 +1298,7 @@ sds ACLLoadFromFile(const char *filename) {
         if (lines[i][0] == '\0') continue;
 
         /* Split into arguments */
-        argv = sdssplitargs(lines[i],&argc);
+        argv = sdssplitlen(lines[i],sdslen(lines[i])," ",1,&argc);
         if (argv == NULL) {
             errors = sdscatprintf(errors,
                      "%s:%d: unbalanced quotes in acl line. ",
@@ -1329,11 +1330,14 @@ sds ACLLoadFromFile(const char *filename) {
             continue;
         }
 
-        /* Try to process the line using the fake user to validate iif
-         * the rules are able to apply cleanly. */
+        /* Try to process the line using the fake user to validate if
+         * the rules are able to apply cleanly. At this stage we also
+         * trim trailing spaces, so that we don't have to handle that
+         * in ACLSetUser(). */
         ACLSetUser(fakeuser,"reset",-1);
         int j;
         for (j = 2; j < argc; j++) {
+            argv[j] = sdstrim(argv[j],"\t\r\n");
             if (ACLSetUser(fakeuser,argv[j],sdslen(argv[j])) != C_OK) {
                 char *errmsg = ACLSetUserStringError();
                 errors = sdscatprintf(errors,

src/aof.c

@@ -611,19 +611,24 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a
     } else if (cmd->proc == setCommand && argc > 3) {
         int i;
         robj *exarg = NULL, *pxarg = NULL;
-        /* Translate SET [EX seconds][PX milliseconds] to SET and PEXPIREAT */
-        buf = catAppendOnlyGenericCommand(buf,3,argv);
         for (i = 3; i < argc; i ++) {
             if (!strcasecmp(argv[i]->ptr, "ex")) exarg = argv[i+1];
             if (!strcasecmp(argv[i]->ptr, "px")) pxarg = argv[i+1];
         }
         serverAssert(!(exarg && pxarg));
-        if (exarg)
-            buf = catAppendOnlyExpireAtCommand(buf,server.expireCommand,argv[1],
-                                               exarg);
-        if (pxarg)
-            buf = catAppendOnlyExpireAtCommand(buf,server.pexpireCommand,argv[1],
-                                               pxarg);
+
+        if (exarg || pxarg) {
+            /* Translate SET [EX seconds][PX milliseconds] to SET and PEXPIREAT */
+            buf = catAppendOnlyGenericCommand(buf,3,argv);
+            if (exarg)
+                buf = catAppendOnlyExpireAtCommand(buf,server.expireCommand,argv[1],
+                                                   exarg);
+            if (pxarg)
+                buf = catAppendOnlyExpireAtCommand(buf,server.pexpireCommand,argv[1],
+                                                   pxarg);
+        } else {
+            buf = catAppendOnlyGenericCommand(buf,argc,argv);
+        }
     } else {
         /* All the other commands don't need translation or need the
          * same translation already operated in the command vector

src/bitops.c

@@ -257,7 +257,7 @@ int64_t getSignedBitfield(unsigned char *p, uint64_t offset, uint64_t bits) {
     /* If the top significant bit is 1, propagate it to all the
      * higher bits for two's complement representation of signed
      * integers. */
-    if (value & ((uint64_t)1 << (bits-1)))
+    if (bits < 64 && (value & ((uint64_t)1 << (bits-1))))
         value |= ((uint64_t)-1) << bits;
     return value;
 }
@@ -356,7 +356,6 @@ int checkSignedBitfieldOverflow(int64_t value, int64_t incr, uint64_t bits, int
 
 handle_wrap:
     {
-        uint64_t mask = ((uint64_t)-1) << bits;
         uint64_t msb = (uint64_t)1 << (bits-1);
         uint64_t a = value, b = incr, c;
         c = a+b; /* Perform addition as unsigned so that's defined. */
@@ -364,10 +363,13 @@ handle_wrap:
         /* If the sign bit is set, propagate to all the higher order
          * bits, to cap the negative value. If it's clear, mask to
          * the positive integer limit. */
-        if (c & msb) {
-            c |= mask;
-        } else {
-            c &= ~mask;
+        if (bits < 64) {
+            uint64_t mask = ((uint64_t)-1) << bits;
+            if (c & msb) {
+                c |= mask;
+            } else {
+                c &= ~mask;
+            }
         }
         *limit = c;
     }
@@ -757,11 +759,12 @@ void bitopCommand(client *c) {
         setKey(c,c->db,targetkey,o);
         notifyKeyspaceEvent(NOTIFY_STRING,"set",targetkey,c->db->id);
         decrRefCount(o);
+        server.dirty++;
     } else if (dbDelete(c->db,targetkey)) {
         signalModifiedKey(c,c->db,targetkey);
         notifyKeyspaceEvent(NOTIFY_GENERIC,"del",targetkey,c->db->id);
+        server.dirty++;
     }
-    server.dirty++;
     addReplyLongLong(c,maxlen); /* Return the output string length in bytes. */
 }
 

src/cluster.c

@@ -691,6 +691,16 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
     }
 }
 
+/* Return the approximated number of sockets we are using in order to
+ * take the cluster bus connections. */
+unsigned long getClusterConnectionsCount(void) {
+    /* We decrement the number of nodes by one, since there is the
+     * "myself" node too in the list. Each node uses two file descriptors,
+     * one incoming and one outgoing, thus the multiplication by 2. */
+    return server.cluster_enabled ?
+           ((dictSize(server.cluster->nodes)-1)*2) : 0;
+}
+
 /* -----------------------------------------------------------------------------
  * Key space handling
  * -------------------------------------------------------------------------- */
@@ -1254,8 +1264,11 @@ void markNodeAsFailingIfNeeded(clusterNode *node) {
     node->fail_time = mstime();
 
     /* Broadcast the failing node name to everybody, forcing all the other
-     * reachable nodes to flag the node as FAIL. */
-    if (nodeIsMaster(myself)) clusterSendFail(node->name);
+     * reachable nodes to flag the node as FAIL.
+     * We do that even if this node is a replica and not a master: anyway
+     * the failing state is triggered collecting failure reports from masters,
+     * so here the replica is only helping propagating this status. */
+    clusterSendFail(node->name);
     clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG);
 }
 
@@ -1463,7 +1476,10 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) {
             }
         } else {
             /* If it's not in NOADDR state and we don't have it, we
-             * start a handshake process against this IP/PORT pairs.
+             * add it to our trusted dict with exact nodeid and flag.
+             * Note that we cannot simply start a handshake against
+             * this IP/PORT pairs, since IP/PORT can be reused already,
+             * otherwise we risk joining another cluster.
              *
              * Note that we require that the sender of this gossip message
              * is a well known node in our cluster, otherwise we risk
@@ -1472,7 +1488,12 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) {
                 !(flags & CLUSTER_NODE_NOADDR) &&
                 !clusterBlacklistExists(g->nodename))
             {
-                clusterStartHandshake(g->ip,ntohs(g->port),ntohs(g->cport));
+                clusterNode *node;
+                node = createClusterNode(g->nodename, flags);
+                memcpy(node->ip,g->ip,NET_IP_STR_LEN);
+                node->port = ntohs(g->port);
+                node->cport = ntohs(g->cport);
+                clusterAddNode(node);
             }
         }
 
@@ -4967,7 +4988,8 @@ void restoreCommand(client *c) {
     }
 
     /* Make sure this key does not already exist here... */
-    if (!replace && lookupKeyWrite(c->db,c->argv[1]) != NULL) {
+    robj *key = c->argv[1];
+    if (!replace && lookupKeyWrite(c->db,key) != NULL) {
         addReply(c,shared.busykeyerr);
         return;
     }
@@ -4989,24 +5011,37 @@ void restoreCommand(client *c) {
 
     rioInitWithBuffer(&payload,c->argv[3]->ptr);
     if (((type = rdbLoadObjectType(&payload)) == -1) ||
-        ((obj = rdbLoadObject(type,&payload,c->argv[1]->ptr)) == NULL))
+        ((obj = rdbLoadObject(type,&payload,key->ptr)) == NULL))
     {
         addReplyError(c,"Bad data format");
         return;
     }
 
     /* Remove the old key if needed. */
-    if (replace) dbDelete(c->db,c->argv[1]);
+    int deleted = 0;
+    if (replace)
+        deleted = dbDelete(c->db,key);
+
+    if (ttl && !absttl) ttl+=mstime();
+    if (ttl && checkAlreadyExpired(ttl)) {
+        if (deleted) {
+            rewriteClientCommandVector(c,2,shared.del,key);
+            signalModifiedKey(c,c->db,key);
+            notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
+            server.dirty++;
+        }
+        addReply(c, shared.ok);
+        return;
+    }
 
     /* Create the key and set the TTL if any */
-    dbAdd(c->db,c->argv[1],obj);
+    dbAdd(c->db,key,obj);
     if (ttl) {
-        if (!absttl) ttl+=mstime();
-        setExpire(c,c->db,c->argv[1],ttl);
+        setExpire(c,c->db,key,ttl);
     }
     objectSetLRUOrLFU(obj,lfu_freq,lru_idle,lru_clock,1000);
-    signalModifiedKey(c,c->db,c->argv[1]);
-    notifyKeyspaceEvent(NOTIFY_GENERIC,"restore",c->argv[1],c->db->id);
+    signalModifiedKey(c,c->db,key);
+    notifyKeyspaceEvent(NOTIFY_GENERIC,"restore",key,c->db->id);
     addReply(c,shared.ok);
     server.dirty++;
 }

src/cluster.h

@@ -283,5 +283,6 @@ typedef struct {
 clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, int argc, int *hashslot, int *ask);
 int clusterRedirectBlockedClientIfNeeded(client *c);
 void clusterRedirectClient(client *c, clusterNode *n, int hashslot, int error_code);
+unsigned long getClusterConnectionsCount(void);
 
 #endif /* __CLUSTER_H */

src/config.c

@@ -2071,7 +2071,7 @@ static int updateTlsCfg(char *val, char *prev, char **err) {
     UNUSED(prev);
     UNUSED(err);
     if (tlsConfigure(&server.tls_ctx_config) == C_ERR) {
-        *err = "Unable to configure tls-cert-file. Check server logs.";
+        *err = "Unable to update TLS configuration. Check server logs.";
         return 0;
     }
     return 1;
@@ -2081,6 +2081,12 @@ static int updateTlsCfgBool(int val, int prev, char **err) {
     UNUSED(prev);
     return updateTlsCfg(NULL, NULL, err);
 }
+
+static int updateTlsCfgInt(long long val, long long prev, char **err) {
+    UNUSED(val);
+    UNUSED(prev);
+    return updateTlsCfg(NULL, NULL, err);
+}
 #endif  /* USE_OPENSSL */
 
 standardConfig configs[] = {
@@ -2216,10 +2222,13 @@ standardConfig configs[] = {
 
 #ifdef USE_OPENSSL
     createIntConfig("tls-port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.tls_port, 0, INTEGER_CONFIG, NULL, NULL), /* TCP port. */
+    createIntConfig("tls-session-cache-size", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tls_ctx_config.session_cache_size, 20*1024, INTEGER_CONFIG, NULL, updateTlsCfgInt),
+    createIntConfig("tls-session-cache-timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tls_ctx_config.session_cache_timeout, 300, INTEGER_CONFIG, NULL, updateTlsCfgInt),
     createBoolConfig("tls-cluster", NULL, MODIFIABLE_CONFIG, server.tls_cluster, 0, NULL, NULL),
     createBoolConfig("tls-replication", NULL, MODIFIABLE_CONFIG, server.tls_replication, 0, NULL, NULL),
     createBoolConfig("tls-auth-clients", NULL, MODIFIABLE_CONFIG, server.tls_auth_clients, 1, NULL, NULL),
     createBoolConfig("tls-prefer-server-ciphers", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.prefer_server_ciphers, 0, NULL, updateTlsCfgBool),
+    createBoolConfig("tls-session-caching", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.session_caching, 1, NULL, updateTlsCfgBool),
     createStringConfig("tls-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.cert_file, NULL, NULL, updateTlsCfg),
     createStringConfig("tls-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.key_file, NULL, NULL, updateTlsCfg),
     createStringConfig("tls-dh-params-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.dh_params_file, NULL, NULL, updateTlsCfg),

src/config.h

@@ -133,12 +133,6 @@ void setproctitle(const char *fmt, ...);
 /* Byte ordering detection */
 #include <sys/types.h> /* This will likely define BYTE_ORDER */
 
-/* Define redis_sendfile. */
-#if defined(__linux__) || (defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_5))
-#define HAVE_SENDFILE 1
-ssize_t redis_sendfile(int out_fd, int in_fd, off_t offset, size_t count);
-#endif
-
 #ifndef BYTE_ORDER
 #if (BSD >= 199103)
 # include <machine/endian.h>

src/db.c

@@ -963,14 +963,6 @@ void shutdownCommand(client *c) {
             return;
         }
     }
-    /* When SHUTDOWN is called while the server is loading a dataset in
-     * memory we need to make sure no attempt is performed to save
-     * the dataset on shutdown (otherwise it could overwrite the current DB
-     * with half-read data).
-     *
-     * Also when in Sentinel mode clear the SAVE flag and force NOSAVE. */
-    if (server.loading || server.sentinel_mode)
-        flags = (flags & ~SHUTDOWN_SAVE) | SHUTDOWN_NOSAVE;
     if (prepareForShutdown(flags) == C_OK) exit(0);
     addReplyError(c,"Errors trying to SHUTDOWN. Check logs.");
 }

src/debug.c

@@ -378,6 +378,7 @@ void debugCommand(client *c) {
 "DEBUG PROTOCOL [string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false]",
 "ERROR <string> -- Return a Redis protocol error with <string> as message. Useful for clients unit tests to simulate Redis errors.",
 "LOG <message> -- write message to the server log.",
+"LEAK <string> -- Create a memory leak of the input string.",
 "HTSTATS <dbid> -- Return hash table statistics of the specified Redis database.",
 "HTSTATS-KEY <key> -- Like htstats but for the hash table stored as key's value.",
 "LOADAOF -- Flush the AOF buffers on disk and reload the AOF in memory.",
@@ -430,6 +431,9 @@ NULL
     } else if (!strcasecmp(c->argv[1]->ptr,"log") && c->argc == 3) {
         serverLog(LL_WARNING, "DEBUG LOG: %s", (char*)c->argv[2]->ptr);
         addReply(c,shared.ok);
+    } else if (!strcasecmp(c->argv[1]->ptr,"leak") && c->argc == 3) {
+        sdsdup(c->argv[2]->ptr);
+        addReply(c,shared.ok);
     } else if (!strcasecmp(c->argv[1]->ptr,"reload")) {
         int flush = 1, save = 1;
         int flags = RDBFLAGS_NONE;
@@ -1569,7 +1573,7 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
     serverLogRaw(LL_WARNING|LL_RAW,
 "\n=== REDIS BUG REPORT END. Make sure to include from START to END. ===\n\n"
 "       Please report the crash by opening an issue on github:\n\n"
-"           http://github.com/antirez/redis/issues\n\n"
+"           http://github.com/redis/redis/issues\n\n"
 "  Suspect RAM error? Use redis-server --test-memory to verify it.\n\n"
 );
 

src/defrag.c

@@ -348,7 +348,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) {
         sdsele = ln->value;
         if ((newsds = activeDefragSds(sdsele))) {
             /* When defragging an sds value, we need to update the dict key */
-            uint64_t hash = dictGetHash(d, sdsele);
+            uint64_t hash = dictGetHash(d, newsds);
             replaceSateliteDictKeyPtrAndOrDefragDictEntry(d, sdsele, newsds, hash, &defragged);
             ln->value = newsds;
             defragged++;

src/expire.c

@@ -475,6 +475,16 @@ void flushSlaveKeysWithExpireList(void) {
     }
 }
 
+int checkAlreadyExpired(long long when) {
+    /* EXPIRE with negative TTL, or EXPIREAT with a timestamp into the past
+     * should never be executed as a DEL when load the AOF or in the context
+     * of a slave instance.
+     *
+     * Instead we add the already expired key to the database with expire time
+     * (possibly in the past) and wait for an explicit DEL from the master. */
+    return (when <= mstime() && !server.loading && !server.masterhost);
+}
+
 /*-----------------------------------------------------------------------------
  * Expires Commands
  *----------------------------------------------------------------------------*/
@@ -502,13 +512,7 @@ void expireGenericCommand(client *c, long long basetime, int unit) {
         return;
     }
 
-    /* EXPIRE with negative TTL, or EXPIREAT with a timestamp into the past
-     * should never be executed as a DEL when load the AOF or in the context
-     * of a slave instance.
-     *
-     * Instead we take the other branch of the IF statement setting an expire
-     * (possibly in the past) and wait for an explicit DEL from the master. */
-    if (when <= mstime() && !server.loading && !server.masterhost) {
+    if (checkAlreadyExpired(when)) {
         robj *aux;
 
         int deleted = server.lazyfree_lazy_expire ? dbAsyncDelete(c->db,key) :

src/help.h

@@ -43,6 +43,16 @@ struct commandHelp {
     "Generate a pseudorandom secure password to use for ACL users",
     9,
     "6.0.0" },
+    { "ACL GETUSER",
+    "username",
+    "Get the rules for a specific ACL user",
+    9,
+    "6.0.0" },
+    { "ACL HELP",
+    "-",
+    "Show helpful text about the different subcommands",
+    9,
+    "6.0.0" },
     { "ACL LIST",
     "-",
     "List the current ACL rules in ACL config file format",
@@ -64,7 +74,7 @@ struct commandHelp {
     9,
     "6.0.0" },
     { "ACL SETUSER",
-    "rule [rule ...]",
+    "username [rule [rule ...]]",
     "Modify or create the rules for a specific ACL user",
     9,
     "6.0.0" },
@@ -164,7 +174,7 @@ struct commandHelp {
     8,
     "5.0.0" },
     { "CLIENT KILL",
-    "[ip:port] [ID client-id] [TYPE normal|master|slave|pubsub] [ADDR ip:port] [SKIPME yes/no]",
+    "[ip:port] [ID client-id] [TYPE normal|master|slave|pubsub] [USER username] [ADDR ip:port] [SKIPME yes/no]",
     "Kill the connection of a client",
     8,
     "2.4.0" },
@@ -182,14 +192,14 @@ struct commandHelp {
     "ON|OFF|SKIP",
     "Instruct the server whether to reply to commands",
     8,
-    "3.2" },
+    "3.2.0" },
     { "CLIENT SETNAME",
     "connection-name",
     "Set the current connection name",
     8,
     "2.6.9" },
     { "CLIENT TRACKING",
-    "ON|OFF [REDIRECT client-id] [PREFIX prefix] [BCAST] [OPTIN] [OPTOUT] [NOLOOP]",
+    "ON|OFF [REDIRECT client-id] [PREFIX prefix [PREFIX prefix ...]] [BCAST] [OPTIN] [OPTOUT] [NOLOOP]",
     "Enable or disable server assisted client side caching support",
     8,
     "6.0.0" },
@@ -619,7 +629,7 @@ struct commandHelp {
     9,
     "2.8.13" },
     { "LATENCY RESET",
-    "[event]",
+    "[event [event ...]]",
     "Reset latency data for one or more events.",
     9,
     "2.8.13" },
@@ -648,6 +658,11 @@ struct commandHelp {
     "Remove and get the first element in a list",
     2,
     "1.0.0" },
+    { "LPOS",
+    "key element [FIRST rank] [COUNT num-matches] [MAXLEN len]",
+    "Return the index of matching elements on a list",
+    2,
+    "6.0.6" },
     { "LPUSH",
     "key element [element ...]",
     "Prepend one or multiple elements to a list",

src/listpack.c

@@ -773,13 +773,13 @@ unsigned char *lpSeek(unsigned char *lp, long index) {
          * is past the half of the listpack. */
         if (index > numele/2) {
             forward = 0;
-            /* Left to right scanning always expects a negative index. Convert
+            /* Right to left scanning always expects a negative index. Convert
              * our index to negative form. */
             index -= numele;
         }
     } else {
         /* If the listpack length is unspecified, for negative indexes we
-         * want to always scan left-to-right. */
+         * want to always scan right-to-left. */
         if (index < 0) forward = 0;
     }
 

src/module.c

@@ -6708,7 +6708,7 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc
         int pos = 0;
         int64_t ll;
         while(intsetGet(o->ptr,pos++,&ll)) {
-            robj *field = createStringObjectFromLongLong(ll);
+            robj *field = createObject(OBJ_STRING,sdsfromlonglong(ll));
             fn(key, field, NULL, privdata);
             decrRefCount(field);
         }
@@ -6724,12 +6724,12 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc
             ziplistGet(p,&vstr,&vlen,&vll);
             robj *field = (vstr != NULL) ?
                 createStringObject((char*)vstr,vlen) :
-                createStringObjectFromLongLong(vll);
+                createObject(OBJ_STRING,sdsfromlonglong(vll));
             p = ziplistNext(o->ptr,p);
             ziplistGet(p,&vstr,&vlen,&vll);
             robj *value = (vstr != NULL) ?
                 createStringObject((char*)vstr,vlen) :
-                createStringObjectFromLongLong(vll);
+                createObject(OBJ_STRING,sdsfromlonglong(vll));
             fn(key, field, value, privdata);
             p = ziplistNext(o->ptr,p);
             decrRefCount(field);

src/multi.c

@@ -36,6 +36,7 @@ void initClientMultiState(client *c) {
     c->mstate.commands = NULL;
     c->mstate.count = 0;
     c->mstate.cmd_flags = 0;
+    c->mstate.cmd_inv_flags = 0;
 }
 
 /* Release all the resources associated with MULTI/EXEC state */
@@ -58,6 +59,13 @@ void queueMultiCommand(client *c) {
     multiCmd *mc;
     int j;
 
+    /* No sense to waste memory if the transaction is already aborted.
+     * this is useful in case client sends these in a pipeline, or doesn't
+     * bother to read previous responses and didn't notice the multi was already
+     * aborted. */
+    if (c->flags & CLIENT_DIRTY_EXEC)
+        return;
+
     c->mstate.commands = zrealloc(c->mstate.commands,
             sizeof(multiCmd)*(c->mstate.count+1));
     mc = c->mstate.commands+c->mstate.count;
@@ -69,6 +77,7 @@ void queueMultiCommand(client *c) {
         incrRefCount(mc->argv[j]);
     c->mstate.count++;
     c->mstate.cmd_flags |= c->cmd->flags;
+    c->mstate.cmd_inv_flags |= ~c->cmd->flags;
 }
 
 void discardTransaction(client *c) {
@@ -115,6 +124,23 @@ void execCommandPropagateExec(client *c) {
               PROPAGATE_AOF|PROPAGATE_REPL);
 }
 
+/* Aborts a transaction, with a specific error message.
+ * The transaction is always aboarted with -EXECABORT so that the client knows
+ * the server exited the multi state, but the actual reason for the abort is
+ * included too. */
+void execCommandAbort(client *c, sds error) {
+    discardTransaction(c);
+
+    if (error[0] == '-') error++;
+    addReplyErrorFormat(c, "-EXECABORT Transaction discarded because of: %s", error);
+
+    /* Send EXEC to clients waiting data from MONITOR. We did send a MULTI
+     * already, and didn't send any of the queued commands, now we'll just send
+     * EXEC so it is clear that the transaction is over. */
+    if (listLength(server.monitors) && !server.loading)
+        replicationFeedMonitors(c,server.monitors,c->db->id,c->argv,c->argc);
+}
+
 void execCommand(client *c) {
     int j;
     robj **orig_argv;
@@ -141,21 +167,6 @@ void execCommand(client *c) {
         goto handle_monitor;
     }
 
-    /* If there are write commands inside the transaction, and this is a read
-     * only slave, we want to send an error. This happens when the transaction
-     * was initiated when the instance was a master or a writable replica and
-     * then the configuration changed (for example instance was turned into
-     * a replica). */
-    if (!server.loading && server.masterhost && server.repl_slave_ro &&
-        !(c->flags & CLIENT_MASTER) && c->mstate.cmd_flags & CMD_WRITE)
-    {
-        addReplyError(c,
-            "Transaction contains write commands but instance "
-            "is now a read-only replica. EXEC aborted.");
-        discardTransaction(c);
-        goto handle_monitor;
-    }
-
     /* Exec all the queued commands */
     unwatchAllKeys(c); /* Unwatch ASAP otherwise we'll waste CPU cycles */
     orig_argv = c->argv;

src/networking.c

@@ -29,6 +29,7 @@
 
 #include "server.h"
 #include "atomicvar.h"
+#include "cluster.h"
 #include <sys/socket.h>
 #include <sys/uio.h>
 #include <math.h>
@@ -396,31 +397,7 @@ void addReplyErrorLength(client *c, const char *s, size_t len) {
         if (ctype == CLIENT_TYPE_MASTER && server.repl_backlog &&
             server.repl_backlog_histlen > 0)
         {
-            long long dumplen = 256;
-            if (server.repl_backlog_histlen < dumplen)
-                dumplen = server.repl_backlog_histlen;
-
-            /* Identify the first byte to dump. */
-            long long idx =
-              (server.repl_backlog_idx + (server.repl_backlog_size - dumplen)) %
-               server.repl_backlog_size;
-
-            /* Scan the circular buffer to collect 'dumplen' bytes. */
-            sds dump = sdsempty();
-            while(dumplen) {
-                long long thislen =
-                    ((server.repl_backlog_size - idx) < dumplen) ?
-                    (server.repl_backlog_size - idx) : dumplen;
-
-                dump = sdscatrepr(dump,server.repl_backlog+idx,thislen);
-                dumplen -= thislen;
-                idx = 0;
-            }
-
-            /* Finally log such bytes: this is vital debugging info to
-             * understand what happened. */
-            serverLog(LL_WARNING,"Latest backlog is: '%s'", dump);
-            sdsfree(dump);
+            showLatestBacklog();
         }
         server.stat_unexpected_error_replies++;
     }
@@ -430,19 +407,23 @@ void addReplyError(client *c, const char *err) {
     addReplyErrorLength(c,err,strlen(err));
 }
 
+/* See addReplyErrorLength.
+ * Makes sure there are no newlines in the string, otherwise invalid protocol
+ * is emitted. */
+void addReplyErrorSafe(client *c, char *s, size_t len) {
+    size_t j;
+    for (j = 0; j < len; j++) {
+        if (s[j] == '\r' || s[j] == '\n') s[j] = ' ';
+    }
+    addReplyErrorLength(c,s,sdslen(s));
+}
+
 void addReplyErrorFormat(client *c, const char *fmt, ...) {
-    size_t l, j;
     va_list ap;
     va_start(ap,fmt);
     sds s = sdscatvprintf(sdsempty(),fmt,ap);
     va_end(ap);
-    /* Make sure there are no newlines in the string, otherwise invalid protocol
-     * is emitted. */
-    l = sdslen(s);
-    for (j = 0; j < l; j++) {
-        if (s[j] == '\r' || s[j] == '\n') s[j] = ' ';
-    }
-    addReplyErrorLength(c,s,sdslen(s));
+    addReplyErrorSafe(c, s, sdslen(s));
     sdsfree(s);
 }
 
@@ -916,17 +897,24 @@ static void acceptCommonHandler(connection *conn, int flags, char *ip) {
     client *c;
     UNUSED(ip);
 
-    /* Admission control will happen before a client is created and connAccept()
+    /* Limit the number of connections we take at the same time.
+     *
+     * Admission control will happen before a client is created and connAccept()
      * called, because we don't want to even start transport-level negotiation
-     * if rejected.
-     */
-    if (listLength(server.clients) >= server.maxclients) {
-        char *err = "-ERR max number of clients reached\r\n";
+     * if rejected. */
+    if (listLength(server.clients) + getClusterConnectionsCount()
+        >= server.maxclients)
+    {
+        char *err;
+        if (server.cluster_enabled)
+            err = "-ERR max number of clients reached\r\n";
+        else
+            err = "-ERR max number of clients + cluster "
+                  "connections reached\r\n";
 
         /* That's a best effort error message, don't check write errors.
-         * Note that for TLS connections, no handshake was done yet so nothing is written
-         * and the connection will just drop.
-         */
+         * Note that for TLS connections, no handshake was done yet so nothing
+         * is written and the connection will just drop. */
         if (connWrite(conn,err,strlen(err)) == -1) {
             /* Nothing to do, Just to avoid the warning... */
         }
@@ -1037,25 +1025,13 @@ static void freeClientArgv(client *c) {
 
 /* Close all the slaves connections. This is useful in chained replication
  * when we resync with our own master and want to force all our slaves to
- * resync with us as well.
- *
- * If 'async' is non-zero we free the clients asynchronously. This is needed
- * when we call this function from a context where in the chain of the
- * callers somebody is iterating the list of clients. For instance when
- * CLIENT KILL TYPE master is called, caching the master client may
- * adjust the meaningful offset of replication, and in turn call
- * discionectSlaves(). Since CLIENT KILL iterates the clients this is
- * not safe. */
-void disconnectSlaves(int async) {
+ * resync with us as well. */
+void disconnectSlaves(void) {
     listIter li;
     listNode *ln;
     listRewind(server.slaves,&li);
     while((ln = listNext(&li))) {
-        listNode *ln = listFirst(server.slaves);
-        if (async)
-            freeClientAsync((client*)ln->value);
-        else
-            freeClient((client*)ln->value);
+        freeClient((client*)ln->value);
     }
 }
 
@@ -1274,14 +1250,20 @@ void freeClientAsync(client *c) {
 /* Free the clietns marked as CLOSE_ASAP, return the number of clients
  * freed. */
 int freeClientsInAsyncFreeQueue(void) {
-    int freed = listLength(server.clients_to_close);
-    while (listLength(server.clients_to_close)) {
-        listNode *ln = listFirst(server.clients_to_close);
+    int freed = 0;
+    listIter li;
+    listNode *ln;
+
+    listRewind(server.clients_to_close,&li);
+    while ((ln = listNext(&li)) != NULL) {
         client *c = listNodeValue(ln);
 
+        if (c->flags & CLIENT_PROTECTED) continue;
+
         c->flags &= ~CLIENT_CLOSE_ASAP;
         freeClient(c);
         listDelNode(server.clients_to_close,ln);
+        freed++;
     }
     return freed;
 }
@@ -1769,7 +1751,6 @@ int processMultibulkBuffer(client *c) {
  * 2. In the case of master clients, the replication offset is updated.
  * 3. Propagate commands we got from our master to replicas down the line. */
 void commandProcessed(client *c) {
-    int cmd_is_ping = c->cmd && c->cmd->proc == pingCommand;
     long long prev_offset = c->reploff;
     if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) {
         /* Update the applied replication offset of our master. */
@@ -1794,16 +1775,11 @@ void commandProcessed(client *c) {
      * sub-replicas and to the replication backlog. */
     if (c->flags & CLIENT_MASTER) {
         long long applied = c->reploff - prev_offset;
-        long long prev_master_repl_meaningful_offset = server.master_repl_meaningful_offset;
         if (applied) {
             replicationFeedSlavesFromMasterStream(server.slaves,
                     c->pending_querybuf, applied);
             sdsrange(c->pending_querybuf,applied,-1);
         }
-        /* The server.master_repl_meaningful_offset variable represents
-         * the offset of the replication stream without the pending PINGs. */
-        if (cmd_is_ping)
-            server.master_repl_meaningful_offset = prev_master_repl_meaningful_offset;
     }
 }
 
@@ -2563,7 +2539,7 @@ void helloCommand(client *c) {
     addReplyBulkCString(c,REDIS_VERSION);
 
     addReplyBulkCString(c,"proto");
-    addReplyLongLong(c,3);
+    addReplyLongLong(c,ver);
 
     addReplyBulkCString(c,"id");
     addReplyLongLong(c,c->id);

src/rax.c

@@ -487,8 +487,8 @@ static inline size_t raxLowWalk(rax *rax, unsigned char *s, size_t len, raxNode
         if (h->iscompr) j = 0; /* Compressed node only child is at index 0. */
         memcpy(&h,children+j,sizeof(h));
         parentlink = children+j;
-        j = 0; /* If the new node is compressed and we do not
-                  iterate again (since i == l) set the split
+        j = 0; /* If the new node is non compressed and we do not
+                  iterate again (since i == len) set the split
                   position to 0 to signal this node represents
                   the searched key. */
     }

src/redis-cli.c

@@ -1989,6 +1989,7 @@ static void repl(void) {
 
             if (argv == NULL) {
                 printf("Invalid argument(s)\n");
+                fflush(stdout);
                 linenoiseFree(line);
                 continue;
             } else if (argc > 0) {
@@ -6784,10 +6785,53 @@ void sendCapa() {
     sendReplconf("capa", "eof");
 }
 
+/* Wrapper around hiredis to allow arbitrary reads and writes.
+ *
+ * We piggybacks on top of hiredis to achieve transparent TLS support,
+ * and use its internal buffers so it can co-exist with commands
+ * previously/later issued on the connection.
+ *
+ * Interface is close to enough to read()/write() so things should mostly
+ * work transparently.
+ */
+
+/* Write a raw buffer through a redisContext. If we already have something
+ * in the buffer (leftovers from hiredis operations) it will be written
+ * as well.
+ */
+static ssize_t writeConn(redisContext *c, const char *buf, size_t buf_len)
+{
+    int done = 0;
+
+    c->obuf = sdscatlen(c->obuf, buf, buf_len);
+    if (redisBufferWrite(c, &done) == REDIS_ERR) {
+        sdsrange(c->obuf, 0, -(buf_len+1));
+        if (!(c->flags & REDIS_BLOCK))
+            errno = EAGAIN;
+        return -1;
+    }
+
+    size_t left = sdslen(c->obuf);
+    sdsclear(c->obuf);
+    if (!done) {
+        return buf_len - left;
+    }
+
+    return buf_len;
+}
+
+/* Read raw bytes through a redisContext. The read operation is not greedy
+ * and may not fill the buffer entirely.
+ */
+static ssize_t readConn(redisContext *c, char *buf, size_t len)
+{
+    return c->funcs->read(c, buf, len);
+}
+
 /* Sends SYNC and reads the number of bytes in the payload. Used both by
  * slaveMode() and getRDB().
  * returns 0 in case an EOF marker is used. */
-unsigned long long sendSync(int fd, char *out_eof) {
+unsigned long long sendSync(redisContext *c, char *out_eof) {
     /* To start we need to send the SYNC command and return the payload.
      * The hiredis client lib does not understand this part of the protocol
      * and we don't want to mess with its buffers, so everything is performed
@@ -6796,7 +6840,7 @@ unsigned long long sendSync(int fd, char *out_eof) {
     ssize_t nread;
 
     /* Send the SYNC command. */
-    if (write(fd,"SYNC\r\n",6) != 6) {
+    if (writeConn(c, "SYNC\r\n", 6) != 6) {
         fprintf(stderr,"Error writing to master\n");
         exit(1);
     }
@@ -6804,7 +6848,7 @@ unsigned long long sendSync(int fd, char *out_eof) {
     /* Read $<payload>\r\n, making sure to read just up to "\n" */
     p = buf;
     while(1) {
-        nread = read(fd,p,1);
+        nread = readConn(c,p,1);
         if (nread <= 0) {
             fprintf(stderr,"Error reading bulk length while SYNCing\n");
             exit(1);
@@ -6825,11 +6869,10 @@ unsigned long long sendSync(int fd, char *out_eof) {
 }
 
 static void slaveMode(void) {
-    int fd = context->fd;
     static char eofmark[RDB_EOF_MARK_SIZE];
     static char lastbytes[RDB_EOF_MARK_SIZE];
     static int usemark = 0;
-    unsigned long long payload = sendSync(fd, eofmark);
+    unsigned long long payload = sendSync(context,eofmark);
     char buf[1024];
     int original_output = config.output;
 
@@ -6849,7 +6892,7 @@ static void slaveMode(void) {
     while(payload) {
         ssize_t nread;
 
-        nread = read(fd,buf,(payload > sizeof(buf)) ? sizeof(buf) : payload);
+        nread = readConn(context,buf,(payload > sizeof(buf)) ? sizeof(buf) : payload);
         if (nread <= 0) {
             fprintf(stderr,"Error reading RDB payload while SYNCing\n");
             exit(1);
@@ -6892,14 +6935,15 @@ static void slaveMode(void) {
 /* This function implements --rdb, so it uses the replication protocol in order
  * to fetch the RDB file from a remote server. */
 static void getRDB(clusterManagerNode *node) {
-    int s, fd;
+    int fd;
+    redisContext *s;
     char *filename;
     if (node != NULL) {
         assert(node->context);
-        s = node->context->fd;
+        s = node->context;
         filename = clusterManagerGetNodeRDBFilename(node);
     } else {
-        s = context->fd;
+        s = context;
         filename = config.rdb_filename;
     }
     static char eofmark[RDB_EOF_MARK_SIZE];
@@ -6934,7 +6978,7 @@ static void getRDB(clusterManagerNode *node) {
     while(payload) {
         ssize_t nread, nwritten;
 
-        nread = read(s,buf,(payload > sizeof(buf)) ? sizeof(buf) : payload);
+        nread = readConn(s,buf,(payload > sizeof(buf)) ? sizeof(buf) : payload);
         if (nread <= 0) {
             fprintf(stderr,"I/O Error reading RDB payload from socket\n");
             exit(1);
@@ -6968,7 +7012,7 @@ static void getRDB(clusterManagerNode *node) {
     } else {
         fprintf(stderr,"Transfer finished with success.\n");
     }
-    close(s); /* Close the file descriptor ASAP as fsync() may take time. */
+    redisFree(s); /* Close the file descriptor ASAP as fsync() may take time. */
     fsync(fd);
     close(fd);
     fprintf(stderr,"Transfer finished with success.\n");
@@ -6985,11 +7029,9 @@ static void getRDB(clusterManagerNode *node) {
 
 #define PIPEMODE_WRITE_LOOP_MAX_BYTES (128*1024)
 static void pipeMode(void) {
-    int fd = context->fd;
     long long errors = 0, replies = 0, obuf_len = 0, obuf_pos = 0;
-    char ibuf[1024*16], obuf[1024*16]; /* Input and output buffers */
+    char obuf[1024*16]; /* Output buffer */
     char aneterr[ANET_ERR_LEN];
-    redisReader *reader = redisReaderCreate();
     redisReply *reply;
     int eof = 0; /* True once we consumed all the standard input. */
     int done = 0;
@@ -6999,47 +7041,38 @@ static void pipeMode(void) {
     srand(time(NULL));
 
     /* Use non blocking I/O. */
-    if (anetNonBlock(aneterr,fd) == ANET_ERR) {
+    if (anetNonBlock(aneterr,context->fd) == ANET_ERR) {
         fprintf(stderr, "Can't set the socket in non blocking mode: %s\n",
             aneterr);
         exit(1);
     }
 
+    context->flags &= ~REDIS_BLOCK;
+
     /* Transfer raw protocol and read replies from the server at the same
      * time. */
     while(!done) {
         int mask = AE_READABLE;
 
         if (!eof || obuf_len != 0) mask |= AE_WRITABLE;
-        mask = aeWait(fd,mask,1000);
+        mask = aeWait(context->fd,mask,1000);
 
         /* Handle the readable state: we can read replies from the server. */
         if (mask & AE_READABLE) {
-            ssize_t nread;
             int read_error = 0;
 
-            /* Read from socket and feed the hiredis reader. */
             do {
-                nread = read(fd,ibuf,sizeof(ibuf));
-                if (nread == -1 && errno != EAGAIN && errno != EINTR) {
-                    fprintf(stderr, "Error reading from the server: %s\n",
-                        strerror(errno));
+                if (!read_error && redisBufferRead(context) == REDIS_ERR) {
                     read_error = 1;
-                    break;
                 }
-                if (nread > 0) {
-                    redisReaderFeed(reader,ibuf,nread);
-                    last_read_time = time(NULL);
-                }
-            } while(nread > 0);
 
-            /* Consume replies. */
-            do {
-                if (redisReaderGetReply(reader,(void**)&reply) == REDIS_ERR) {
+                reply = NULL;
+                if (redisGetReply(context, (void **) &reply) == REDIS_ERR) {
                     fprintf(stderr, "Error reading replies from server\n");
                     exit(1);
                 }
                 if (reply) {
+                    last_read_time = time(NULL);
                     if (reply->type == REDIS_REPLY_ERROR) {
                         fprintf(stderr,"%s\n", reply->str);
                         errors++;
@@ -7072,7 +7105,7 @@ static void pipeMode(void) {
             while(1) {
                 /* Transfer current buffer to server. */
                 if (obuf_len != 0) {
-                    ssize_t nwritten = write(fd,obuf+obuf_pos,obuf_len);
+                    ssize_t nwritten = writeConn(context,obuf+obuf_pos,obuf_len);
 
                     if (nwritten == -1) {
                         if (errno != EAGAIN && errno != EINTR) {
@@ -7088,6 +7121,10 @@ static void pipeMode(void) {
                     loop_nwritten += nwritten;
                     if (obuf_len != 0) break; /* Can't accept more data. */
                 }
+                if (context->err) {
+                    fprintf(stderr, "Server I/O Error: %s\n", context->errstr);
+                    exit(1);
+                }
                 /* If buffer is empty, load from stdin. */
                 if (obuf_len == 0 && !eof) {
                     ssize_t nread = read(STDIN_FILENO,obuf,sizeof(obuf));
@@ -7138,7 +7175,6 @@ static void pipeMode(void) {
             break;
         }
     }
-    redisReaderFree(reader);
     printf("errors: %lld, replies: %lld\n", errors, replies);
     if (errors)
         exit(1);
@@ -7246,7 +7282,9 @@ static void getKeyTypes(dict *types_dict, redisReply *keys, typeinfo **types) {
 
     /* Pipeline TYPE commands */
     for(i=0;i<keys->elements;i++) {
-        redisAppendCommand(context, "TYPE %s", keys->element[i]->str);
+        const char* argv[] = {"TYPE", keys->element[i]->str};
+        size_t lens[] = {4, keys->element[i]->len};
+        redisAppendCommandArgv(context, 2, argv, lens);
     }
 
     /* Retrieve types */
@@ -7292,15 +7330,21 @@ static void getKeySizes(redisReply *keys, typeinfo **types,
         if(!types[i] || (!types[i]->sizecmd && !memkeys))
             continue;
 
-        if (!memkeys)
-            redisAppendCommand(context, "%s %s",
-                types[i]->sizecmd, keys->element[i]->str);
-        else if (memkeys_samples==0)
-            redisAppendCommand(context, "%s %s %s",
-                "MEMORY", "USAGE", keys->element[i]->str);
-        else
-            redisAppendCommand(context, "%s %s %s SAMPLES %u",
-                "MEMORY", "USAGE", keys->element[i]->str, memkeys_samples);
+        if (!memkeys) {
+            const char* argv[] = {types[i]->sizecmd, keys->element[i]->str};
+            size_t lens[] = {strlen(types[i]->sizecmd), keys->element[i]->len};
+            redisAppendCommandArgv(context, 2, argv, lens);
+        } else if (memkeys_samples==0) {
+            const char* argv[] = {"MEMORY", "USAGE", keys->element[i]->str};
+            size_t lens[] = {6, 5, keys->element[i]->len};
+            redisAppendCommandArgv(context, 3, argv, lens);
+        } else {
+            sds samplesstr = sdsfromlonglong(memkeys_samples);
+            const char* argv[] = {"MEMORY", "USAGE", keys->element[i]->str, "SAMPLES", samplesstr};
+            size_t lens[] = {6, 5, keys->element[i]->len, 7, sdslen(samplesstr)};
+            redisAppendCommandArgv(context, 5, argv, lens);
+            sdsfree(samplesstr);
+        }
     }
 
     /* Retrieve sizes */
@@ -7396,20 +7440,20 @@ static void findBigKeys(int memkeys, unsigned memkeys_samples) {
             sampled++;
 
             if(type->biggest<sizes[i]) {
-                printf(
-                   "[%05.2f%%] Biggest %-6s found so far '%s' with %llu %s\n",
-                   pct, type->name, keys->element[i]->str, sizes[i],
-                   !memkeys? type->sizeunit: "bytes");
-
                 /* Keep track of biggest key name for this type */
                 if (type->biggest_key)
                     sdsfree(type->biggest_key);
-                type->biggest_key = sdsnew(keys->element[i]->str);
+                type->biggest_key = sdscatrepr(sdsempty(), keys->element[i]->str, keys->element[i]->len);
                 if(!type->biggest_key) {
                     fprintf(stderr, "Failed to allocate memory for key!\n");
                     exit(1);
                 }
 
+                printf(
+                   "[%05.2f%%] Biggest %-6s found so far '%s' with %llu %s\n",
+                   pct, type->name, type->biggest_key, sizes[i],
+                   !memkeys? type->sizeunit: "bytes");
+
                 /* Keep track of the biggest size for this type */
                 type->biggest = sizes[i];
             }
@@ -7473,21 +7517,27 @@ static void getKeyFreqs(redisReply *keys, unsigned long long *freqs) {
 
     /* Pipeline OBJECT freq commands */
     for(i=0;i<keys->elements;i++) {
-        redisAppendCommand(context, "OBJECT freq %s", keys->element[i]->str);
+        const char* argv[] = {"OBJECT", "FREQ", keys->element[i]->str};
+        size_t lens[] = {6, 4, keys->element[i]->len};
+        redisAppendCommandArgv(context, 3, argv, lens);
     }
 
     /* Retrieve freqs */
     for(i=0;i<keys->elements;i++) {
         if(redisGetReply(context, (void**)&reply)!=REDIS_OK) {
+            sds keyname = sdscatrepr(sdsempty(), keys->element[i]->str, keys->element[i]->len);
             fprintf(stderr, "Error getting freq for key '%s' (%d: %s)\n",
-                keys->element[i]->str, context->err, context->errstr);
+                keyname, context->err, context->errstr);
+            sdsfree(keyname);
             exit(1);
         } else if(reply->type != REDIS_REPLY_INTEGER) {
             if(reply->type == REDIS_REPLY_ERROR) {
                 fprintf(stderr, "Error: %s\n", reply->str);
                 exit(1);
             } else {
-                fprintf(stderr, "Warning: OBJECT freq on '%s' failed (may have been deleted)\n", keys->element[i]->str);
+                sds keyname = sdscatrepr(sdsempty(), keys->element[i]->str, keys->element[i]->len);
+                fprintf(stderr, "Warning: OBJECT freq on '%s' failed (may have been deleted)\n", keyname);
+                sdsfree(keyname);
                 freqs[i] = 0;
             }
         } else {
@@ -7558,10 +7608,10 @@ static void findHotKeys(void) {
                 memmove(hotkeys,hotkeys+1,sizeof(hotkeys[0])*k);
             }
             counters[k] = freqs[i];
-            hotkeys[k] = sdsnew(keys->element[i]->str);
+            hotkeys[k] = sdscatrepr(sdsempty(), keys->element[i]->str, keys->element[i]->len);
             printf(
                "[%05.2f%%] Hot key '%s' found so far with counter %llu\n",
-               pct, keys->element[i]->str, freqs[i]);
+               pct, hotkeys[k], freqs[i]);
         }
 
         /* Sleep if we've been directed to do so */

src/replication.c

@@ -39,7 +39,6 @@
 #include <sys/socket.h>
 #include <sys/stat.h>
 
-long long adjustMeaningfulReplOffset(int *adjusted);
 void replicationDiscardCachedMaster(void);
 void replicationResurrectCachedMaster(connection *conn);
 void replicationSendAck(void);
@@ -163,7 +162,6 @@ void feedReplicationBacklog(void *ptr, size_t len) {
     unsigned char *p = ptr;
 
     server.master_repl_offset += len;
-    server.master_repl_meaningful_offset = server.master_repl_offset;
 
     /* This is a circular buffer, so write as much data we can at every
      * iteration and rewind the "idx" index if we reach the limit. */
@@ -309,6 +307,40 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) {
     }
 }
 
+/* This is a debugging function that gets called when we detect something
+ * wrong with the replication protocol: the goal is to peek into the
+ * replication backlog and show a few final bytes to make simpler to
+ * guess what kind of bug it could be. */
+void showLatestBacklog(void) {
+    if (server.repl_backlog == NULL) return;
+
+    long long dumplen = 256;
+    if (server.repl_backlog_histlen < dumplen)
+        dumplen = server.repl_backlog_histlen;
+
+    /* Identify the first byte to dump. */
+    long long idx =
+      (server.repl_backlog_idx + (server.repl_backlog_size - dumplen)) %
+       server.repl_backlog_size;
+
+    /* Scan the circular buffer to collect 'dumplen' bytes. */
+    sds dump = sdsempty();
+    while(dumplen) {
+        long long thislen =
+            ((server.repl_backlog_size - idx) < dumplen) ?
+            (server.repl_backlog_size - idx) : dumplen;
+
+        dump = sdscatrepr(dump,server.repl_backlog+idx,thislen);
+        dumplen -= thislen;
+        idx = 0;
+    }
+
+    /* Finally log such bytes: this is vital debugging info to
+     * understand what happened. */
+    serverLog(LL_WARNING,"Latest backlog is: '%s'", dump);
+    sdsfree(dump);
+}
+
 /* This function is used in order to proxy what we receive from our master
  * to our sub-slaves. */
 #include <ctype.h>
@@ -976,41 +1008,10 @@ void removeRDBUsedToSyncReplicas(void) {
     }
 }
 
-#if HAVE_SENDFILE
-/* Implements redis_sendfile to transfer data between file descriptors and
- * avoid transferring data to and from user space.
- * 
- * The function prototype is just like sendfile(2) on Linux. in_fd is a file
- * descriptor opened for reading and out_fd is a descriptor opened for writing.
- * offset specifies where to start reading data from in_fd. count is the number
- * of bytes to copy between the file descriptors.
- * 
- * The return value is the number of bytes written to out_fd, if the transfer
- * was successful. On error, -1 is returned, and errno is set appropriately. */
-ssize_t redis_sendfile(int out_fd, int in_fd, off_t offset, size_t count) {
-#if defined(__linux__)
-    #include <sys/sendfile.h>
-    return sendfile(out_fd, in_fd, &offset, count);
-
-#elif defined(__APPLE__)
-    off_t len = count;
-    /* Notice that it may return -1 and errno is set to EAGAIN even if some
-     * bytes have been sent successfully and the len argument is set correctly
-     * when using a socket marked for non-blocking I/O. */
-    if (sendfile(in_fd, out_fd, offset, &len, NULL, 0) == -1 &&
-        errno != EAGAIN) return -1;
-    else
-        return (ssize_t)len;
-
-#endif
-    errno = ENOSYS;
-    return -1;
-}
-#endif
-
 void sendBulkToSlave(connection *conn) {
     client *slave = connGetPrivateData(conn);
-    ssize_t nwritten;
+    char buf[PROTO_IOBUF_LEN];
+    ssize_t nwritten, buflen;
 
     /* Before sending the RDB file, we send the preamble as configured by the
      * replication process. Currently the preamble is just the bulk count of
@@ -1035,46 +1036,23 @@ void sendBulkToSlave(connection *conn) {
         }
     }
 
-    /* If the preamble was already transferred, send the RDB bulk data.
-     * try to use sendfile system call if supported, unless tls is enabled.
-     * fallback to normal read+write otherwise. */
-    nwritten = 0;
-#if HAVE_SENDFILE
-    if (!server.tls_replication) {
-        if ((nwritten = redis_sendfile(conn->fd,slave->repldbfd,
-            slave->repldboff,PROTO_IOBUF_LEN)) == -1)
-        {
-            if (errno != EAGAIN) {
-                serverLog(LL_WARNING,"Sendfile error sending DB to replica: %s",
-                    strerror(errno));
-                freeClient(slave);
-            }
-            return;
-        }
+    /* If the preamble was already transferred, send the RDB bulk data. */
+    lseek(slave->repldbfd,slave->repldboff,SEEK_SET);
+    buflen = read(slave->repldbfd,buf,PROTO_IOBUF_LEN);
+    if (buflen <= 0) {
+        serverLog(LL_WARNING,"Read error sending DB to replica: %s",
+            (buflen == 0) ? "premature EOF" : strerror(errno));
+        freeClient(slave);
+        return;
     }
-#endif
-    if (!nwritten) {
-        ssize_t buflen;
-        char buf[PROTO_IOBUF_LEN];
-
-        lseek(slave->repldbfd,slave->repldboff,SEEK_SET);
-        buflen = read(slave->repldbfd,buf,PROTO_IOBUF_LEN);
-        if (buflen <= 0) {
-            serverLog(LL_WARNING,"Read error sending DB to replica: %s",
-                (buflen == 0) ? "premature EOF" : strerror(errno));
+    if ((nwritten = connWrite(conn,buf,buflen)) == -1) {
+        if (connGetState(conn) != CONN_STATE_CONNECTED) {
+            serverLog(LL_WARNING,"Write error sending DB to replica: %s",
+                connGetLastError(conn));
             freeClient(slave);
-            return;
-        }
-        if ((nwritten = connWrite(conn,buf,buflen)) == -1) {
-            if (connGetState(conn) != CONN_STATE_CONNECTED) {
-                serverLog(LL_WARNING,"Write error sending DB to replica: %s",
-                    connGetLastError(conn));
-                freeClient(slave);
-            }
-            return;
         }
+        return;
     }
-
     slave->repldboff += nwritten;
     server.stat_net_output_bytes += nwritten;
     if (slave->repldboff == slave->repldbsize) {
@@ -1831,7 +1809,6 @@ void readSyncBulkPayload(connection *conn) {
      * we are starting a new history. */
     memcpy(server.replid,server.master->replid,sizeof(server.replid));
     server.master_repl_offset = server.master->reploff;
-    server.master_repl_meaningful_offset = server.master->reploff;
     clearReplicationId2();
 
     /* Let's create the replication backlog if needed. Slaves need to
@@ -2086,7 +2063,7 @@ int slaveTryPartialResynchronization(connection *conn, int read_reply) {
                 memcpy(server.cached_master->replid,new,sizeof(server.replid));
 
                 /* Disconnect all the sub-slaves: they need to be notified. */
-                disconnectSlaves(0);
+                disconnectSlaves();
             }
         }
 
@@ -2359,7 +2336,7 @@ void syncWithMaster(connection *conn) {
      * as well, if we have any sub-slaves. The master may transfer us an
      * entirely different data set and we have no way to incrementally feed
      * our slaves after that. */
-    disconnectSlaves(0); /* Force our slaves to resync with us as well. */
+    disconnectSlaves(); /* Force our slaves to resync with us as well. */
     freeReplicationBacklog(); /* Don't allow our chained slaves to PSYNC. */
 
     /* Fall back to SYNC if needed. Otherwise psync_result == PSYNC_FULLRESYNC
@@ -2506,7 +2483,7 @@ void replicationSetMaster(char *ip, int port) {
 
     /* Force our slaves to resync with us as well. They may hopefully be able
      * to partially resync with us, but we can notify the replid change. */
-    disconnectSlaves(0);
+    disconnectSlaves();
     cancelReplicationHandshake();
     /* Before destroying our master state, create a cached master using
      * our own parameters, to later PSYNC with the new master. */
@@ -2557,7 +2534,7 @@ void replicationUnsetMaster(void) {
      * of the replication ID change (see shiftReplicationId() call). However
      * the slaves will be able to partially resync with us, so it will be
      * a very fast reconnection. */
-    disconnectSlaves(0);
+    disconnectSlaves();
     server.repl_state = REPL_STATE_NONE;
 
     /* We need to make sure the new master will start the replication stream
@@ -2759,11 +2736,6 @@ void replicationCacheMaster(client *c) {
      * pending outputs to the master. */
     sdsclear(server.master->querybuf);
     sdsclear(server.master->pending_querybuf);
-
-    /* Adjust reploff and read_reploff to the last meaningful offset we
-     * executed. This is the offset the replica will use for future PSYNC. */
-    int offset_adjusted;
-    server.master->reploff = adjustMeaningfulReplOffset(&offset_adjusted);
     server.master->read_reploff = server.master->reploff;
     if (c->flags & CLIENT_MULTI) discardTransaction(c);
     listEmpty(c->reply);
@@ -2786,53 +2758,6 @@ void replicationCacheMaster(client *c) {
      * so make sure to adjust the replication state. This function will
      * also set server.master to NULL. */
     replicationHandleMasterDisconnection();
-
-    /* If we trimmed this replica backlog, we need to disconnect our chained
-     * replicas (if any), otherwise they may have the PINGs we removed
-     * from the stream and their offset would no longer match: upon
-     * disconnection they will also trim the final PINGs and will be able
-     * to incrementally sync without issues. */
-    if (offset_adjusted) disconnectSlaves(1);
-}
-
-/* If the "meaningful" offset, that is the offset without the final PINGs
- * in the stream, is different than the last offset, use it instead:
- * often when the master is no longer reachable, replicas will never
- * receive the PINGs, however the master will end with an incremented
- * offset because of the PINGs and will not be able to incrementally
- * PSYNC with the new master.
- * This function trims the replication backlog when needed, and returns
- * the offset to be used for future partial sync.
- *
- * If the integer 'adjusted' was passed by reference, it is set to 1
- * if the function call actually modified the offset and the replication
- * backlog, otherwise it is set to 0. It can be NULL if the caller is
- * not interested in getting this info. */
-long long adjustMeaningfulReplOffset(int *adjusted) {
-    if (server.master_repl_offset > server.master_repl_meaningful_offset) {
-        long long delta = server.master_repl_offset -
-                          server.master_repl_meaningful_offset;
-        serverLog(LL_NOTICE,
-            "Using the meaningful offset %lld instead of %lld to exclude "
-            "the final PINGs (%lld bytes difference)",
-                server.master_repl_meaningful_offset,
-                server.master_repl_offset,
-                delta);
-        server.master_repl_offset = server.master_repl_meaningful_offset;
-        if (server.repl_backlog_histlen <= delta) {
-            server.repl_backlog_histlen = 0;
-            server.repl_backlog_idx = 0;
-        } else {
-            server.repl_backlog_histlen -= delta;
-            server.repl_backlog_idx =
-                (server.repl_backlog_idx + (server.repl_backlog_size - delta)) %
-                server.repl_backlog_size;
-        }
-        if (adjusted) *adjusted = 1;
-    } else {
-        if (adjusted) *adjusted = 0;
-    }
-    return server.master_repl_offset;
 }
 
 /* This function is called when a master is turend into a slave, in order to
@@ -2854,7 +2779,7 @@ void replicationCacheMasterUsingMyself(void) {
      * by replicationCreateMasterClient(). We'll later set the created
      * master as server.cached_master, so the replica will use such
      * offset for PSYNC. */
-    server.master_initial_offset = adjustMeaningfulReplOffset(NULL);
+    server.master_initial_offset = server.master_repl_offset;
 
     /* The master client we create can be set to any DBID, because
      * the new master will start its replication stream with SELECT. */
@@ -3246,18 +3171,10 @@ void replicationCron(void) {
             clientsArePaused();
 
         if (!manual_failover_in_progress) {
-            long long before_ping = server.master_repl_meaningful_offset;
             ping_argv[0] = createStringObject("PING",4);
             replicationFeedSlaves(server.slaves, server.slaveseldb,
                 ping_argv, 1);
             decrRefCount(ping_argv[0]);
-            /* The server.master_repl_meaningful_offset variable represents
-             * the offset of the replication stream without the pending PINGs.
-             * This is useful to set the right replication offset for PSYNC
-             * when the master is turned into a replica. Otherwise pending
-             * PINGs may not allow it to perform an incremental sync with the
-             * new master. */
-            server.master_repl_meaningful_offset = before_ping;
         }
     }
 

src/sentinel.c

@@ -1076,6 +1076,7 @@ int sentinelTryConnectionSharing(sentinelRedisInstance *ri) {
         releaseInstanceLink(ri->link,NULL);
         ri->link = match->link;
         match->link->refcount++;
+        dictReleaseIterator(di);
         return C_OK;
     }
     dictReleaseIterator(di);

src/server.c

@@ -326,6 +326,10 @@ struct redisCommand redisCommandTable[] = {
      "write @list",
      0,NULL,1,1,1,0,0,0},
 
+    {"lpos",lposCommand,-3,
+     "read-only @list",
+     0,NULL,1,1,1,0,0,0},
+
     {"lrem",lremCommand,4,
      "write @list",
      0,NULL,1,1,1,0,0,0},
@@ -776,11 +780,11 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"watch",watchCommand,-2,
-     "no-script fast @transaction",
+     "no-script fast ok-loading ok-stale @transaction",
      0,NULL,1,-1,1,0,0,0},
 
     {"unwatch",unwatchCommand,1,
-     "no-script fast @transaction",
+     "no-script fast ok-loading ok-stale @transaction",
      0,NULL,0,0,0,0,0,0},
 
     {"cluster",clusterCommand,-2,
@@ -2394,7 +2398,6 @@ void initServerConfig(void) {
     server.repl_syncio_timeout = CONFIG_REPL_SYNCIO_TIMEOUT;
     server.repl_down_since = 0; /* Never connected, repl is down since EVER. */
     server.master_repl_offset = 0;
-    server.master_repl_meaningful_offset = 0;
 
     /* Replication partial resync backlog */
     server.repl_backlog = NULL;
@@ -3399,6 +3402,34 @@ void call(client *c, int flags) {
     server.stat_numcommands++;
 }
 
+/* Used when a command that is ready for execution needs to be rejected, due to
+ * varios pre-execution checks. it returns the appropriate error to the client.
+ * If there's a transaction is flags it as dirty, and if the command is EXEC,
+ * it aborts the transaction. */
+void rejectCommand(client *c, robj *reply) {
+    flagTransaction(c);
+    if (c->cmd && c->cmd->proc == execCommand) {
+        execCommandAbort(c, reply->ptr);
+    } else {
+        /* using addReplyError* rather than addReply so that the error can be logged. */
+        addReplyErrorSafe(c, reply->ptr, sdslen(reply->ptr));
+    }
+}
+
+void rejectCommandFormat(client *c, const char *fmt, ...) {
+    flagTransaction(c);
+    va_list ap;
+    va_start(ap,fmt);
+    sds s = sdscatvprintf(sdsempty(),fmt,ap);
+    va_end(ap);
+    if (c->cmd && c->cmd->proc == execCommand) {
+        execCommandAbort(c, s);
+    } else {
+        addReplyErrorSafe(c, s, sdslen(s));
+    }
+    sdsfree(s);
+}
+
 /* If this function gets called we already read a whole
  * command, arguments are in the client argv/argc fields.
  * processCommand() execute the command or prepare the
@@ -3424,23 +3455,30 @@ int processCommand(client *c) {
      * such as wrong arity, bad command name and so forth. */
     c->cmd = c->lastcmd = lookupCommand(c->argv[0]->ptr);
     if (!c->cmd) {
-        flagTransaction(c);
         sds args = sdsempty();
         int i;
         for (i=1; i < c->argc && sdslen(args) < 128; i++)
             args = sdscatprintf(args, "`%.*s`, ", 128-(int)sdslen(args), (char*)c->argv[i]->ptr);
-        addReplyErrorFormat(c,"unknown command `%s`, with args beginning with: %s",
+        rejectCommandFormat(c,"unknown command `%s`, with args beginning with: %s",
             (char*)c->argv[0]->ptr, args);
         sdsfree(args);
         return C_OK;
     } else if ((c->cmd->arity > 0 && c->cmd->arity != c->argc) ||
                (c->argc < -c->cmd->arity)) {
-        flagTransaction(c);
-        addReplyErrorFormat(c,"wrong number of arguments for '%s' command",
+        rejectCommandFormat(c,"wrong number of arguments for '%s' command",
             c->cmd->name);
         return C_OK;
     }
 
+    int is_write_command = (c->cmd->flags & CMD_WRITE) ||
+                           (c->cmd->proc == execCommand && (c->mstate.cmd_flags & CMD_WRITE));
+    int is_denyoom_command = (c->cmd->flags & CMD_DENYOOM) ||
+                             (c->cmd->proc == execCommand && (c->mstate.cmd_flags & CMD_DENYOOM));
+    int is_denystale_command = !(c->cmd->flags & CMD_STALE) ||
+                               (c->cmd->proc == execCommand && (c->mstate.cmd_inv_flags & CMD_STALE));
+    int is_denyloading_command = !(c->cmd->flags & CMD_LOADING) ||
+                                 (c->cmd->proc == execCommand && (c->mstate.cmd_inv_flags & CMD_LOADING));
+
     /* Check if the user is authenticated. This check is skipped in case
      * the default user is flagged as "nopass" and is active. */
     int auth_required = (!(DefaultUser->flags & USER_FLAG_NOPASS) ||
@@ -3450,8 +3488,7 @@ int processCommand(client *c) {
         /* AUTH and HELLO and no auth modules are valid even in
          * non-authenticated state. */
         if (!(c->cmd->flags & CMD_NO_AUTH)) {
-            flagTransaction(c);
-            addReply(c,shared.noautherr);
+            rejectCommand(c,shared.noautherr);
             return C_OK;
         }
     }
@@ -3462,13 +3499,12 @@ int processCommand(client *c) {
     int acl_retval = ACLCheckCommandPerm(c,&acl_keypos);
     if (acl_retval != ACL_OK) {
         addACLLogEntry(c,acl_retval,acl_keypos,NULL);
-        flagTransaction(c);
         if (acl_retval == ACL_DENIED_CMD)
-            addReplyErrorFormat(c,
+            rejectCommandFormat(c,
                 "-NOPERM this user has no permissions to run "
                 "the '%s' command or its subcommand", c->cmd->name);
         else
-            addReplyErrorFormat(c,
+            rejectCommandFormat(c,
                 "-NOPERM this user has no permissions to access "
                 "one of the keys used as arguments");
         return C_OK;
@@ -3516,13 +3552,11 @@ int processCommand(client *c) {
          * is trying to execute is denied during OOM conditions or the client
          * is in MULTI/EXEC context? Error. */
         if (out_of_memory &&
-            (c->cmd->flags & CMD_DENYOOM ||
+            (is_denyoom_command ||
              (c->flags & CLIENT_MULTI &&
-              c->cmd->proc != execCommand &&
               c->cmd->proc != discardCommand)))
         {
-            flagTransaction(c);
-            addReply(c, shared.oomerr);
+            rejectCommand(c, shared.oomerr);
             return C_OK;
         }
 
@@ -3543,17 +3577,14 @@ int processCommand(client *c) {
     int deny_write_type = writeCommandsDeniedByDiskError();
     if (deny_write_type != DISK_ERROR_TYPE_NONE &&
         server.masterhost == NULL &&
-        (c->cmd->flags & CMD_WRITE ||
-         c->cmd->proc == pingCommand))
+        (is_write_command ||c->cmd->proc == pingCommand))
     {
-        flagTransaction(c);
         if (deny_write_type == DISK_ERROR_TYPE_RDB)
-            addReply(c, shared.bgsaveerr);
+            rejectCommand(c, shared.bgsaveerr);
         else
-            addReplySds(c,
-                sdscatprintf(sdsempty(),
+            rejectCommandFormat(c,
                 "-MISCONF Errors writing to the AOF file: %s\r\n",
-                strerror(server.aof_last_write_errno)));
+                strerror(server.aof_last_write_errno));
         return C_OK;
     }
 
@@ -3562,11 +3593,10 @@ int processCommand(client *c) {
     if (server.masterhost == NULL &&
         server.repl_min_slaves_to_write &&
         server.repl_min_slaves_max_lag &&
-        c->cmd->flags & CMD_WRITE &&
+        is_write_command &&
         server.repl_good_slaves_count < server.repl_min_slaves_to_write)
     {
-        flagTransaction(c);
-        addReply(c, shared.noreplicaserr);
+        rejectCommand(c, shared.noreplicaserr);
         return C_OK;
     }
 
@@ -3574,10 +3604,9 @@ int processCommand(client *c) {
      * accept write commands if this is our master. */
     if (server.masterhost && server.repl_slave_ro &&
         !(c->flags & CLIENT_MASTER) &&
-        c->cmd->flags & CMD_WRITE)
+        is_write_command)
     {
-        flagTransaction(c);
-        addReply(c, shared.roslaveerr);
+        rejectCommand(c, shared.roslaveerr);
         return C_OK;
     }
 
@@ -3589,7 +3618,7 @@ int processCommand(client *c) {
         c->cmd->proc != unsubscribeCommand &&
         c->cmd->proc != psubscribeCommand &&
         c->cmd->proc != punsubscribeCommand) {
-        addReplyErrorFormat(c,
+        rejectCommandFormat(c,
             "Can't execute '%s': only (P)SUBSCRIBE / "
             "(P)UNSUBSCRIBE / PING / QUIT are allowed in this context",
             c->cmd->name);
@@ -3601,17 +3630,16 @@ int processCommand(client *c) {
      * link with master. */
     if (server.masterhost && server.repl_state != REPL_STATE_CONNECTED &&
         server.repl_serve_stale_data == 0 &&
-        !(c->cmd->flags & CMD_STALE))
+        is_denystale_command)
     {
-        flagTransaction(c);
-        addReply(c, shared.masterdownerr);
+        rejectCommand(c, shared.masterdownerr);
         return C_OK;
     }
 
     /* Loading DB? Return an error if the command has not the
      * CMD_LOADING flag. */
-    if (server.loading && !(c->cmd->flags & CMD_LOADING)) {
-        addReply(c, shared.loadingerr);
+    if (server.loading && is_denyloading_command) {
+        rejectCommand(c, shared.loadingerr);
         return C_OK;
     }
 
@@ -3626,8 +3654,9 @@ int processCommand(client *c) {
           c->cmd->proc != helloCommand &&
           c->cmd->proc != replconfCommand &&
           c->cmd->proc != multiCommand &&
-          c->cmd->proc != execCommand &&
           c->cmd->proc != discardCommand &&
+          c->cmd->proc != watchCommand &&
+          c->cmd->proc != unwatchCommand &&
         !(c->cmd->proc == shutdownCommand &&
           c->argc == 2 &&
           tolower(((char*)c->argv[1]->ptr)[0]) == 'n') &&
@@ -3635,8 +3664,7 @@ int processCommand(client *c) {
           c->argc == 2 &&
           tolower(((char*)c->argv[1]->ptr)[0]) == 'k'))
     {
-        flagTransaction(c);
-        addReply(c, shared.slowscripterr);
+        rejectCommand(c, shared.slowscripterr);
         return C_OK;
     }
 
@@ -3675,6 +3703,15 @@ void closeListeningSockets(int unlink_unix_socket) {
 }
 
 int prepareForShutdown(int flags) {
+    /* When SHUTDOWN is called while the server is loading a dataset in
+     * memory we need to make sure no attempt is performed to save
+     * the dataset on shutdown (otherwise it could overwrite the current DB
+     * with half-read data).
+     *
+     * Also when in Sentinel mode clear the SAVE flag and force NOSAVE. */
+    if (server.loading || server.sentinel_mode)
+        flags = (flags & ~SHUTDOWN_SAVE) | SHUTDOWN_NOSAVE;
+
     int save = flags & SHUTDOWN_SAVE;
     int nosave = flags & SHUTDOWN_NOSAVE;
 
@@ -4471,7 +4508,6 @@ sds genRedisInfoString(const char *section) {
             "master_replid:%s\r\n"
             "master_replid2:%s\r\n"
             "master_repl_offset:%lld\r\n"
-            "master_repl_meaningful_offset:%lld\r\n"
             "second_repl_offset:%lld\r\n"
             "repl_backlog_active:%d\r\n"
             "repl_backlog_size:%lld\r\n"
@@ -4480,7 +4516,6 @@ sds genRedisInfoString(const char *section) {
             server.replid,
             server.replid2,
             server.master_repl_offset,
-            server.master_repl_meaningful_offset,
             server.second_replid_offset,
             server.repl_backlog != NULL,
             server.repl_backlog_size,
@@ -4858,7 +4893,6 @@ void loadDataFromDisk(void) {
             {
                 memcpy(server.replid,rsi.repl_id,sizeof(server.replid));
                 server.master_repl_offset = rsi.repl_offset;
-                server.master_repl_meaningful_offset = rsi.repl_offset;
                 /* If we are a slave, create a cached master from this
                  * information, in order to allow partial resynchronizations
                  * with masters. */

src/server.h

@@ -666,6 +666,9 @@ typedef struct multiState {
     int cmd_flags;          /* The accumulated command flags OR-ed together.
                                So if at least a command has a given flag, it
                                will be set in this field. */
+    int cmd_inv_flags;      /* Same as cmd_flags, OR-ing the ~flags. so that it
+                               is possible to know if all the commands have a
+                               certain flag. */
     int minreplicas;        /* MINREPLICAS for synchronous replication */
     time_t minreplicas_timeout; /* MINREPLICAS timeout as unixtime. */
 } multiState;
@@ -1008,6 +1011,9 @@ typedef struct redisTLSContextConfig {
     char *ciphers;
     char *ciphersuites;
     int prefer_server_ciphers;
+    int session_caching;
+    int session_cache_size;
+    int session_cache_timeout;
 } redisTLSContextConfig;
 
 /*-----------------------------------------------------------------------------
@@ -1261,7 +1267,6 @@ struct redisServer {
     char replid[CONFIG_RUN_ID_SIZE+1];  /* My current replication ID. */
     char replid2[CONFIG_RUN_ID_SIZE+1]; /* replid inherited from master*/
     long long master_repl_offset;   /* My current replication offset */
-    long long master_repl_meaningful_offset; /* Offset minus latest PINGs. */
     long long second_replid_offset; /* Accept offsets up to this for replid2. */
     int slaveseldb;                 /* Last SELECTed DB in replication output */
     int repl_ping_slave_period;     /* Master pings the slave every N seconds */
@@ -1627,6 +1632,7 @@ void addReplyBulkLongLong(client *c, long long ll);
 void addReply(client *c, robj *obj);
 void addReplySds(client *c, sds s);
 void addReplyBulkSds(client *c, sds s);
+void addReplyErrorSafe(client *c, char *s, size_t len);
 void addReplyError(client *c, const char *err);
 void addReplyStatus(client *c, const char *status);
 void addReplyDouble(client *c, double d);
@@ -1660,7 +1666,7 @@ int getClientType(client *c);
 int getClientTypeByName(char *name);
 char *getClientTypeName(int class);
 void flushSlavesOutputBuffers(void);
-void disconnectSlaves(int async);
+void disconnectSlaves(void);
 int listenToPort(int port, int *fds, int *count);
 void pauseClients(mstime_t duration);
 int clientsArePaused(void);
@@ -1725,6 +1731,7 @@ void touchWatchedKey(redisDb *db, robj *key);
 void touchWatchedKeysOnFlush(int dbid);
 void discardTransaction(client *c);
 void flagTransaction(client *c);
+void execCommandAbort(client *c, sds error);
 void execCommandPropagateMulti(client *c);
 void execCommandPropagateExec(client *c);
 
@@ -1811,6 +1818,7 @@ void clearReplicationId2(void);
 void chopReplicationBacklog(void);
 void replicationCacheMasterUsingMyself(void);
 void feedReplicationBacklog(void *ptr, size_t len);
+void showLatestBacklog(void);
 void rdbPipeReadHandler(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask);
 void rdbPipeWriteHandlerConnRemoved(struct connection *conn);
 
@@ -2065,6 +2073,7 @@ void propagateExpire(redisDb *db, robj *key, int lazy);
 int expireIfNeeded(redisDb *db, robj *key);
 long long getExpire(redisDb *db, robj *key);
 void setExpire(client *c, redisDb *db, robj *key, long long when);
+int checkAlreadyExpired(long long when);
 robj *lookupKey(redisDb *db, robj *key, int flags);
 robj *lookupKeyRead(redisDb *db, robj *key);
 robj *lookupKeyWrite(redisDb *db, robj *key);
@@ -2269,6 +2278,7 @@ void flushdbCommand(client *c);
 void flushallCommand(client *c);
 void sortCommand(client *c);
 void lremCommand(client *c);
+void lposCommand(client *c);
 void rpoplpushCommand(client *c);
 void infoCommand(client *c);
 void mgetCommand(client *c);

src/t_list.c

@@ -487,6 +487,122 @@ void ltrimCommand(client *c) {
     addReply(c,shared.ok);
 }
 
+/* LPOS key element [RANK rank] [COUNT num-matches] [MAXLEN len]
+ *
+ * The "rank" is the position of the match, so if it is 1, the first match
+ * is returned, if it is 2 the second match is returned and so forth.
+ * It is 1 by default. If negative has the same meaning but the search is
+ * performed starting from the end of the list.
+ *
+ * If COUNT is given, instead of returning the single element, a list of
+ * all the matching elements up to "num-matches" are returned. COUNT can
+ * be combiled with RANK in order to returning only the element starting
+ * from the Nth. If COUNT is zero, all the matching elements are returned.
+ *
+ * MAXLEN tells the command to scan a max of len elements. If zero (the
+ * default), all the elements in the list are scanned if needed.
+ *
+ * The returned elements indexes are always referring to what LINDEX
+ * would return. So first element from head is 0, and so forth. */
+void lposCommand(client *c) {
+    robj *o, *ele;
+    ele = c->argv[2];
+    int direction = LIST_TAIL;
+    long rank = 1, count = -1, maxlen = 0; /* Count -1: option not given. */
+
+    /* Parse the optional arguments. */
+    for (int j = 3; j < c->argc; j++) {
+        char *opt = c->argv[j]->ptr;
+        int moreargs = (c->argc-1)-j;
+
+        if (!strcasecmp(opt,"RANK") && moreargs) {
+            j++;
+            if (getLongFromObjectOrReply(c, c->argv[j], &rank, NULL) != C_OK)
+                return;
+            if (rank == 0) {
+                addReplyError(c,"RANK can't be zero: use 1 to start from "
+                                "the first match, 2 from the second, ...");
+                return;
+            }
+        } else if (!strcasecmp(opt,"COUNT") && moreargs) {
+            j++;
+            if (getLongFromObjectOrReply(c, c->argv[j], &count, NULL) != C_OK)
+                return;
+            if (count < 0) {
+                addReplyError(c,"COUNT can't be negative");
+                return;
+            }
+        } else if (!strcasecmp(opt,"MAXLEN") && moreargs) {
+            j++;
+            if (getLongFromObjectOrReply(c, c->argv[j], &maxlen, NULL) != C_OK)
+                return;
+            if (maxlen < 0) {
+                addReplyError(c,"MAXLEN can't be negative");
+                return;
+            }
+        } else {
+            addReply(c,shared.syntaxerr);
+            return;
+        }
+    }
+
+    /* A negative rank means start from the tail. */
+    if (rank < 0) {
+        rank = -rank;
+        direction = LIST_HEAD;
+    }
+
+    /* We return NULL or an empty array if there is no such key (or
+     * if we find no matches, depending on the presence of the COUNT option. */
+    if ((o = lookupKeyRead(c->db,c->argv[1])) == NULL) {
+        if (count != -1)
+            addReply(c,shared.emptyarray);
+        else
+            addReply(c,shared.null[c->resp]);
+        return;
+    }
+    if (checkType(c,o,OBJ_LIST)) return;
+
+    /* If we got the COUNT option, prepare to emit an array. */
+    void *arraylenptr = NULL;
+    if (count != -1) arraylenptr = addReplyDeferredLen(c);
+
+    /* Seek the element. */
+    listTypeIterator *li;
+    li = listTypeInitIterator(o,direction == LIST_HEAD ? -1 : 0,direction);
+    listTypeEntry entry;
+    long llen = listTypeLength(o);
+    long index = 0, matches = 0, matchindex = -1;
+    while (listTypeNext(li,&entry) && (maxlen == 0 || index < maxlen)) {
+        if (listTypeEqual(&entry,ele)) {
+            matches++;
+            matchindex = (direction == LIST_TAIL) ? index : llen - index - 1;
+            if (matches >= rank) {
+                if (arraylenptr) {
+                    addReplyLongLong(c,matchindex);
+                    if (count && matches-rank+1 >= count) break;
+                } else {
+                    break;
+                }
+            }
+        }
+        index++;
+        matchindex = -1; /* Remember if we exit the loop without a match. */
+    }
+    listTypeReleaseIterator(li);
+
+    /* Reply to the client. Note that arraylenptr is not NULL only if
+     * the COUNT option was selected. */
+    if (arraylenptr != NULL) {
+        setDeferredArrayLen(c,arraylenptr,matches-rank+1);
+    } else {
+        if (matchindex != -1)
+            addReplyLongLong(c,matchindex);
+        else
+            addReply(c,shared.null[c->resp]);
+    }
+}
+
 void lremCommand(client *c) {
     robj *subject, *obj;
     obj = c->argv[3];

src/t_string.c

@@ -516,13 +516,13 @@ void stralgoLCS(client *c) {
             withmatchlen = 1;
         } else if (!strcasecmp(opt,"MINMATCHLEN") && moreargs) {
             if (getLongLongFromObjectOrReply(c,c->argv[j+1],&minmatchlen,NULL)
-                != C_OK) return;
+                != C_OK) goto cleanup;
             if (minmatchlen < 0) minmatchlen = 0;
             j++;
         } else if (!strcasecmp(opt,"STRINGS") && moreargs > 1) {
             if (a != NULL) {
                 addReplyError(c,"Either use STRINGS or KEYS");
-                return;
+                goto cleanup;
             }
             a = c->argv[j+1]->ptr;
             b = c->argv[j+2]->ptr;
@@ -530,10 +530,21 @@ void stralgoLCS(client *c) {
         } else if (!strcasecmp(opt,"KEYS") && moreargs > 1) {
             if (a != NULL) {
                 addReplyError(c,"Either use STRINGS or KEYS");
-                return;
+                goto cleanup;
             }
             obja = lookupKeyRead(c->db,c->argv[j+1]);
             objb = lookupKeyRead(c->db,c->argv[j+2]);
+            if ((obja && obja->type != OBJ_STRING) ||
+                (objb && objb->type != OBJ_STRING))
+            {
+                addReplyError(c,
+                    "The specified keys must contain string values");
+                /* Don't cleanup the objects, we need to do that
+                 * only after callign getDecodedObject(). */
+                obja = NULL;
+                objb = NULL;
+                goto cleanup;
+            }
             obja = obja ? getDecodedObject(obja) : createStringObject("",0);
             objb = objb ? getDecodedObject(objb) : createStringObject("",0);
             a = obja->ptr;
@@ -541,7 +552,7 @@ void stralgoLCS(client *c) {
             j += 2;
         } else {
             addReply(c,shared.syntaxerr);
-            return;
+            goto cleanup;
         }
     }
 
@@ -549,12 +560,12 @@ void stralgoLCS(client *c) {
     if (a == NULL) {
         addReplyError(c,"Please specify two strings: "
                         "STRINGS or KEYS options are mandatory");
-        return;
+        goto cleanup;
     } else if (getlen && getidx) {
         addReplyError(c,
             "If you want both the length and indexes, please "
             "just use IDX.");
-        return;
+        goto cleanup;
     }
 
     /* Compute the LCS using the vanilla dynamic programming technique of
@@ -689,10 +700,12 @@ void stralgoLCS(client *c) {
     }
 
     /* Cleanup. */
-    if (obja) decrRefCount(obja);
-    if (objb) decrRefCount(objb);
     sdsfree(result);
     zfree(lcs);
+
+cleanup:
+    if (obja) decrRefCount(obja);
+    if (objb) decrRefCount(objb);
     return;
 }
 

src/tls.c

@@ -148,9 +148,6 @@ void tlsInit(void) {
     }
 
     pending_list = listCreate();
-
-    /* Server configuration */
-    server.tls_auth_clients = 1;    /* Secure by default */
 }
 
 /* Attempt to configure/reconfigure TLS. This operation is atomic and will
@@ -184,6 +181,15 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
     SSL_CTX_set_options(ctx, SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS);
 #endif
 
+    if (ctx_config->session_caching) {
+        SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_SERVER);
+        SSL_CTX_sess_set_cache_size(ctx, ctx_config->session_cache_size);
+        SSL_CTX_set_timeout(ctx, ctx_config->session_cache_timeout);
+        SSL_CTX_set_session_id_context(ctx, (void *) "redis", 5);
+    } else {
+        SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_OFF);
+    }
+
     int protocols = parseProtocolsConfig(ctx_config->protocols);
     if (protocols == -1) goto error;
 
@@ -217,7 +223,7 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
     SSL_CTX_set_ecdh_auto(ctx, 1);
 #endif
 
-    if (SSL_CTX_use_certificate_file(ctx, ctx_config->cert_file, SSL_FILETYPE_PEM) <= 0) {
+    if (SSL_CTX_use_certificate_chain_file(ctx, ctx_config->cert_file) <= 0) {
         ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf));
         serverLog(LL_WARNING, "Failed to load certificate: %s: %s", ctx_config->cert_file, errbuf);
         goto error;
@@ -337,9 +343,7 @@ connection *connCreateAcceptedTLS(int fd, int require_auth) {
     conn->c.state = CONN_STATE_ACCEPTING;
 
     if (!require_auth) {
-        /* We still verify certificates if provided, but don't require them.
-         */
-        SSL_set_verify(conn->ssl, SSL_VERIFY_PEER, NULL);
+        SSL_set_verify(conn->ssl, SSL_VERIFY_NONE, NULL);
     }
 
     SSL_set_fd(conn->ssl, conn->c.fd);

src/tracking.c

@@ -102,7 +102,7 @@ void disableTracking(client *c) {
 /* Set the client 'c' to track the prefix 'prefix'. If the client 'c' is
  * already registered for the specified prefix, no operation is performed. */
 void enableBcastTrackingForPrefix(client *c, char *prefix, size_t plen) {
-    bcastState *bs = raxFind(PrefixTable,(unsigned char*)prefix,sdslen(prefix));
+    bcastState *bs = raxFind(PrefixTable,(unsigned char*)prefix,plen);
     /* If this is the first client subscribing to such prefix, create
      * the prefix in the table. */
     if (bs == raxNotFound) {

tests/instances.tcl

@@ -25,6 +25,7 @@ set ::sentinel_instances {}
 set ::redis_instances {}
 set ::sentinel_base_port 20000
 set ::redis_base_port 30000
+set ::redis_port_count 1024
 set ::pids {} ; # We kill everything at exit
 set ::dirs {} ; # We remove all the temp dirs at exit
 set ::run_matching {} ; # If non empty, only tests matching pattern are run.
@@ -57,7 +58,7 @@ proc exec_instance {type cfgfile} {
 # Spawn a redis or sentinel instance, depending on 'type'.
 proc spawn_instance {type base_port count {conf {}}} {
     for {set j 0} {$j < $count} {incr j} {
-        set port [find_available_port $base_port]
+        set port [find_available_port $base_port $::redis_port_count]
         incr base_port
         puts "Starting $type #$j at port $port"
 

tests/integration/psync2-pingoff.tcl

@@ -1,6 +1,9 @@
-# Test the meaningful offset implementation to make sure masters
-# are able to PSYNC with replicas even if the replication stream
-# has pending PINGs at the end.
+# These tests were added together with the meaningful offset implementation
+# in redis 6.0.0, which was later abandoned in 6.0.4, they used to test that
+# servers are able to PSYNC with replicas even if the replication stream has
+# PINGs at the end which present in one sever and missing on another.
+# We keep these tests just because they reproduce edge cases in the replication
+# logic in hope they'll be able to spot some problem in the future.
 
 start_server {tags {"psync2"}} {
 start_server {} {
@@ -16,7 +19,7 @@ start_server {} {
     }
 
     # Setup replication
-    test "PSYNC2 meaningful offset: setup" {
+    test "PSYNC2 pingoff: setup" {
         $R(1) replicaof $R_host(0) $R_port(0)
         $R(0) set foo bar
         wait_for_condition 50 1000 {
@@ -27,7 +30,7 @@ start_server {} {
         }
     }
 
-    test "PSYNC2 meaningful offset: write and wait replication" {
+    test "PSYNC2 pingoff: write and wait replication" {
         $R(0) INCR counter
         $R(0) INCR counter
         $R(0) INCR counter
@@ -41,7 +44,7 @@ start_server {} {
     # In this test we'll make sure the replica will get stuck, but with
     # an active connection: this way the master will continue to send PINGs
     # every second (we modified the PING period earlier)
-    test "PSYNC2 meaningful offset: pause replica and promote it" {
+    test "PSYNC2 pingoff: pause replica and promote it" {
         $R(1) MULTI
         $R(1) DEBUG SLEEP 5
         $R(1) SLAVEOF NO ONE
@@ -50,18 +53,127 @@ start_server {} {
     }
 
     test "Make the old master a replica of the new one and check conditions" {
-        set sync_partial [status $R(1) sync_partial_ok]
-        assert {$sync_partial == 0}
+        assert_equal [status $R(1) sync_full] 0
         $R(0) REPLICAOF $R_host(1) $R_port(1)
         wait_for_condition 50 1000 {
-            [status $R(1) sync_partial_ok] == 1
+            [status $R(1) sync_full] == 1
         } else {
-            fail "The new master was not able to partial sync"
+            fail "The new master was not able to sync"
         }
+
+        # make sure replication is still alive and kicking
+        $R(1) incr x
+        wait_for_condition 50 1000 {
+            [status $R(0) loading] == 0 &&
+            [$R(0) get x] == 1
+        } else {
+            fail "replica didn't get incr"
+        }
+        assert_equal [status $R(0) master_repl_offset] [status $R(1) master_repl_offset]
     }
 }}
 
 
+start_server {tags {"psync2"}} {
+start_server {} {
+start_server {} {
+start_server {} {
+start_server {} {
+    test {test various edge cases of repl topology changes with missing pings at the end} {
+        set master [srv -4 client]
+        set master_host [srv -4 host]
+        set master_port [srv -4 port]
+        set replica1 [srv -3 client]
+        set replica2 [srv -2 client]
+        set replica3 [srv -1 client]
+        set replica4 [srv -0 client]
+
+        $replica1 replicaof $master_host $master_port
+        $replica2 replicaof $master_host $master_port
+        $replica3 replicaof $master_host $master_port
+        $replica4 replicaof $master_host $master_port
+        wait_for_condition 50 1000 {
+            [status $master connected_slaves] == 4
+        } else {
+            fail "replicas didn't connect"
+        }
+
+        $master incr x
+        wait_for_condition 50 1000 {
+            [$replica1 get x] == 1 && [$replica2 get x] == 1 &&
+            [$replica3 get x] == 1 && [$replica4 get x] == 1
+        } else {
+            fail "replicas didn't get incr"
+        }
+
+        # disconnect replica1 and replica2
+        # and wait for the master to send a ping to replica3 and replica4
+        $replica1 replicaof no one
+        $replica2 replicaof 127.0.0.1 1 ;# we can't promote it to master since that will cycle the replication id
+        $master config set repl-ping-replica-period 1
+        after 1500
+
+        # make everyone sync from the replica1 that didn't get the last ping from the old master
+        # replica4 will keep syncing from the old master which now syncs from replica1
+        # and replica2 will re-connect to the old master (which went back in time)
+        set new_master_host [srv -3 host]
+        set new_master_port [srv -3 port]
+        $replica3 replicaof $new_master_host $new_master_port
+        $master replicaof $new_master_host $new_master_port
+        $replica2 replicaof $master_host $master_port
+        wait_for_condition 50 1000 {
+            [status $replica2 master_link_status] == "up" &&
+            [status $replica3 master_link_status] == "up" &&
+            [status $replica4 master_link_status] == "up" &&
+            [status $master master_link_status] == "up"
+        } else {
+            fail "replicas didn't connect"
+        }
+
+        # make sure replication is still alive and kicking
+        $replica1 incr x
+        wait_for_condition 50 1000 {
+            [$replica2 get x] == 2 &&
+            [$replica3 get x] == 2 &&
+            [$replica4 get x] == 2 &&
+            [$master get x] == 2
+        } else {
+            fail "replicas didn't get incr"
+        }
+
+        # make sure we have the right amount of full syncs
+        assert_equal [status $master sync_full] 6
+        assert_equal [status $replica1 sync_full] 2
+        assert_equal [status $replica2 sync_full] 0
+        assert_equal [status $replica3 sync_full] 0
+        assert_equal [status $replica4 sync_full] 0
+
+        # force psync
+        $master client kill type master
+        $replica2 client kill type master
+        $replica3 client kill type master
+        $replica4 client kill type master
+
+        # make sure replication is still alive and kicking
+        $replica1 incr x
+        wait_for_condition 50 1000 {
+            [$replica2 get x] == 3 &&
+            [$replica3 get x] == 3 &&
+            [$replica4 get x] == 3 &&
+            [$master get x] == 3
+        } else {
+            fail "replicas didn't get incr"
+        }
+
+        # make sure we have the right amount of full syncs
+        assert_equal [status $master sync_full] 6
+        assert_equal [status $replica1 sync_full] 2
+        assert_equal [status $replica2 sync_full] 0
+        assert_equal [status $replica3 sync_full] 0
+        assert_equal [status $replica4 sync_full] 0
+}
+}}}}}
+
 start_server {tags {"psync2"}} {
 start_server {} {
 start_server {} {
@@ -97,7 +209,7 @@ start_server {} {
             [status $R(1) master_link_status] == "up" && 
             [status $R(2) master_link_status] == "up" &&
             [status $R(0) sync_partial_ok] == $sync_partial_master + 1 &&
-            [status $R(1) sync_partial_ok] == $sync_partial_replica + 1
+            [status $R(1) sync_partial_ok] == $sync_partial_replica
         } else {
             fail "Disconnected replica failed to PSYNC with master"
         }
@@ -106,7 +218,15 @@ start_server {} {
         # offsets match with the master
         assert_equal [status $R(0) master_repl_offset] [status $R(1) master_repl_offset]
         assert_equal [status $R(0) master_repl_offset] [status $R(2) master_repl_offset]
-        assert_equal [status $R(0) master_repl_meaningful_offset] [status $R(1) master_repl_meaningful_offset]
-        assert_equal [status $R(0) master_repl_meaningful_offset] [status $R(2) master_repl_meaningful_offset]
+
+        # make sure replication is still alive and kicking
+        $R(0) incr counter2
+        wait_for_condition 50 1000 {
+            [$R(1) get counter2] == 2 && [$R(2) get counter2] == 2
+        } else {
+            fail "replicas didn't get incr"
+        }
+        assert_equal [status $R(0) master_repl_offset] [status $R(1) master_repl_offset]
+        assert_equal [status $R(0) master_repl_offset] [status $R(2) master_repl_offset]
     }
 }}}

tests/integration/psync2.tcl

@@ -242,7 +242,6 @@ start_server {} {
             show_cluster_status
             fail "Replicas and master offsets were unable to match *exactly*."
         }
-        $R($master_id) config set repl-ping-replica-period 10
 
         # Limit anyway the maximum number of cycles. This is useful when the
         # test is skipped via --only option of the test suite. In that case
@@ -281,7 +280,8 @@ start_server {} {
         set sync_partial_err [status $R($master_id) sync_partial_err]
         catch {
             $R($slave_id) config rewrite
-            $R($slave_id) debug restart
+            restart_server [expr {0-$slave_id}] true
+            set R($slave_id) [srv [expr {0-$slave_id}] client]
         }
         # note: just waiting for connected_slaves==4 has a race condition since
         # we might do the check before the master realized that the slave disconnected
@@ -329,7 +329,8 @@ start_server {} {
 
         catch {
             $R($slave_id) config rewrite
-            $R($slave_id) debug restart
+            restart_server [expr {0-$slave_id}] true
+            set R($slave_id) [srv [expr {0-$slave_id}] client]
         }
 
         # Reconfigure the slave correctly again, when it's back online.
@@ -370,103 +371,3 @@ start_server {} {
     }
 
 }}}}}
-
-start_server {tags {"psync2"}} {
-start_server {} {
-start_server {} {
-start_server {} {
-start_server {} {
-    test {pings at the end of replication stream are ignored for psync} {
-        set master [srv -4 client]
-        set master_host [srv -4 host]
-        set master_port [srv -4 port]
-        set replica1 [srv -3 client]
-        set replica2 [srv -2 client]
-        set replica3 [srv -1 client]
-        set replica4 [srv -0 client]
-
-        $replica1 replicaof $master_host $master_port
-        $replica2 replicaof $master_host $master_port
-        $replica3 replicaof $master_host $master_port
-        $replica4 replicaof $master_host $master_port
-        wait_for_condition 50 1000 {
-            [status $master connected_slaves] == 4
-        } else {
-            fail "replicas didn't connect"
-        }
-
-        $master incr x
-        wait_for_condition 50 1000 {
-            [$replica1 get x] == 1 && [$replica2 get x] == 1 &&
-            [$replica3 get x] == 1 && [$replica4 get x] == 1
-        } else {
-            fail "replicas didn't get incr"
-        }
-
-        # disconnect replica1 and replica2
-        # and wait for the master to send a ping to replica3 and replica4
-        $replica1 replicaof no one
-        $replica2 replicaof 127.0.0.1 1 ;# we can't promote it to master since that will cycle the replication id
-        $master config set repl-ping-replica-period 1
-        after 1500
-
-        # make everyone sync from the replica1 that didn't get the last ping from the old master
-        # replica4 will keep syncing from the old master which now syncs from replica1
-        # and replica2 will re-connect to the old master (which went back in time)
-        set new_master_host [srv -3 host]
-        set new_master_port [srv -3 port]
-        $replica3 replicaof $new_master_host $new_master_port
-        $master replicaof $new_master_host $new_master_port
-        $replica2 replicaof $master_host $master_port
-        wait_for_condition 50 1000 {
-            [status $replica2 master_link_status] == "up" &&
-            [status $replica3 master_link_status] == "up" &&
-            [status $replica4 master_link_status] == "up" &&
-            [status $master master_link_status] == "up"
-        } else {
-            fail "replicas didn't connect"
-        }
-
-        # make sure replication is still alive and kicking
-        $replica1 incr x
-        wait_for_condition 50 1000 {
-            [$replica2 get x] == 2 &&
-            [$replica3 get x] == 2 &&
-            [$replica4 get x] == 2 &&
-            [$master get x] == 2
-        } else {
-            fail "replicas didn't get incr"
-        }
-
-        # make sure there are full syncs other than the initial ones
-        assert_equal [status $master sync_full] 4
-        assert_equal [status $replica1 sync_full] 0
-        assert_equal [status $replica2 sync_full] 0
-        assert_equal [status $replica3 sync_full] 0
-        assert_equal [status $replica4 sync_full] 0
-
-        # force psync
-        $master client kill type master
-        $replica2 client kill type master
-        $replica3 client kill type master
-        $replica4 client kill type master
-
-        # make sure replication is still alive and kicking
-        $replica1 incr x
-        wait_for_condition 50 1000 {
-            [$replica2 get x] == 3 &&
-            [$replica3 get x] == 3 &&
-            [$replica4 get x] == 3 &&
-            [$master get x] == 3
-        } else {
-            fail "replicas didn't get incr"
-        }
-
-        # make sure there are full syncs other than the initial ones
-        assert_equal [status $master sync_full] 4
-        assert_equal [status $replica1 sync_full] 0
-        assert_equal [status $replica2 sync_full] 0
-        assert_equal [status $replica3 sync_full] 0
-        assert_equal [status $replica4 sync_full] 0
-}
-}}}}}

tests/integration/rdb.tcl

@@ -137,18 +137,8 @@ test {client freed during loading} {
         # 100mb of rdb, 100k keys will load in more than 1 second
         r debug populate 100000 key 1000
 
-        catch {
-            r debug restart
-        }
+        restart_server 0 false
 
-        set stdout [srv 0 stdout]
-        while 1 {
-            # check that the new server actually started and is ready for connections
-            if {[exec grep -i "Server initialized" | wc -l < $stdout] > 1} {
-                break
-            }
-            after 10
-        }
         # make sure it's still loading
         assert_equal [s loading] 1
 
@@ -180,4 +170,4 @@ test {client freed during loading} {
         # no need to keep waiting for loading to complete
         exec kill [srv 0 pid]
     }
-}
+}

tests/integration/redis-cli.tcl

@@ -1,14 +1,13 @@
 source tests/support/cli.tcl
 
 start_server {tags {"cli"}} {
-    proc open_cli {} {
+    proc open_cli {{opts "-n 9"}} {
         set ::env(TERM) dumb
-        set cmdline [rediscli [srv port] "-n 9"]
+        set cmdline [rediscli [srv port] $opts]
         set fd [open "|$cmdline" "r+"]
         fconfigure $fd -buffering none
         fconfigure $fd -blocking false
         fconfigure $fd -translation binary
-        assert_equal "redis> " [read_cli $fd]
         set _ $fd
     }
 
@@ -32,11 +31,14 @@ start_server {tags {"cli"}} {
     }
 
     # Helpers to run tests in interactive mode
+
+    proc format_output {output} {
+        set _ [string trimright [regsub -all "\r" $output ""] "\n"]
+    }
+
     proc run_command {fd cmd} {
         write_cli $fd $cmd
-        set lines [split [read_cli $fd] "\n"]
-        assert_equal "redis> " [lindex $lines end]
-        join [lrange $lines 0 end-1] "\n"
+        set _ [format_output [read_cli $fd]]
     }
 
     proc test_interactive_cli {name code} {
@@ -58,7 +60,7 @@ start_server {tags {"cli"}} {
 
     proc _run_cli {opts args} {
         set cmd [rediscli [srv port] [list -n 9 {*}$args]]
-        foreach {key value} $args {
+        foreach {key value} $opts {
             if {$key eq "pipe"} {
                 set cmd "sh -c \"$value | $cmd\""
             }
@@ -72,7 +74,7 @@ start_server {tags {"cli"}} {
         fconfigure $fd -translation binary
         set resp [read $fd 1048576]
         close $fd
-        set _ $resp
+        set _ [format_output $resp]
     }
 
     proc run_cli {args} {
@@ -80,11 +82,11 @@ start_server {tags {"cli"}} {
     }
 
     proc run_cli_with_input_pipe {cmd args} {
-        _run_cli [list pipe $cmd] {*}$args
+        _run_cli [list pipe $cmd] -x {*}$args
     }
 
     proc run_cli_with_input_file {path args} {
-        _run_cli [list path $path] {*}$args
+        _run_cli [list path $path] -x {*}$args
     }
 
     proc test_nontty_cli {name code} {
@@ -101,7 +103,7 @@ start_server {tags {"cli"}} {
     test_interactive_cli "INFO response should be printed raw" {
         set lines [split [run_command $fd info] "\n"]
         foreach line $lines {
-            assert [regexp {^[a-z0-9_]+:[a-z0-9_]+} $line]
+            assert [regexp {^$|^#|^[a-z0-9_]+:.+} $line]
         }
     }
 
@@ -121,7 +123,7 @@ start_server {tags {"cli"}} {
     test_interactive_cli "Multi-bulk reply" {
         r rpush list foo
         r rpush list bar
-        assert_equal "1. \"foo\"\n2. \"bar\"" [run_command $fd "lrange list 0 -1"]
+        assert_equal "1) \"foo\"\n2) \"bar\"" [run_command $fd "lrange list 0 -1"]
     }
 
     test_interactive_cli "Parsing quotes" {
@@ -144,35 +146,35 @@ start_server {tags {"cli"}} {
     }
 
     test_tty_cli "Status reply" {
-        assert_equal "OK\n" [run_cli set key bar]
+        assert_equal "OK" [run_cli set key bar]
         assert_equal "bar" [r get key]
     }
 
     test_tty_cli "Integer reply" {
         r del counter
-        assert_equal "(integer) 1\n" [run_cli incr counter]
+        assert_equal "(integer) 1" [run_cli incr counter]
     }
 
     test_tty_cli "Bulk reply" {
         r set key "tab\tnewline\n"
-        assert_equal "\"tab\\tnewline\\n\"\n" [run_cli get key]
+        assert_equal "\"tab\\tnewline\\n\"" [run_cli get key]
     }
 
     test_tty_cli "Multi-bulk reply" {
         r del list
         r rpush list foo
         r rpush list bar
-        assert_equal "1. \"foo\"\n2. \"bar\"\n" [run_cli lrange list 0 -1]
+        assert_equal "1) \"foo\"\n2) \"bar\"" [run_cli lrange list 0 -1]
     }
 
     test_tty_cli "Read last argument from pipe" {
-        assert_equal "OK\n" [run_cli_with_input_pipe "echo foo" set key]
+        assert_equal "OK" [run_cli_with_input_pipe "echo foo" set key]
         assert_equal "foo\n" [r get key]
     }
 
     test_tty_cli "Read last argument from file" {
         set tmpfile [write_tmpfile "from file"]
-        assert_equal "OK\n" [run_cli_with_input_file $tmpfile set key]
+        assert_equal "OK" [run_cli_with_input_file $tmpfile set key]
         assert_equal "from file" [r get key]
     }
 
@@ -188,7 +190,7 @@ start_server {tags {"cli"}} {
 
     test_nontty_cli "Bulk reply" {
         r set key "tab\tnewline\n"
-        assert_equal "tab\tnewline\n" [run_cli get key]
+        assert_equal "tab\tnewline" [run_cli get key]
     }
 
     test_nontty_cli "Multi-bulk reply" {
@@ -208,4 +210,79 @@ start_server {tags {"cli"}} {
         assert_equal "OK" [run_cli_with_input_file $tmpfile set key]
         assert_equal "from file" [r get key]
     }
+
+    proc test_redis_cli_rdb_dump {} {
+        r flushdb
+
+        set dir [lindex [r config get dir] 1]
+
+        assert_equal "OK" [r debug populate 100000 key 1000]
+        catch {run_cli --rdb "$dir/cli.rdb"} output
+        assert_match {*Transfer finished with success*} $output
+
+        file delete "$dir/dump.rdb"
+        file rename "$dir/cli.rdb" "$dir/dump.rdb"
+
+        assert_equal "OK" [r set should-not-exist 1]
+        assert_equal "OK" [r debug reload nosave]
+        assert_equal {} [r get should-not-exist]
+    }
+
+    test_nontty_cli "Dumping an RDB" {
+        # Disk-based master
+        assert_match "OK" [r config set repl-diskless-sync no]
+        test_redis_cli_rdb_dump
+
+        # Disk-less master
+        assert_match "OK" [r config set repl-diskless-sync yes]
+        assert_match "OK" [r config set repl-diskless-sync-delay 0]
+        test_redis_cli_rdb_dump
+    }
+
+    test_nontty_cli "Connecting as a replica" {
+        set fd [open_cli "--replica"]
+        wait_for_condition 50 500 {
+            [string match {*slave0:*state=online*} [r info]]
+        } else {
+            fail "redis-cli --replica did not connect"
+        }
+
+        for {set i 0} {$i < 100} {incr i} {
+           r set test-key test-value-$i
+        }
+        r client kill type slave
+        catch {
+            assert_match {*SET*key-a*} [read_cli $fd]
+        }
+
+        close_cli $fd
+    }
+
+    test_nontty_cli "Piping raw protocol" {
+        set fd [open_cli "--pipe"]
+        fconfigure $fd -blocking true
+
+        # Create a new deferring client and overwrite its fd
+        set client [redis [srv 0 "host"] [srv 0 "port"] 1 0]
+        set ::redis::fd($::redis::id) $fd
+        $client select 9
+
+        r del test-counter
+        for {set i 0} {$i < 10000} {incr i} {
+            $client incr test-counter
+            $client set large-key [string repeat "x" 20000]
+        }
+
+        for {set i 0} {$i < 1000} {incr i} {
+            $client set very-large-key [string repeat "x" 512000]
+        }
+
+        close $fd write
+        set output [read_cli $fd]
+
+        assert_equal {10000} [r get test-counter]
+        assert_match {*All data transferred*errors: 0*replies: 21001*} $output
+
+        close_cli $fd
+    }
 }

tests/integration/replication.tcl

@@ -430,6 +430,7 @@ test {diskless loading short read} {
             }
 
             # Start the replication process...
+            set loglines [count_log_lines -1]
             $master config set repl-diskless-sync-delay 0
             $replica replicaof $master_host $master_port
 
@@ -439,7 +440,7 @@ test {diskless loading short read} {
             for {set i 0} {$i < $attempts} {incr i} {
                 # wait for the replica to start reading the rdb
                 # using the log file since the replica only responds to INFO once in 2mb
-                wait_for_log_message -1 "*Loading DB in memory*" 5 2000 1
+                wait_for_log_message -1 "*Loading DB in memory*" $loglines 2000 1
 
                 # add some additional random sleep so that we kill the master on a different place each time
                 after [expr {int(rand()*100)}]
@@ -448,7 +449,7 @@ test {diskless loading short read} {
                 set killed [$master client kill type replica]
 
                 if {[catch {
-                    set res [wait_for_log_message -1 "*Internal error in RDB*" 5 100 10]
+                    set res [wait_for_log_message -1 "*Internal error in RDB*" $loglines 100 10]
                     if {$::verbose} {
                         puts $res
                     }
@@ -461,6 +462,7 @@ test {diskless loading short read} {
                     $master config set repl-backlog-size [expr {16384 + $i}]
                 }
                 # wait for loading to stop (fail)
+                set loglines [count_log_lines -1]
                 wait_for_condition 100 10 {
                     [s -1 loading] eq 0
                 } else {
@@ -535,6 +537,7 @@ start_server {tags {"repl"}} {
                     # start replication
                     # it's enough for just one replica to be slow, and have it's write handler enabled
                     # so that the whole rdb generation process is bound to that
+                    set loglines [count_log_lines -1]
                     [lindex $replicas 0] config set repl-diskless-load swapdb
                     [lindex $replicas 0] config set key-load-delay 100
                     [lindex $replicas 0] replicaof $master_host $master_port
@@ -542,7 +545,7 @@ start_server {tags {"repl"}} {
 
                     # wait for the replicas to start reading the rdb
                     # using the log file since the replica only responds to INFO once in 2mb
-                    wait_for_log_message -1 "*Loading DB in memory*" 8 800 10
+                    wait_for_log_message -1 "*Loading DB in memory*" $loglines 800 10
 
                     if {$measure_time} {
                         set master_statfile "/proc/$master_pid/stat"
@@ -558,6 +561,7 @@ start_server {tags {"repl"}} {
                     $master incr $all_drop
 
                     # disconnect replicas depending on the current test
+                    set loglines [count_log_lines -2]
                     if {$all_drop == "all" || $all_drop == "fast"} {
                         exec kill [srv 0 pid]
                         set replicas_alive [lreplace $replicas_alive 1 1]
@@ -576,13 +580,13 @@ start_server {tags {"repl"}} {
 
                     # make sure we got what we were aiming for, by looking for the message in the log file
                     if {$all_drop == "all"} {
-                        wait_for_log_message -2 "*Diskless rdb transfer, last replica dropped, killing fork child*" 12 1 1
+                        wait_for_log_message -2 "*Diskless rdb transfer, last replica dropped, killing fork child*" $loglines 1 1
                     }
                     if {$all_drop == "no"} {
-                        wait_for_log_message -2 "*Diskless rdb transfer, done reading from pipe, 2 replicas still up*" 12 1 1
+                        wait_for_log_message -2 "*Diskless rdb transfer, done reading from pipe, 2 replicas still up*" $loglines 1 1
                     }
                     if {$all_drop == "slow" || $all_drop == "fast"} {
-                        wait_for_log_message -2 "*Diskless rdb transfer, done reading from pipe, 1 replicas still up*" 12 1 1
+                        wait_for_log_message -2 "*Diskless rdb transfer, done reading from pipe, 1 replicas still up*" $loglines 1 1
                     }
 
                     # make sure we don't have a busy loop going thought epoll_wait

tests/modules/Makefile

@@ -28,11 +28,14 @@ TEST_MODULES = \
 
 all: $(TEST_MODULES)
 
+32bit:
+	$(MAKE) CFLAGS="-m32" LDFLAGS="-melf_i386"
+
 %.xo: %.c ../../src/redismodule.h
 	$(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@
 
 %.so: %.xo
-	$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc
+	$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LDFLAGS) $(LIBS) -lc
 
 .PHONY: clean
 

tests/modules/scan.c

@@ -55,11 +55,23 @@ void scan_key_callback(RedisModuleKey *key, RedisModuleString* field, RedisModul
     REDISMODULE_NOT_USED(key);
     scan_key_pd* pd = privdata;
     RedisModule_ReplyWithArray(pd->ctx, 2);
-    RedisModule_ReplyWithString(pd->ctx, field);
-    if (value)
-        RedisModule_ReplyWithString(pd->ctx, value);
-    else
+    size_t fieldCStrLen;
+
+    // The implementation of RedisModuleString is robj with lots of encodings.
+    // We want to make sure the robj that passes to this callback in
+    // String encoded, this is why we use RedisModule_StringPtrLen and
+    // RedisModule_ReplyWithStringBuffer instead of directly use
+    // RedisModule_ReplyWithString.
+    const char* fieldCStr = RedisModule_StringPtrLen(field, &fieldCStrLen);
+    RedisModule_ReplyWithStringBuffer(pd->ctx, fieldCStr, fieldCStrLen);
+    if(value){
+        size_t valueCStrLen;
+        const char* valueCStr = RedisModule_StringPtrLen(value, &valueCStrLen);
+        RedisModule_ReplyWithStringBuffer(pd->ctx, valueCStr, valueCStrLen);
+    } else {
         RedisModule_ReplyWithNull(pd->ctx);
+    }
+
     pd->nreplies++;
 }
 

tests/support/server.tcl

@@ -17,7 +17,14 @@ proc check_valgrind_errors stderr {
     set buf [read $fd]
     close $fd
 
+    # look for stack trace and other errors, or the absense of a leak free summary
     if {[regexp -- { at 0x} $buf] ||
+        [regexp -- {Warning} $buf] ||
+        [regexp -- {Invalid} $buf] ||
+        [regexp -- {Mismatched} $buf] ||
+        [regexp -- {uninitialized} $buf] ||
+        [regexp -- {has a fishy} $buf] ||
+        [regexp -- {overlap} $buf] ||
         (![regexp -- {definitely lost: 0 bytes} $buf] &&
          ![regexp -- {no leaks are possible} $buf])} {
         send_data_packet $::test_server_fd err "Valgrind error: $buf\n"
@@ -29,7 +36,13 @@ proc kill_server config {
     if {$::external} return
 
     # nevermind if its already dead
-    if {![is_alive $config]} { return }
+    if {![is_alive $config]} {
+        # Check valgrind errors if needed
+        if {$::valgrind} {
+            check_valgrind_errors [dict get $config stderr]
+        }
+        return
+    }
     set pid [dict get $config pid]
 
     # check for leaks
@@ -153,6 +166,55 @@ proc create_server_config_file {filename config} {
     close $fp
 }
 
+proc spawn_server {config_file stdout stderr} {
+    if {$::valgrind} {
+        set pid [exec valgrind --track-origins=yes --trace-children=yes --suppressions=[pwd]/src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file >> $stdout 2>> $stderr &]
+    } elseif ($::stack_logging) {
+        set pid [exec /usr/bin/env MallocStackLogging=1 MallocLogFile=/tmp/malloc_log.txt src/redis-server $config_file >> $stdout 2>> $stderr &]
+    } else {
+        set pid [exec src/redis-server $config_file >> $stdout 2>> $stderr &]
+    }
+
+    if {$::wait_server} {
+        set msg "server started PID: $pid. press any key to continue..."
+        puts $msg
+        read stdin 1
+    }
+
+    # Tell the test server about this new instance.
+    send_data_packet $::test_server_fd server-spawned $pid
+    return $pid
+}
+
+# Wait for actual startup, return 1 if port is busy, 0 otherwise
+proc wait_server_started {config_file stdout pid} {
+    set checkperiod 100; # Milliseconds
+    set maxiter [expr {120*1000/$checkperiod}] ; # Wait up to 2 minutes.
+    set port_busy 0
+    while 1 {
+        if {[regexp -- " PID: $pid" [exec cat $stdout]]} {
+            break
+        }
+        after $checkperiod
+        incr maxiter -1
+        if {$maxiter == 0} {
+            start_server_error $config_file "No PID detected in log $stdout"
+            puts "--- LOG CONTENT ---"
+            puts [exec cat $stdout]
+            puts "-------------------"
+            break
+        }
+
+        # Check if the port is actually busy and the server failed
+        # for this reason.
+        if {[regexp {Could not create server TCP} [exec cat $stdout]]} {
+            set port_busy 1
+            break
+        }
+    }
+    return $port_busy
+}
+
 proc start_server {options {code undefined}} {
     # If we are running against an external server, we just push the
     # host/port pair in the stack the first time
@@ -214,14 +276,14 @@ proc start_server {options {code undefined}} {
     dict set config dir [tmpdir server]
 
     # start every server on a different port
-    set ::port [find_available_port [expr {$::port+1}]]
+    set port [find_available_port $::baseport $::portcount]
     if {$::tls} {
         dict set config "port" 0
-        dict set config "tls-port" $::port
+        dict set config "tls-port" $port
         dict set config "tls-cluster" "yes"
         dict set config "tls-replication" "yes"
     } else {
-        dict set config port $::port
+        dict set config port $port
     }
 
     set unixsocket [file normalize [format "%s/%s" [dict get $config "dir"] "socket"]]
@@ -243,67 +305,34 @@ proc start_server {options {code undefined}} {
     set server_started 0
     while {$server_started == 0} {
         if {$::verbose} {
-            puts -nonewline "=== ($tags) Starting server ${::host}:${::port} "
+            puts -nonewline "=== ($tags) Starting server ${::host}:${port} "
         }
 
-        send_data_packet $::test_server_fd "server-spawning" "port $::port"
+        send_data_packet $::test_server_fd "server-spawning" "port $port"
 
-        if {$::valgrind} {
-            set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &]
-        } elseif ($::stack_logging) {
-            set pid [exec /usr/bin/env MallocStackLogging=1 MallocLogFile=/tmp/malloc_log.txt src/redis-server $config_file > $stdout 2> $stderr &]
-        } else {
-            set pid [exec src/redis-server $config_file > $stdout 2> $stderr &]
-        }
-
-        # Tell the test server about this new instance.
-        send_data_packet $::test_server_fd server-spawned $pid
+        set pid [spawn_server $config_file $stdout $stderr]
 
         # check that the server actually started
-        # ugly but tries to be as fast as possible...
-        if {$::valgrind} {set retrynum 1000} else {set retrynum 100}
-
-        # Wait for actual startup
-        set checkperiod 100; # Milliseconds
-        set maxiter [expr {120*1000/100}] ; # Wait up to 2 minutes.
-        set port_busy 0
-        while {![info exists _pid]} {
-            regexp {PID:\s(\d+)} [exec cat $stdout] _ _pid
-            after $checkperiod
-            incr maxiter -1
-            if {$maxiter == 0} {
-                start_server_error $config_file "No PID detected in log $stdout"
-                puts "--- LOG CONTENT ---"
-                puts [exec cat $stdout]
-                puts "-------------------"
-                break
-            }
-
-            # Check if the port is actually busy and the server failed
-            # for this reason.
-            if {[regexp {Could not create server TCP} [exec cat $stdout]]} {
-                set port_busy 1
-                break
-            }
-        }
+        set port_busy [wait_server_started $config_file $stdout $pid]
 
         # Sometimes we have to try a different port, even if we checked
         # for availability. Other test clients may grab the port before we
         # are able to do it for example.
         if {$port_busy} {
-            puts "Port $::port was already busy, trying another port..."
-            set ::port [find_available_port [expr {$::port+1}]]
+            puts "Port $port was already busy, trying another port..."
+            set port [find_available_port $::baseport $::portcount]
             if {$::tls} {
-                dict set config "tls-port" $::port
+                dict set config "tls-port" $port
             } else {
-                dict set config port $::port
+                dict set config port $port
             }
             create_server_config_file $config_file $config
             continue; # Try again
         }
 
+        if {$::valgrind} {set retrynum 1000} else {set retrynum 100}
         if {$code ne "undefined"} {
-            set serverisup [server_is_up $::host $::port $retrynum]
+            set serverisup [server_is_up $::host $port $retrynum]
         } else {
             set serverisup 1
         }
@@ -324,7 +353,6 @@ proc start_server {options {code undefined}} {
     # setup properties to be able to initialize a client object
     set port_param [expr $::tls ? {"tls-port"} : {"port"}]
     set host $::host
-    set port $::port
     if {[dict exists $config bind]} { set host [dict get $config bind] }
     if {[dict exists $config $port_param]} { set port [dict get $config $port_param] }
 
@@ -346,12 +374,6 @@ proc start_server {options {code undefined}} {
             error_and_quit $config_file $line
         }
 
-        if {$::wait_server} {
-            set msg "server started PID: [dict get $srv "pid"]. press any key to continue..."
-            puts $msg
-            read stdin 1
-        }
-
         while 1 {
             # check that the server actually started and is ready for connections
             if {[exec grep -i "Ready to accept" | wc -l < $stdout] > 0} {
@@ -371,6 +393,9 @@ proc start_server {options {code undefined}} {
         if {[catch { uplevel 1 $code } error]} {
             set backtrace $::errorInfo
 
+            # fetch srv back from the server list, in case it was restarted by restart_server (new PID)
+            set srv [lindex $::servers end]
+
             # Kill the server without checking for leaks
             dict set srv "skipleaks" 1
             kill_server $srv
@@ -388,6 +413,9 @@ proc start_server {options {code undefined}} {
             error $error $backtrace
         }
 
+        # fetch srv back from the server list, in case it was restarted by restart_server (new PID)
+        set srv [lindex $::servers end]
+
         # Don't do the leak check when no tests were run
         if {$num_tests == $::num_tests} {
             dict set srv "skipleaks" 1
@@ -403,3 +431,35 @@ proc start_server {options {code undefined}} {
         set _ $srv
     }
 }
+
+proc restart_server {level wait_ready} {
+    set srv [lindex $::servers end+$level]
+    kill_server $srv
+
+    set stdout [dict get $srv "stdout"]
+    set stderr [dict get $srv "stderr"]
+    set config_file [dict get $srv "config_file"]
+
+    set prev_ready_count [exec grep -i "Ready to accept" | wc -l < $stdout]
+
+    set pid [spawn_server $config_file $stdout $stderr]
+
+    # check that the server actually started
+    wait_server_started $config_file $stdout $pid
+
+    # update the pid in the servers list
+    dict set srv "pid" $pid
+    # re-set $srv in the servers list
+    lset ::servers end+$level $srv
+
+    if {$wait_ready} {
+        while 1 {
+            # check that the server actually started and is ready for connections
+            if {[exec grep -i "Ready to accept" | wc -l < $stdout] > $prev_ready_count + 1} {
+                break
+            }
+            after 10
+        }
+    }
+    reconnect $level
+}

tests/support/util.tcl

@@ -99,11 +99,27 @@ proc wait_for_ofs_sync {r1 r2} {
     }
 }
 
-proc wait_for_log_message {srv_idx pattern last_lines maxtries delay} {
+# count current log lines in server's stdout
+proc count_log_lines {srv_idx} {
+    set _ [exec wc -l < [srv $srv_idx stdout]]
+}
+
+# verify pattern exists in server's sdtout after a certain line number
+proc verify_log_message {srv_idx pattern from_line} {
+    set lines_after [count_log_lines]
+    set lines [expr $lines_after - $from_line]
+    set result [exec tail -$lines < [srv $srv_idx stdout]]
+    if {![string match $pattern $result]} {
+        error "assertion:expected message not found in log file: $pattern"
+    }
+}
+
+# wait for pattern to be found in server's stdout after certain line number
+proc wait_for_log_message {srv_idx pattern from_line maxtries delay} {
     set retry $maxtries
     set stdout [srv $srv_idx stdout]
     while {$retry} {
-        set result [exec tail -$last_lines < $stdout]
+        set result [exec tail +$from_line < $stdout]
         set result [split $result "\n"]
         foreach line $result {
             if {[string match $pattern $line]} {
@@ -114,7 +130,7 @@ proc wait_for_log_message {srv_idx pattern last_lines maxtries delay} {
         after $delay
     }
     if {$retry == 0} {
-        fail "log message of '$pattern' not found"
+        fail "log message of '$pattern' not found in $stdout after line: $from_line"
     }
 }
 
@@ -344,21 +360,26 @@ proc roundFloat f {
     format "%.10g" $f
 }
 
-proc find_available_port start {
-    for {set j $start} {$j < $start+1024} {incr j} {
-        if {[catch {set fd1 [socket 127.0.0.1 $j]}] &&
-            [catch {set fd2 [socket 127.0.0.1 [expr $j+10000]]}]} {
-            return $j
+set ::last_port_attempted 0
+proc find_available_port {start count} {
+    set port [expr $::last_port_attempted + 1]
+    for {set attempts 0} {$attempts < $count} {incr attempts} {
+        if {$port < $start || $port >= $start+$count} {
+            set port $start
+        }
+        if {[catch {set fd1 [socket 127.0.0.1 $port]}] &&
+            [catch {set fd2 [socket 127.0.0.1 [expr $port+10000]]}]} {
+            set ::last_port_attempted $port
+            return $port
         } else {
             catch {
                 close $fd1
                 close $fd2
             }
         }
+        incr port
     }
-    if {$j == $start+1024} {
-        error "Can't find a non busy port in the $start-[expr {$start+1023}] range."
-    }
+    error "Can't find a non busy port in the $start-[expr {$start+$count-1}] range."
 }
 
 # Test if TERM looks like to support colors

tests/test_helper.tcl

@@ -35,6 +35,7 @@ set ::all_tests {
     unit/quit
     unit/aofrw
     unit/acl
+    unit/latency-monitor
     integration/block-repl
     integration/replication
     integration/replication-2
@@ -48,6 +49,7 @@ set ::all_tests {
     integration/psync2
     integration/psync2-reg
     integration/psync2-pingoff
+    integration/redis-cli
     unit/pubsub
     unit/slowlog
     unit/scripting
@@ -65,12 +67,15 @@ set ::all_tests {
     unit/wait
     unit/pendingquerybuf
     unit/tls
+    unit/tracking
 }
 # Index to the next test to run in the ::all_tests list.
 set ::next_test 0
 
 set ::host 127.0.0.1
-set ::port 21111
+set ::port 6379; # port for external server
+set ::baseport 21111; # initial port for spawned redis servers
+set ::portcount 8000; # we don't wanna use more than 10000 to avoid collision with cluster bus ports
 set ::traceleaks 0
 set ::valgrind 0
 set ::tls 0
@@ -193,6 +198,21 @@ proc redis_deferring_client {args} {
     return $client
 }
 
+proc redis_client {args} {
+    set level 0
+    if {[llength $args] > 0 && [string is integer [lindex $args 0]]} {
+        set level [lindex $args 0]
+        set args [lrange $args 1 end]
+    }
+
+    # create client that defers reading reply
+    set client [redis [srv $level "host"] [srv $level "port"] 0 $::tls]
+
+    # select the right db and read the response (OK)
+    $client select 9
+    return $client
+}
+
 # Provide easy access to INFO properties. Same semantic as "proc r".
 proc s {args} {
     set level 0
@@ -228,26 +248,26 @@ proc test_server_main {} {
     set tclsh [info nameofexecutable]
     # Open a listening socket, trying different ports in order to find a
     # non busy one.
-    set port [find_available_port 11111]
+    set clientport [find_available_port 11111 32]
     if {!$::quiet} {
-        puts "Starting test server at port $port"
+        puts "Starting test server at port $clientport"
     }
-    socket -server accept_test_clients  -myaddr 127.0.0.1 $port
+    socket -server accept_test_clients  -myaddr 127.0.0.1 $clientport
 
     # Start the client instances
     set ::clients_pids {}
     if {$::external} {
         set p [exec $tclsh [info script] {*}$::argv \
-            --client $port --port $::port &]
+            --client $clientport &]
         lappend ::clients_pids $p
     } else {
-        set start_port [expr {$::port+100}]
+        set start_port $::baseport
+        set port_count [expr {$::portcount / $::numclients}]
         for {set j 0} {$j < $::numclients} {incr j} {
-            set start_port [find_available_port $start_port]
             set p [exec $tclsh [info script] {*}$::argv \
-                --client $port --port $start_port &]
+                --client $clientport --baseport $start_port --portcount $port_count &]
             lappend ::clients_pids $p
-            incr start_port 10
+            incr start_port $port_count
         }
     }
 
@@ -510,6 +530,10 @@ proc print_help_screen {} {
         "--loop             Execute the specified set of tests forever."
         "--wait-server      Wait after server is started (so that you can attach a debugger)."
         "--tls              Run tests in TLS mode."
+        "--host <addr>      Run tests against an external host."
+        "--port <port>      TCP port to use against external host."
+        "--baseport <port>  Initial port number for spawned redis servers."
+        "--portcount <num>  Port range for spawned redis servers."
         "--help             Print this help screen."
     } "\n"]
 }
@@ -560,6 +584,12 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
     } elseif {$opt eq {--port}} {
         set ::port $arg
         incr j
+    } elseif {$opt eq {--baseport}} {
+        set ::baseport $arg
+        incr j
+    } elseif {$opt eq {--portcount}} {
+        set ::portcount $arg
+        incr j
     } elseif {$opt eq {--accurate}} {
         set ::accurate 1
     } elseif {$opt eq {--force-failure}} {

tests/unit/dump.tcl

@@ -36,7 +36,18 @@ start_server {tags {"dump"}} {
         assert {$ttl >= 2900 && $ttl <= 3100}
         r get foo
     } {bar}
-    
+
+    test {RESTORE with ABSTTL in the past} {
+        r set foo bar
+        set encoded [r dump foo]
+        set now [clock milliseconds]
+        r debug set-active-expire 0
+        r restore foo [expr $now-3000] $encoded absttl REPLACE
+        catch {r debug object foo} e
+        r debug set-active-expire 1
+        set e
+    } {ERR no such key}
+
     test {RESTORE can set LRU} {
         r set foo bar
         set encoded [r dump foo]

tests/unit/expire.tcl

@@ -232,4 +232,14 @@ start_server {tags {"expire"}} {
         set ttl [r ttl foo]
         assert {$ttl <= 100 && $ttl > 90}
     }
+
+    test {SET - use KEEPTTL option, TTL should not be removed after loadaof} {
+        r config set appendonly yes
+        r set foo bar EX 100
+        r set foo bar2 KEEPTTL
+        after 2000
+        r debug loadaof
+        set ttl [r ttl foo]
+        assert {$ttl <= 98 && $ttl > 90}
+    }
 }

tests/unit/introspection.tcl

@@ -78,17 +78,8 @@ start_server {tags {"introspection"}} {
             syslog-facility
             databases
             port
-            io-threads
             tls-port
-            tls-prefer-server-ciphers
-            tls-cert-file
-            tls-key-file
-            tls-dh-params-file
-            tls-ca-cert-file
-            tls-ca-cert-dir
-            tls-protocols
-            tls-ciphers
-            tls-ciphersuites
+            io-threads
             logfile
             unixsocketperm
             slaveof
@@ -100,6 +91,23 @@ start_server {tags {"introspection"}} {
             bgsave_cpulist
         }
 
+        if {!$::tls} {
+            append skip_configs {
+                tls-prefer-server-ciphers
+                tls-session-cache-timeout
+                tls-session-cache-size
+                tls-session-caching
+                tls-cert-file
+                tls-key-file
+                tls-dh-params-file
+                tls-ca-cert-file
+                tls-ca-cert-dir
+                tls-protocols
+                tls-ciphers
+                tls-ciphersuites
+            }
+        }
+
         set configs {}
         foreach {k v} [r config get *] {
             if {[lsearch $skip_configs $k] != -1} {

tests/unit/moduleapi/scan.tcl

@@ -16,6 +16,11 @@ start_server {tags {"modules"}} {
         r hmset hh f1 v1 f2 v2
         lsort [r scan.scan_key hh]
     } {{f1 v1} {f2 v2}}
+    
+    test {Module scan hash dict with int value} {
+        r hmset hh1 f1 1 
+        lsort [r scan.scan_key hh1]
+    } {{f1 1}}
 
     test {Module scan hash dict} {
         r config set hash-max-ziplist-entries 2

tests/unit/moduleapi/testrdb.tcl

@@ -67,6 +67,7 @@ tags "modules" {
                     }
 
                     # Start the replication process...
+                    set loglines [count_log_lines -1]
                     $master config set repl-diskless-sync-delay 0
                     $replica replicaof $master_host $master_port
 
@@ -76,7 +77,7 @@ tags "modules" {
                     for {set i 0} {$i < $attempts} {incr i} {
                         # wait for the replica to start reading the rdb
                         # using the log file since the replica only responds to INFO once in 2mb
-                        wait_for_log_message -1 "*Loading DB in memory*" 5 2000 1
+                        wait_for_log_message -1 "*Loading DB in memory*" $loglines 2000 1
 
                         # add some additional random sleep so that we kill the master on a different place each time
                         after [expr {int(rand()*100)}]
@@ -85,7 +86,7 @@ tags "modules" {
                         set killed [$master client kill type replica]
 
                         if {[catch {
-                            set res [wait_for_log_message -1 "*Internal error in RDB*" 5 100 10]
+                            set res [wait_for_log_message -1 "*Internal error in RDB*" $loglines 100 10]
                             if {$::verbose} {
                                 puts $res
                             }
@@ -98,6 +99,7 @@ tags "modules" {
                             $master config set repl-backlog-size [expr {16384 + $i}]
                         }
                         # wait for loading to stop (fail)
+                        set loglines [count_log_lines -1]
                         wait_for_condition 100 10 {
                             [s -1 loading] eq 0
                         } else {

tests/unit/multi.tcl

@@ -325,71 +325,145 @@ start_server {tags {"multi"}} {
         # check that if MULTI arrives during timeout, it is either refused, or
         # allowed to pass, and we don't end up executing half of the transaction
         set rd1 [redis_deferring_client]
-        set rd2 [redis_deferring_client]
+        set r2 [redis_client]
         r config set lua-time-limit 10
         r set xx 1
         $rd1 eval {while true do end} 0
         after 200
-        catch { $rd2 multi; $rd2 read } e
-        catch { $rd2 incr xx; $rd2 read } e
+        catch { $r2 multi; } e
+        catch { $r2 incr xx; } e
         r script kill
         after 200 ; # Give some time to Lua to call the hook again...
-        catch { $rd2 incr xx; $rd2 read } e
-        catch { $rd2 exec; $rd2 read } e
+        catch { $r2 incr xx; } e
+        catch { $r2 exec; } e
+        assert_match {EXECABORT*previous errors*} $e
         set xx [r get xx]
         # make sure that either the whole transcation passed or none of it (we actually expect none)
         assert { $xx == 1 || $xx == 3}
         # check that the connection is no longer in multi state
-        $rd2 ping asdf
-        set pong [$rd2 read]
+        set pong [$r2 ping asdf]
         assert_equal $pong "asdf"
+        $rd1 close; $r2 close
     }
 
     test {EXEC and script timeout} {
         # check that if EXEC arrives during timeout, we don't end up executing
         # half of the transaction, and also that we exit the multi state
         set rd1 [redis_deferring_client]
-        set rd2 [redis_deferring_client]
+        set r2 [redis_client]
         r config set lua-time-limit 10
         r set xx 1
-        catch { $rd2 multi; $rd2 read } e
-        catch { $rd2 incr xx; $rd2 read } e
+        catch { $r2 multi; } e
+        catch { $r2 incr xx; } e
         $rd1 eval {while true do end} 0
         after 200
-        catch { $rd2 incr xx; $rd2 read } e
-        catch { $rd2 exec; $rd2 read } e
+        catch { $r2 incr xx; } e
+        catch { $r2 exec; } e
+        assert_match {EXECABORT*BUSY*} $e
         r script kill
         after 200 ; # Give some time to Lua to call the hook again...
         set xx [r get xx]
         # make sure that either the whole transcation passed or none of it (we actually expect none)
         assert { $xx == 1 || $xx == 3}
         # check that the connection is no longer in multi state
-        $rd2 ping asdf
-        set pong [$rd2 read]
+        set pong [$r2 ping asdf]
         assert_equal $pong "asdf"
+        $rd1 close; $r2 close
     }
 
     test {MULTI-EXEC body and script timeout} {
         # check that we don't run an imcomplete transaction due to some commands
         # arriving during busy script
         set rd1 [redis_deferring_client]
-        set rd2 [redis_deferring_client]
+        set r2 [redis_client]
         r config set lua-time-limit 10
         r set xx 1
-        catch { $rd2 multi; $rd2 read } e
-        catch { $rd2 incr xx; $rd2 read } e
+        catch { $r2 multi; } e
+        catch { $r2 incr xx; } e
         $rd1 eval {while true do end} 0
         after 200
-        catch { $rd2 incr xx; $rd2 read } e
+        catch { $r2 incr xx; } e
         r script kill
         after 200 ; # Give some time to Lua to call the hook again...
-        catch { $rd2 exec; $rd2 read } e
+        catch { $r2 exec; } e
+        assert_match {EXECABORT*previous errors*} $e
         set xx [r get xx]
         # make sure that either the whole transcation passed or none of it (we actually expect none)
         assert { $xx == 1 || $xx == 3}
         # check that the connection is no longer in multi state
-        $rd2 ping asdf
-        set pong [$rd2 read]
+        set pong [$r2 ping asdf]
         assert_equal $pong "asdf"
+        $rd1 close; $r2 close
+    }
+
+    test {just EXEC and script timeout} {
+        # check that if EXEC arrives during timeout, we don't end up executing
+        # actual commands during busy script, and also that we exit the multi state
+        set rd1 [redis_deferring_client]
+        set r2 [redis_client]
+        r config set lua-time-limit 10
+        r set xx 1
+        catch { $r2 multi; } e
+        catch { $r2 incr xx; } e
+        $rd1 eval {while true do end} 0
+        after 200
+        catch { $r2 exec; } e
+        assert_match {EXECABORT*BUSY*} $e
+        r script kill
+        after 200 ; # Give some time to Lua to call the hook again...
+        set xx [r get xx]
+        # make we didn't execute the transaction
+        assert { $xx == 1}
+        # check that the connection is no longer in multi state
+        set pong [$r2 ping asdf]
+        assert_equal $pong "asdf"
+        $rd1 close; $r2 close
+    }
+
+    test {exec with write commands and state change} {
+        # check that exec that contains write commands fails if server state changed since they were queued
+        set r1 [redis_client]
+        r set xx 1
+        r multi
+        r incr xx
+        $r1 config set min-replicas-to-write 2
+        catch {r exec} e
+        assert_match {*EXECABORT*NOREPLICAS*} $e
+        set xx [r get xx]
+        # make sure that the INCR wasn't executed
+        assert { $xx == 1}
+        $r1 config set min-replicas-to-write 0
+        $r1 close;
+    }
+
+    test {exec with read commands and stale replica state change} {
+        # check that exec that contains read commands fails if server state changed since they were queued
+        r config set replica-serve-stale-data no
+        set r1 [redis_client]
+        r set xx 1
+
+        # check that GET is disallowed on stale replica, even if the replica becomes stale only after queuing.
+        r multi
+        r get xx
+        $r1 replicaof localhsot 0
+        catch {r exec} e
+        assert_match {*EXECABORT*MASTERDOWN*} $e
+
+        # check that PING is allowed
+        r multi
+        r ping
+        $r1 replicaof localhsot 0
+        set pong [r exec]
+        assert {$pong == "PONG"}
+
+        # check that when replica is not stale, GET is allowed
+        # while we're at it, let's check that multi is allowed on stale replica too
+        r multi
+        $r1 replicaof no one
+        r get xx
+        set xx [r exec]
+        # make sure that the INCR was executed
+        assert { $xx == 1 }
+        $r1 close;
     }
 }

tests/unit/other.tcl

@@ -167,9 +167,9 @@ start_server {tags {"other"}} {
     tags {protocol} {
         test {PIPELINING stresser (also a regression for the old epoll bug)} {
             if {$::tls} {
-                set fd2 [::tls::socket $::host $::port]
+                set fd2 [::tls::socket [srv host] [srv port]]
             } else {
-                set fd2 [socket $::host $::port]
+                set fd2 [socket [srv host] [srv port]]
             }
             fconfigure $fd2 -encoding binary -translation binary
             puts -nonewline $fd2 "SELECT 9\r\n"

tests/unit/type/incr.tcl

@@ -130,15 +130,18 @@ start_server {tags {"incr"}} {
         format $err
     } {WRONGTYPE*}
 
-    test {INCRBYFLOAT does not allow NaN or Infinity} {
-        r set foo 0
-        set err {}
-        catch {r incrbyfloat foo +inf} err
-        set err
-        # p.s. no way I can force NaN to test it from the API because
-        # there is no way to increment / decrement by infinity nor to
-        # perform divisions.
-    } {ERR*would produce*}
+    # On some platforms strtold("+inf") with valgrind returns a non-inf result
+    if {!$::valgrind} {
+        test {INCRBYFLOAT does not allow NaN or Infinity} {
+            r set foo 0
+            set err {}
+            catch {r incrbyfloat foo +inf} err
+            set err
+            # p.s. no way I can force NaN to test it from the API because
+            # there is no way to increment / decrement by infinity nor to
+            # perform divisions.
+        } {ERR*would produce*}
+    }
 
     test {INCRBYFLOAT decrement} {
         r set foo 1

tests/unit/type/list.tcl

@@ -6,6 +6,50 @@ start_server {
 } {
     source "tests/unit/type/list-common.tcl"
 
+    test {LPOS basic usage} {
+        r DEL mylist
+        r RPUSH mylist a b c 1 2 3 c c
+        assert {[r LPOS mylist a] == 0}
+        assert {[r LPOS mylist c] == 2}
+    }
+
+    test {LPOS RANK (positive and negative rank) option} {
+        assert {[r LPOS mylist c RANK 1] == 2}
+        assert {[r LPOS mylist c RANK 2] == 6}
+        assert {[r LPOS mylist c RANK 4] eq ""}
+        assert {[r LPOS mylist c RANK -1] == 7}
+        assert {[r LPOS mylist c RANK -2] == 6}
+    }
+
+    test {LPOS COUNT option} {
+        assert {[r LPOS mylist c COUNT 0] == {2 6 7}}
+        assert {[r LPOS mylist c COUNT 1] == {2}}
+        assert {[r LPOS mylist c COUNT 2] == {2 6}}
+        assert {[r LPOS mylist c COUNT 100] == {2 6 7}}
+    }
+
+    test {LPOS COUNT + RANK option} {
+        assert {[r LPOS mylist c COUNT 0 RANK 2] == {6 7}}
+        assert {[r LPOS mylist c COUNT 2 RANK -1] == {7 6}}
+    }
+
+    test {LPOS non existing key} {
+        assert {[r LPOS mylistxxx c COUNT 0 RANK 2] eq {}}
+    }
+
+    test {LPOS no match} {
+        assert {[r LPOS mylist x COUNT 2 RANK -1] eq {}}
+        assert {[r LPOS mylist x RANK -1] eq {}}
+    }
+
+    test {LPOS MAXLEN} {
+        assert {[r LPOS mylist a COUNT 0 MAXLEN 1] == {0}}
+        assert {[r LPOS mylist c COUNT 0 MAXLEN 1] == {}}
+        assert {[r LPOS mylist c COUNT 0 MAXLEN 3] == {2}}
+        assert {[r LPOS mylist c COUNT 0 MAXLEN 3 RANK -1] == {7 6}}
+        assert {[r LPOS mylist c COUNT 0 MAXLEN 7 RANK 2] == {6}}
+    }
+
     test {LPUSH, RPUSH, LLENGTH, LINDEX, LPOP - ziplist} {
         # first lpush then rpush
         assert_equal 1 [r lpush myziplist1 aa]

utils/generate-command-help.rb

@@ -53,7 +53,7 @@ def commands
   require "json"
   require "uri"
 
-  url = URI.parse "https://raw.githubusercontent.com/antirez/redis-doc/master/commands.json"
+  url = URI.parse "https://raw.githubusercontent.com/redis/redis-doc/master/commands.json"
   client = Net::HTTP.new url.host, url.port
   client.use_ssl = true
   response = client.get url.path

utils/releasetools/changelog.tcl

@@ -30,6 +30,6 @@ append template [exec git log $branch~$count..$branch "--format=format:%an in co
 
 #Older, more verbose version.
 #
-#append template [exec git log $branch~30..$branch "--format=format:+-------------------------------------------------------------------------------%n| %s%n| By %an, %ai%n+--------------------------------------------------------------------------------%nhttps://github.com/antirez/redis/commit/%H%n%n%b" --stat]
+#append template [exec git log $branch~30..$branch "--format=format:+-------------------------------------------------------------------------------%n| %s%n| By %an, %ai%n+--------------------------------------------------------------------------------%nhttps://github.com/redis/redis/commit/%H%n%n%b" --stat]
 
 puts $template