added a few more files to gitignore

.gitignore

@@ -16,3 +16,5 @@ src/release.h
 appendonly.aof
 SHORT_TERM_TODO
 redis.conf.*
+release.h
+src/transfer.sh

Makefile

@@ -8,9 +8,12 @@ all:
 install: dummy
 	cd src && $(MAKE) $@
 
-$(TARGETS) clean:
+clean:
 	cd src && $(MAKE) $@
 	cd deps/hiredis && $(MAKE) $@
 	cd deps/linenoise && $(MAKE) $@
 
+$(TARGETS):
+	cd src && $(MAKE) $@
+
 dummy:

deps/hiredis/hiredis.c

@@ -164,8 +164,17 @@ static char *readBytes(redisReader *r, unsigned int bytes) {
 
 static char *seekNewline(char *s) {
     /* Find pointer to \r\n without strstr */
-    while(s != NULL && s[0] != '\r' && s[1] != '\n')
+    while (s != NULL) {
         s = strchr(s,'\r');
+        if (s != NULL) {
+            if (s[1] == '\n')
+                break;
+            else
+                s++;
+        } else {
+            break;
+        }
+    }
     return s;
 }
 
@@ -424,7 +433,7 @@ char *redisReplyReaderGetError(void *reader) {
     return r->error;
 }
 
-void redisReplyReaderFeed(void *reader, char *buf, int len) {
+void redisReplyReaderFeed(void *reader, char *buf, size_t len) {
     redisReader *r = reader;
 
     /* Copy the provided buffer. */

deps/hiredis/hiredis.h

@@ -115,7 +115,7 @@ int redisReplyReaderSetReplyObjectFunctions(void *reader, redisReplyObjectFuncti
 void *redisReplyReaderGetObject(void *reader);
 char *redisReplyReaderGetError(void *reader);
 void redisReplyReaderFree(void *ptr);
-void redisReplyReaderFeed(void *reader, char *buf, int len);
+void redisReplyReaderFeed(void *reader, char *buf, size_t len);
 int redisReplyReaderGetReply(void *reader, void **reply);
 
 /* Functions to format a command according to the protocol. */

deps/hiredis/test.c

@@ -5,6 +5,7 @@
 #include <sys/time.h>
 #include <assert.h>
 #include <unistd.h>
+#include <signal.h>
 
 #include "hiredis.h"
 
@@ -219,6 +220,17 @@ static void test_reply_reader() {
     ret = redisReplyReaderGetReply(reader,&reply);
     test_cond(ret == REDIS_OK && reply == (void*)REDIS_REPLY_STATUS);
     redisReplyReaderFree(reader);
+
+    test("Works when a single newline (\\r\\n) covers two calls to feed: ");
+    reader = redisReplyReaderCreate();
+    redisReplyReaderSetReplyObjectFunctions(reader,NULL);
+    redisReplyReaderFeed(reader,(char*)"+OK\r",4);
+    ret = redisReplyReaderGetReply(reader,&reply);
+    assert(ret == REDIS_OK && reply == NULL);
+    redisReplyReaderFeed(reader,(char*)"\n",1);
+    ret = redisReplyReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_OK && reply == (void*)REDIS_REPLY_STATUS);
+    redisReplyReaderFree(reader);
 }
 
 static void test_throughput() {

deps/linenoise/Makefile

@@ -1,10 +1,10 @@
 linenoise_example: linenoise.h linenoise.c
 
 linenoise_example: linenoise.o example.o
-	$(CC) -Wall -W -Os -g -o linenoise_example linenoise.o example.o
+	$(CC) $(ARCH) -Wall -W -Os -g -o linenoise_example linenoise.o example.o
 
 .c.o:
-	$(CC) -c -Wall -W -Os -g $<
+	$(CC) $(ARCH) -c -Wall -W -Os -g $<
 
 clean:
-	rm -f linenoise_example
+	rm -f linenoise_example *.o

redis.conf

@@ -110,6 +110,19 @@ dir ./
 #
 # masterauth <master-password>
 
+# When a slave lost the connection with the master, or when the replication
+# is still in progress, the slave can act in two different ways:
+#
+# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will
+#    still reply to client requests, possibly with out of data data, or the
+#    data set may just be empty if this is the first synchronization.
+#
+# 2) if slave-serve-stale data is set to 'no' the slave will reply with
+#    an error "SYNC with master in progress" to all the kind of commands
+#    but to INFO and SLAVEOF.
+#
+slave-serve-stale-data yes
+
 ################################## SECURITY ###################################
 
 # Require clients to issue AUTH <PASSWORD> before processing any other
@@ -178,6 +191,18 @@ dir ./
 # volatile-random -> remove a random key with an expire set
 # allkeys->random -> remove a random key, any key
 # volatile-ttl -> remove the key with the nearest expire time (minor TTL)
+# noeviction -> don't expire at all, just return an error on write operations
+# 
+# Note: with all the kind of policies, Redis will return an error on write
+#       operations, when there are not suitable keys for eviction.
+#
+#       At the date of writing this commands are: set setnx setex append
+#       incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd
+#       sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby
+#       zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby
+#       getset mset msetnx exec sort
+#
+# The default is:
 #
 # maxmemory-policy volatile-lru
 

src/Makefile

@@ -26,7 +26,7 @@ INSTALL= cp -p
 
 OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o vm.o pubsub.o multi.o debug.o sort.o intset.o syncio.o
 BENCHOBJ = ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o
-CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o
+CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o
 CHECKDUMPOBJ = redis-check-dump.o lzf_c.o lzf_d.o
 CHECKAOFOBJ = redis-check-aof.o
 
@@ -36,7 +36,7 @@ CLIPRGNAME = redis-cli
 CHECKDUMPPRGNAME = redis-check-dump
 CHECKAOFPRGNAME = redis-check-aof
 
-all: redis-server redis-benchmark redis-cli redis-check-dump redis-check-aof
+all: redis-benchmark redis-cli redis-check-dump redis-check-aof redis-server
 
 # Deps (use make dep to generate this)
 adlist.o: adlist.c adlist.h zmalloc.h
@@ -116,8 +116,8 @@ redis-benchmark.o:
 	$(CC) -c $(CFLAGS) -I../deps/hiredis $(DEBUG) $(COMPILE_TIME) $<
 
 redis-cli: $(CLIOBJ)
-	cd ../deps/hiredis && make static
-	cd ../deps/linenoise && make
+	cd ../deps/hiredis && make static ARCH="$(ARCH)"
+	cd ../deps/linenoise && make ARCH="$(ARCH)"
 	$(CC) -o $(CLIPRGNAME) $(CCOPT) $(DEBUG) $(CLIOBJ) ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o
 
 redis-cli.o:
@@ -134,6 +134,8 @@ redis-check-aof: $(CHECKAOFOBJ)
 
 clean:
 	rm -rf $(PRGNAME) $(BENCHPRGNAME) $(CLIPRGNAME) $(CHECKDUMPPRGNAME) $(CHECKAOFPRGNAME) *.o *.gcda *.gcno *.gcov
+	cd ../deps/hiredis && make clean
+	cd ../deps/linenoise && make clean
 
 dep:
 	$(CC) -MM *.c

src/aof.c

@@ -218,6 +218,7 @@ int loadAppendOnlyFile(char *filename) {
     FILE *fp = fopen(filename,"r");
     struct redis_stat sb;
     int appendonly = server.appendonly;
+    long loops = 0;
 
     if (redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0)
         return REDIS_ERR;
@@ -232,6 +233,8 @@ int loadAppendOnlyFile(char *filename) {
     server.appendonly = 0;
 
     fakeClient = createFakeClient();
+    startLoading(fp);
+
     while(1) {
         int argc, j;
         unsigned long len;
@@ -241,6 +244,12 @@ int loadAppendOnlyFile(char *filename) {
         struct redisCommand *cmd;
         int force_swapout;
 
+        /* Serve the clients from time to time */
+        if (!(loops++ % 1000)) {
+            loadingProgress(ftello(fp));
+            aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT);
+        }
+
         if (fgets(buf,sizeof(buf),fp) == NULL) {
             if (feof(fp))
                 break;
@@ -297,6 +306,7 @@ int loadAppendOnlyFile(char *filename) {
     fclose(fp);
     freeFakeClient(fakeClient);
     server.appendonly = appendonly;
+    stopLoading();
     return REDIS_OK;
 
 readerr:

src/config.c

@@ -136,6 +136,8 @@ void loadServerConfig(char *filename) {
                 server.maxmemory_policy = REDIS_MAXMEMORY_ALLKEYS_LRU;
             } else if (!strcasecmp(argv[1],"allkeys-random")) {
                 server.maxmemory_policy = REDIS_MAXMEMORY_ALLKEYS_RANDOM;
+            } else if (!strcasecmp(argv[1],"noeviction")) {
+                server.maxmemory_policy = REDIS_MAXMEMORY_NO_EVICTION;
             } else {
                 err = "Invalid maxmemory policy";
                 goto loaderr;
@@ -152,6 +154,10 @@ void loadServerConfig(char *filename) {
             server.replstate = REDIS_REPL_CONNECT;
         } else if (!strcasecmp(argv[0],"masterauth") && argc == 2) {
         	server.masterauth = zstrdup(argv[1]);
+        } else if (!strcasecmp(argv[0],"slave-serve-stale-data") && argc == 2) {
+            if ((server.repl_serve_stale_data = yesnotoi(argv[1])) == -1) {
+                err = "argument must be 'yes' or 'no'"; goto loaderr;
+            }
         } else if (!strcasecmp(argv[0],"glueoutputbuf") && argc == 2) {
             if ((server.glueoutputbuf = yesnotoi(argv[1])) == -1) {
                 err = "argument must be 'yes' or 'no'"; goto loaderr;
@@ -303,6 +309,8 @@ void configSetCommand(redisClient *c) {
             server.maxmemory_policy = REDIS_MAXMEMORY_ALLKEYS_LRU;
         } else if (!strcasecmp(o->ptr,"allkeys-random")) {
             server.maxmemory_policy = REDIS_MAXMEMORY_ALLKEYS_RANDOM;
+        } else if (!strcasecmp(o->ptr,"noeviction")) {
+            server.maxmemory_policy = REDIS_MAXMEMORY_NO_EVICTION;
         } else {
             goto badfmt;
         }
@@ -379,6 +387,11 @@ void configSetCommand(redisClient *c) {
             appendServerSaveParams(seconds, changes);
         }
         sdsfreesplitres(v,vlen);
+    } else if (!strcasecmp(c->argv[2]->ptr,"slave-serve-stale-data")) {
+        int yn = yesnotoi(o->ptr);
+
+        if (yn == -1) goto badfmt;
+        server.repl_serve_stale_data = yn;
     } else {
         addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s",
             (char*)c->argv[2]->ptr);
@@ -431,6 +444,7 @@ void configGetCommand(redisClient *c) {
         case REDIS_MAXMEMORY_VOLATILE_RANDOM: s = "volatile-random"; break;
         case REDIS_MAXMEMORY_ALLKEYS_LRU: s = "allkeys-lru"; break;
         case REDIS_MAXMEMORY_ALLKEYS_RANDOM: s = "allkeys-random"; break;
+        case REDIS_MAXMEMORY_NO_EVICTION: s = "noeviction"; break;
         default: s = "unknown"; break; /* too harmless to panic */
         }
         addReplyBulkCString(c,"maxmemory-policy");
@@ -488,6 +502,11 @@ void configGetCommand(redisClient *c) {
         sdsfree(buf);
         matches++;
     }
+    if (stringmatch(pattern,"slave-serve-stale-data",0)) {
+        addReplyBulkCString(c,"slave-serve-stale-data");
+        addReplyBulkCString(c,server.repl_serve_stale_data ? "yes" : "no");
+        matches++;
+    }
     setDeferredMultiBulkLength(c,replylen,matches*2);
 }
 

src/networking.c

@@ -467,6 +467,7 @@ void freeClient(redisClient *c) {
     /* Case 2: we lost the connection with the master. */
     if (c->flags & REDIS_MASTER) {
         server.master = NULL;
+        /* FIXME */
         server.replstate = REDIS_REPL_CONNECT;
         /* Since we lost the connection with the master, we should also
          * close the connection with all our slaves if we have any, so

src/rdb.c

@@ -7,6 +7,7 @@
 #include <sys/resource.h>
 #include <sys/wait.h>
 #include <arpa/inet.h>
+#include <sys/stat.h>
 
 int rdbSaveType(FILE *fp, unsigned char type) {
     if (fwrite(&type,1,1,fp) == 0) return -1;
@@ -793,6 +794,31 @@ robj *rdbLoadObject(int type, FILE *fp) {
     return o;
 }
 
+/* Mark that we are loading in the global state and setup the fields
+ * needed to provide loading stats. */
+void startLoading(FILE *fp) {
+    struct stat sb;
+
+    /* Load the DB */
+    server.loading = 1;
+    server.loading_start_time = time(NULL);
+    if (fstat(fileno(fp), &sb) == -1) {
+        server.loading_total_bytes = 1; /* just to avoid division by zero */
+    } else {
+        server.loading_total_bytes = sb.st_size;
+    }
+}
+
+/* Refresh the loading progress info */
+void loadingProgress(off_t pos) {
+    server.loading_loaded_bytes = pos;
+}
+
+/* Loading finished */
+void stopLoading(void) {
+    server.loading = 0;
+}
+
 int rdbLoad(char *filename) {
     FILE *fp;
     uint32_t dbid;
@@ -801,6 +827,7 @@ int rdbLoad(char *filename) {
     redisDb *db = server.db+0;
     char buf[1024];
     time_t expiretime, now = time(NULL);
+    long loops = 0;
 
     fp = fopen(filename,"r");
     if (!fp) return REDIS_ERR;
@@ -817,11 +844,20 @@ int rdbLoad(char *filename) {
         redisLog(REDIS_WARNING,"Can't handle RDB format version %d",rdbver);
         return REDIS_ERR;
     }
+
+    startLoading(fp);
     while(1) {
         robj *key, *val;
         int force_swapout;
 
         expiretime = -1;
+
+        /* Serve the clients from time to time */
+        if (!(loops++ % 1000)) {
+            loadingProgress(ftello(fp));
+            aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT);
+        }
+
         /* Read type. */
         if ((type = rdbLoadType(fp)) == -1) goto eoferr;
         if (type == REDIS_EXPIRETIME) {
@@ -900,6 +936,7 @@ int rdbLoad(char *filename) {
         }
     }
     fclose(fp);
+    stopLoading();
     return REDIS_OK;
 
 eoferr: /* unexpected end of file is handled here with a fatal exit */

src/redis-cli.c

@@ -67,6 +67,7 @@ static struct config {
 } config;
 
 static void usage();
+char *redisGitSHA1(void);
 
 /*------------------------------------------------------------------------------
  * Utility functions
@@ -291,6 +292,7 @@ static int cliSendCommand(int argc, char **argv, int repeat) {
         redisAppendCommandArgv(context,argc,(const char**)argv,argvlen);
         while (config.monitor_mode) {
             if (cliReadReply() != REDIS_OK) exit(1);
+            fflush(stdout);
         }
 
         if (config.pubsub_mode) {
@@ -350,7 +352,7 @@ static int parseOptions(int argc, char **argv) {
 "automatically used as last argument.\n"
             );
         } else if (!strcmp(argv[i],"-v")) {
-            printf("redis-cli shipped with Redis version %s\n", REDIS_VERSION);
+            printf("redis-cli shipped with Redis version %s (%s)\n", REDIS_VERSION, redisGitSHA1());
             exit(0);
         } else {
             break;

src/redis.c

@@ -124,7 +124,7 @@ struct redisCommand readonlyCommandTable[] = {
     {"zcount",zcountCommand,4,0,NULL,1,1,1},
     {"zrevrange",zrevrangeCommand,-4,0,NULL,1,1,1},
     {"zcard",zcardCommand,2,0,NULL,1,1,1},
-    {"zscore",zscoreCommand,3,REDIS_CMD_DENYOOM,NULL,1,1,1},
+    {"zscore",zscoreCommand,3,0,NULL,1,1,1},
     {"zrank",zrankCommand,3,0,NULL,1,1,1},
     {"zrevrank",zrevrankCommand,3,0,NULL,1,1,1},
     {"hset",hsetCommand,4,REDIS_CMD_DENYOOM,NULL,1,1,1},
@@ -633,14 +633,10 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
         }
     }
 
-    /* Check if we should connect to a MASTER */
-    if (server.replstate == REDIS_REPL_CONNECT && !(loops % 10)) {
-        redisLog(REDIS_NOTICE,"Connecting to MASTER...");
-        if (syncWithMaster() == REDIS_OK) {
-            redisLog(REDIS_NOTICE,"MASTER <-> SLAVE sync succeeded");
-            if (server.appendonly) rewriteAppendOnlyFileBackground();
-        }
-    }
+    /* Replication cron function -- used to reconnect to master and
+     * to detect transfer failures. */
+    if (!(loops % 10)) replicationCron();
+
     return 100;
 }
 
@@ -706,6 +702,8 @@ void createSharedObjects(void) {
         "-ERR source and destination objects are the same\r\n"));
     shared.outofrangeerr = createObject(REDIS_STRING,sdsnew(
         "-ERR index out of range\r\n"));
+    shared.loadingerr = createObject(REDIS_STRING,sdsnew(
+        "-LOADING Redis is loading the dataset in memory\r\n"));
     shared.space = createObject(REDIS_STRING,sdsnew(" "));
     shared.colon = createObject(REDIS_STRING,sdsnew(":"));
     shared.plus = createObject(REDIS_STRING,sdsnew("+"));
@@ -743,6 +741,7 @@ void initServerConfig() {
     server.verbosity = REDIS_VERBOSE;
     server.maxidletime = REDIS_MAXIDLETIME;
     server.saveparams = NULL;
+    server.loading = 0;
     server.logfile = NULL; /* NULL = log on standard output */
     server.glueoutputbuf = 1;
     server.daemonize = 0;
@@ -790,6 +789,7 @@ void initServerConfig() {
     server.masterport = 6379;
     server.master = NULL;
     server.replstate = REDIS_REPL_NONE;
+    server.repl_serve_stale_data = 1;
 
     /* Double constants initialization */
     R_Zero = 0.0;
@@ -998,6 +998,23 @@ int processCommand(redisClient *c) {
         return REDIS_OK;
     }
 
+    /* Only allow INFO and SLAVEOF when slave-serve-stale-data is no and
+     * we are a slave with a broken link with master. */
+    if (server.masterhost && server.replstate != REDIS_REPL_CONNECTED &&
+        server.repl_serve_stale_data == 0 &&
+        cmd->proc != infoCommand && cmd->proc != slaveofCommand)
+    {
+        addReplyError(c,
+            "link with MASTER is down and slave-serve-stale-data is set to no");
+        return REDIS_OK;
+    }
+
+    /* Loading DB? Return an error if the command is not INFO */
+    if (server.loading && cmd->proc != infoCommand) {
+        addReply(c, shared.loadingerr);
+        return REDIS_OK;
+    }
+
     /* Exec the command */
     if (c->flags & REDIS_MULTI &&
         cmd->proc != execCommand && cmd->proc != discardCommand &&
@@ -1125,6 +1142,8 @@ sds genRedisInfoString(void) {
         "used_memory_rss:%zu\r\n"
         "mem_fragmentation_ratio:%.2f\r\n"
         "use_tcmalloc:%d\r\n"
+        "loading:%d\r\n"
+        "aof_enabled:%d\r\n"
         "changes_since_last_save:%lld\r\n"
         "bgsave_in_progress:%d\r\n"
         "last_save_time:%ld\r\n"
@@ -1165,6 +1184,8 @@ sds genRedisInfoString(void) {
 #else
         0,
 #endif
+        server.loading,
+        server.appendonly,
         server.dirty,
         server.bgsavechildpid != -1,
         server.lastsave,
@@ -1187,12 +1208,23 @@ sds genRedisInfoString(void) {
             "master_port:%d\r\n"
             "master_link_status:%s\r\n"
             "master_last_io_seconds_ago:%d\r\n"
+            "master_sync_in_progress:%d\r\n"
             ,server.masterhost,
             server.masterport,
             (server.replstate == REDIS_REPL_CONNECTED) ?
                 "up" : "down",
-            server.master ? ((int)(time(NULL)-server.master->lastinteraction)) : -1
+            server.master ? ((int)(time(NULL)-server.master->lastinteraction)) : -1,
+            server.replstate == REDIS_REPL_TRANSFER
         );
+
+        if (server.replstate == REDIS_REPL_TRANSFER) {
+            info = sdscatprintf(info,
+                "master_sync_left_bytes:%ld\r\n"
+                "master_sync_last_io_seconds_ago:%d\r\n"
+                ,(long)server.repl_transfer_left,
+                (int)(time(NULL)-server.repl_transfer_lastio)
+            );
+        }
     }
     if (server.vm_enabled) {
         lockThreadedIO();
@@ -1224,6 +1256,35 @@ sds genRedisInfoString(void) {
         );
         unlockThreadedIO();
     }
+    if (server.loading) {
+        double perc;
+        time_t eta, elapsed;
+        off_t remaining_bytes = server.loading_total_bytes-
+                                server.loading_loaded_bytes;
+
+        perc = ((double)server.loading_loaded_bytes /
+               server.loading_total_bytes) * 100;
+
+        elapsed = time(NULL)-server.loading_start_time;
+        if (elapsed == 0) {
+            eta = 1; /* A fake 1 second figure if we don't have enough info */
+        } else {
+            eta = (elapsed*remaining_bytes)/server.loading_loaded_bytes;
+        }
+
+        info = sdscatprintf(info,
+            "loading_start_time:%ld\r\n"
+            "loading_total_bytes:%llu\r\n"
+            "loading_loaded_bytes:%llu\r\n"
+            "loading_loaded_perc:%.2f\r\n"
+            "loading_eta_seconds:%ld\r\n"
+            ,(unsigned long) server.loading_start_time,
+            (unsigned long long) server.loading_total_bytes,
+            (unsigned long long) server.loading_loaded_bytes,
+            perc,
+            eta
+        );
+    }
     for (j = 0; j < server.dbnum; j++) {
         long long keys, vkeys;
 
@@ -1271,6 +1332,8 @@ void monitorCommand(redisClient *c) {
 void freeMemoryIfNeeded(void) {
     /* Remove keys accordingly to the active policy as long as we are
      * over the memory limit. */
+    if (server.maxmemory_policy == REDIS_MAXMEMORY_NO_EVICTION) return;
+
     while (server.maxmemory && zmalloc_used_memory() > server.maxmemory) {
         int j, k, freed = 0;
 
@@ -1309,6 +1372,10 @@ void freeMemoryIfNeeded(void) {
 
                     de = dictGetRandomKey(dict);
                     thiskey = dictGetEntryKey(de);
+                    /* When policy is volatile-lru we need an additonal lookup
+                     * to locate the real key, as dict is set to db->expires. */
+                    if (server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_LRU)
+                        de = dictFind(db->dict, thiskey);
                     o = dictGetEntryVal(de);
                     thisval = estimateObjectIdleTime(o);
 

src/redis.h

@@ -152,7 +152,8 @@
 /* Slave replication state - slave side */
 #define REDIS_REPL_NONE 0   /* No active replication */
 #define REDIS_REPL_CONNECT 1    /* Must connect to master */
-#define REDIS_REPL_CONNECTED 2  /* Connected to master */
+#define REDIS_REPL_TRANSFER 2    /* Receiving .rdb from master */
+#define REDIS_REPL_CONNECTED 3  /* Connected to master */
 
 /* Slave replication state - from the point of view of master
  * Note that in SEND_BULK and ONLINE state the slave receives new updates
@@ -208,6 +209,7 @@
 #define REDIS_MAXMEMORY_VOLATILE_RANDOM 2
 #define REDIS_MAXMEMORY_ALLKEYS_LRU 3
 #define REDIS_MAXMEMORY_ALLKEYS_RANDOM 4
+#define REDIS_MAXMEMORY_NO_EVICTION 5
 
 /* We can print the stacktrace, so our assert is defined this way: */
 #define redisAssert(_e) ((_e)?(void)0 : (_redisAssert(#_e,__FILE__,__LINE__),_exit(1)))
@@ -339,7 +341,7 @@ struct sharedObjectsStruct {
     robj *crlf, *ok, *err, *emptybulk, *czero, *cone, *cnegone, *pong, *space,
     *colon, *nullbulk, *nullmultibulk, *queued,
     *emptymultibulk, *wrongtypeerr, *nokeyerr, *syntaxerr, *sameobjecterr,
-    *outofrangeerr, *plus,
+    *outofrangeerr, *loadingerr, *plus,
     *select0, *select1, *select2, *select3, *select4,
     *select5, *select6, *select7, *select8, *select9,
     *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, *mbulk3,
@@ -360,7 +362,13 @@ struct redisServer {
     long long dirty_before_bgsave; /* used to restore dirty on failed BGSAVE */
     list *clients;
     dict *commands;             /* Command table hahs table */
-    struct redisCommand *delCommand, *multiCommand; /* often lookedup cmds */
+    /* RDB / AOF loading information */
+    int loading;
+    off_t loading_total_bytes;
+    off_t loading_loaded_bytes;
+    time_t loading_start_time;
+    /* Fast pointers to often looked up command */
+    struct redisCommand *delCommand, *multiCommand;
     list *slaves, *monitors;
     char neterr[ANET_ERR_LEN];
     aeEventLoop *el;
@@ -401,15 +409,24 @@ struct redisServer {
     int activerehashing;
     /* Replication related */
     int isslave;
+    /* Slave specific fields */
     char *masterauth;
     char *masterhost;
     int masterport;
     redisClient *master;    /* client that is master for this slave */
-    int replstate;
+    int replstate;          /* replication status if the instance is a slave */
+    off_t repl_transfer_left;  /* bytes left reading .rdb  */
+    int repl_transfer_s;    /* slave -> master SYNC socket */
+    int repl_transfer_fd;   /* slave -> master SYNC temp file descriptor */
+    char *repl_transfer_tmpfile; /* slave-> master SYNC temp file name */
+    time_t repl_transfer_lastio; /* unix time of the latest read, for timeout */
+    int repl_serve_stale_data; /* Serve stale data when link is down? */
+    /* Limits */
     unsigned int maxclients;
     unsigned long long maxmemory;
     int maxmemory_policy;
     int maxmemory_samples;
+    /* Blocked clients */
     unsigned int blpop_blocked_clients;
     unsigned int vm_blocked_clients;
     /* Sort parameters - qsort_r() is only available under BSD so we
@@ -713,6 +730,12 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc);
 void replicationFeedMonitors(list *monitors, int dictid, robj **argv, int argc);
 int syncWithMaster(void);
 void updateSlavesWaitingBgsave(int bgsaveerr);
+void replicationCron(void);
+
+/* Generic persistence functions */
+void startLoading(FILE *fp);
+void loadingProgress(off_t pos);
+void stopLoading(void);
 
 /* RDB persistence */
 int rdbLoad(char *filename);

src/replication.c

@@ -5,6 +5,8 @@
 #include <fcntl.h>
 #include <sys/stat.h>
 
+/* ---------------------------------- MASTER -------------------------------- */
+
 void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) {
     listNode *ln;
     listIter li;
@@ -288,9 +290,105 @@ void updateSlavesWaitingBgsave(int bgsaveerr) {
     }
 }
 
+/* ----------------------------------- SLAVE -------------------------------- */
+
+/* Abort the async download of the bulk dataset while SYNC-ing with master */
+void replicationAbortSyncTransfer(void) {
+    redisAssert(server.replstate == REDIS_REPL_TRANSFER);
+
+    aeDeleteFileEvent(server.el,server.repl_transfer_s,AE_READABLE);
+    close(server.repl_transfer_s);
+    close(server.repl_transfer_fd);
+    unlink(server.repl_transfer_tmpfile);
+    zfree(server.repl_transfer_tmpfile);
+    server.replstate = REDIS_REPL_CONNECT;
+}
+
+/* Asynchronously read the SYNC payload we receive from a master */
+void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) {
+    char buf[4096];
+    ssize_t nread, readlen;
+    REDIS_NOTUSED(el);
+    REDIS_NOTUSED(privdata);
+    REDIS_NOTUSED(mask);
+
+    /* If repl_transfer_left == -1 we still have to read the bulk length
+     * from the master reply. */
+    if (server.repl_transfer_left == -1) {
+        if (syncReadLine(fd,buf,1024,3600) == -1) {
+            redisLog(REDIS_WARNING,
+                "I/O error reading bulk count from MASTER: %s",
+                strerror(errno));
+            replicationAbortSyncTransfer();
+            return;
+        }
+        if (buf[0] == '-') {
+            redisLog(REDIS_WARNING,
+                "MASTER aborted replication with an error: %s",
+                buf+1);
+            replicationAbortSyncTransfer();
+            return;
+        } else if (buf[0] != '$') {
+            redisLog(REDIS_WARNING,"Bad protocol from MASTER, the first byte is not '$', are you sure the host and port are right?");
+            replicationAbortSyncTransfer();
+            return;
+        }
+        server.repl_transfer_left = strtol(buf+1,NULL,10);
+        redisLog(REDIS_NOTICE,
+            "MASTER <-> SLAVE sync: receiving %ld bytes from master",
+            server.repl_transfer_left);
+        return;
+    }
+
+    /* Read bulk data */
+    readlen = (server.repl_transfer_left < (signed)sizeof(buf)) ?
+        server.repl_transfer_left : (signed)sizeof(buf);
+    nread = read(fd,buf,readlen);
+    if (nread <= 0) {
+        redisLog(REDIS_WARNING,"I/O error trying to sync with MASTER: %s",
+            (nread == -1) ? strerror(errno) : "connection lost");
+        replicationAbortSyncTransfer();
+        return;
+    }
+    server.repl_transfer_lastio = time(NULL);
+    if (write(server.repl_transfer_fd,buf,nread) != nread) {
+        redisLog(REDIS_WARNING,"Write error or short write writing to the DB dump file needed for MASTER <-> SLAVE synchrnonization: %s", strerror(errno));
+        replicationAbortSyncTransfer();
+        return;
+    }
+    server.repl_transfer_left -= nread;
+    /* Check if the transfer is now complete */
+    if (server.repl_transfer_left == 0) {
+        if (rename(server.repl_transfer_tmpfile,server.dbfilename) == -1) {
+            redisLog(REDIS_WARNING,"Failed trying to rename the temp DB into dump.rdb in MASTER <-> SLAVE synchronization: %s", strerror(errno));
+            replicationAbortSyncTransfer();
+            return;
+        }
+        redisLog(REDIS_NOTICE, "MASTER <-> SLAVE sync: Loading DB in memory");
+        emptyDb();
+        /* Before loading the DB into memory we need to delete the readable
+         * handler, otherwise it will get called recursively since
+         * rdbLoad() will call the event loop to process events from time to
+         * time for non blocking loading. */
+        aeDeleteFileEvent(server.el,server.repl_transfer_s,AE_READABLE);
+        if (rdbLoad(server.dbfilename) != REDIS_OK) {
+            redisLog(REDIS_WARNING,"Failed trying to load the MASTER synchronization DB from disk");
+            replicationAbortSyncTransfer();
+            return;
+        }
+        /* Final setup of the connected slave <- master link */
+        zfree(server.repl_transfer_tmpfile);
+        close(server.repl_transfer_fd);
+        server.master = createClient(server.repl_transfer_s);
+        server.master->flags |= REDIS_MASTER;
+        server.master->authenticated = 1;
+        server.replstate = REDIS_REPL_CONNECTED;
+        redisLog(REDIS_NOTICE, "MASTER <-> SLAVE sync: Finished with success");
+    }
+}
+
 int syncWithMaster(void) {
     char buf[1024], tmpfile[256], authcmd[1024];
-    long dumpsize;
     int fd = anetTcpConnect(NULL,server.masterhost,server.masterport);
     int dfd, maxtries = 5;
 
@@ -330,26 +428,8 @@ int syncWithMaster(void) {
             strerror(errno));
         return REDIS_ERR;
     }
-    /* Read the bulk write count */
-    if (syncReadLine(fd,buf,1024,3600) == -1) {
-        close(fd);
-        redisLog(REDIS_WARNING,"I/O error reading bulk count from MASTER: %s",
-            strerror(errno));
-        return REDIS_ERR;
-    }
-    if (buf[0] == '-') {
-        close(fd);
-        redisLog(REDIS_WARNING,"MASTER aborted replication with an error: %s",
-            buf+1);
-        return REDIS_ERR;
-    } else if (buf[0] != '$') {
-        close(fd);
-        redisLog(REDIS_WARNING,"Bad protocol from MASTER, the first byte is not '$', are you sure the host and port are right?");
-        return REDIS_ERR;
-    }
-    dumpsize = strtol(buf+1,NULL,10);
-    redisLog(REDIS_NOTICE,"Receiving %ld bytes data dump from MASTER",dumpsize);
-    /* Read the bulk write data on a temp file */
+
+    /* Prepare a suitable temp file for bulk transfer */
     while(maxtries--) {
         snprintf(tmpfile,256,
             "temp-%d.%ld.rdb",(int)time(NULL),(long int)getpid());
@@ -362,43 +442,21 @@ int syncWithMaster(void) {
         redisLog(REDIS_WARNING,"Opening the temp file needed for MASTER <-> SLAVE synchronization: %s",strerror(errno));
         return REDIS_ERR;
     }
-    while(dumpsize) {
-        int nread, nwritten;
 
-        nread = read(fd,buf,(dumpsize < 1024)?dumpsize:1024);
-        if (nread <= 0) {
-            redisLog(REDIS_WARNING,"I/O error trying to sync with MASTER: %s",
-                (nread == -1) ? strerror(errno) : "connection lost");
-            close(fd);
-            close(dfd);
-            return REDIS_ERR;
-        }
-        nwritten = write(dfd,buf,nread);
-        if (nwritten == -1) {
-            redisLog(REDIS_WARNING,"Write error writing to the DB dump file needed for MASTER <-> SLAVE synchrnonization: %s", strerror(errno));
-            close(fd);
-            close(dfd);
-            return REDIS_ERR;
-        }
-        dumpsize -= nread;
-    }
-    close(dfd);
-    if (rename(tmpfile,server.dbfilename) == -1) {
-        redisLog(REDIS_WARNING,"Failed trying to rename the temp DB into dump.rdb in MASTER <-> SLAVE synchronization: %s", strerror(errno));
-        unlink(tmpfile);
+    /* Setup the non blocking download of the bulk file. */
+    if (aeCreateFileEvent(server.el, fd, AE_READABLE, readSyncBulkPayload, NULL)
+            == AE_ERR)
+    {
         close(fd);
+        redisLog(REDIS_WARNING,"Can't create readable event for SYNC");
         return REDIS_ERR;
     }
-    emptyDb();
-    if (rdbLoad(server.dbfilename) != REDIS_OK) {
-        redisLog(REDIS_WARNING,"Failed trying to load the MASTER synchronization DB from disk");
-        close(fd);
-        return REDIS_ERR;
-    }
-    server.master = createClient(fd);
-    server.master->flags |= REDIS_MASTER;
-    server.master->authenticated = 1;
-    server.replstate = REDIS_REPL_CONNECTED;
+    server.replstate = REDIS_REPL_TRANSFER;
+    server.repl_transfer_left = -1;
+    server.repl_transfer_s = fd;
+    server.repl_transfer_fd = dfd;
+    server.repl_transfer_lastio = time(NULL);
+    server.repl_transfer_tmpfile = zstrdup(tmpfile);
     return REDIS_OK;
 }
 
@@ -409,6 +467,8 @@ void slaveofCommand(redisClient *c) {
             sdsfree(server.masterhost);
             server.masterhost = NULL;
             if (server.master) freeClient(server.master);
+            if (server.replstate == REDIS_REPL_TRANSFER)
+                replicationAbortSyncTransfer();
             server.replstate = REDIS_REPL_NONE;
             redisLog(REDIS_NOTICE,"MASTER MODE enabled (user request)");
         }
@@ -417,9 +477,34 @@ void slaveofCommand(redisClient *c) {
         server.masterhost = sdsdup(c->argv[1]->ptr);
         server.masterport = atoi(c->argv[2]->ptr);
         if (server.master) freeClient(server.master);
+        if (server.replstate == REDIS_REPL_TRANSFER)
+            replicationAbortSyncTransfer();
         server.replstate = REDIS_REPL_CONNECT;
         redisLog(REDIS_NOTICE,"SLAVE OF %s:%d enabled (user request)",
             server.masterhost, server.masterport);
     }
     addReply(c,shared.ok);
 }
+
+/* --------------------------- REPLICATION CRON  ---------------------------- */
+
+#define REDIS_REPL_TRANSFER_TIMEOUT 60
+
+void replicationCron(void) {
+    /* Bulk transfer I/O timeout? */
+    if (server.masterhost && server.replstate == REDIS_REPL_TRANSFER &&
+        (time(NULL)-server.repl_transfer_lastio) > REDIS_REPL_TRANSFER_TIMEOUT)
+    {
+        redisLog(REDIS_WARNING,"Timeout receiving bulk data from MASTER...");
+        replicationAbortSyncTransfer();
+    }
+
+    /* Check if we should connect to a MASTER */
+    if (server.replstate == REDIS_REPL_CONNECT) {
+        redisLog(REDIS_NOTICE,"Connecting to MASTER...");
+        if (syncWithMaster() == REDIS_OK) {
+            redisLog(REDIS_NOTICE,"MASTER <-> SLAVE sync started: SYNC sent");
+            if (server.appendonly) rewriteAppendOnlyFileBackground();
+        }
+    }
+}

src/version.h

@@ -1 +1 @@
-#define REDIS_VERSION "2.1.6"
+#define REDIS_VERSION "2.1.7"