Non-LZF aware object.c APIs fixed.

This will be a non-op most of the times since the object will be
unshared / decoded, however it is more technically correct to start this
way since the object may be decoded even in the read-only code path.

redis.conf

@@ -762,3 +762,7 @@ hz 10
 # big latency spikes.
 aof-rewrite-incremental-fsync yes
 
+# Transparently compress string objects in memory using LZF.
+# For default this is set to no since it may affect performances, however
+# when caching things like HTML fragments this may result into a big win.
+memcompression no

src/bitops.c

@@ -261,6 +261,7 @@ void getbitCommand(redisClient *c) {
 
     byte = bitoffset >> 3;
     bit = 7 - (bitoffset & 0x7);
+    if (lzfEncodedObject(o)) o = dbUnshareStringValue(c->db,c->argv[1],o);
     if (sdsEncodedObject(o)) {
         if (byte < sdslen(o->ptr))
             bitval = ((uint8_t*)o->ptr)[byte] & (1 << bit);
@@ -457,6 +458,7 @@ void bitcountCommand(redisClient *c) {
 
     /* Set the 'p' pointer to the string, that can be just a stack allocated
      * array if our string was integer encoded. */
+    if (lzfEncodedObject(o)) o = dbUnshareStringValue(c->db,c->argv[1],o);
     if (o->encoding == REDIS_ENCODING_INT) {
         p = (unsigned char*) llbuf;
         strlen = ll2string(llbuf,sizeof(llbuf),(long)o->ptr);
@@ -526,6 +528,7 @@ void bitposCommand(redisClient *c) {
 
     /* Set the 'p' pointer to the string, that can be just a stack allocated
      * array if our string was integer encoded. */
+    if (lzfEncodedObject(o)) o = dbUnshareStringValue(c->db,c->argv[1],o);
     if (o->encoding == REDIS_ENCODING_INT) {
         p = (unsigned char*) llbuf;
         strlen = ll2string(llbuf,sizeof(llbuf),(long)o->ptr);

src/config.c

@@ -297,6 +297,10 @@ void loadServerConfigFromString(char *config) {
             if ((server.rdb_compression = yesnotoi(argv[1])) == -1) {
                 err = "argument must be 'yes' or 'no'"; goto loaderr;
             }
+        } else if (!strcasecmp(argv[0],"memcompression") && argc == 2) {
+            if ((server.mem_compression = yesnotoi(argv[1])) == -1) {
+                err = "argument must be 'yes' or 'no'"; goto loaderr;
+            }
         } else if (!strcasecmp(argv[0],"rdbchecksum") && argc == 2) {
             if ((server.rdb_checksum = yesnotoi(argv[1])) == -1) {
                 err = "argument must be 'yes' or 'no'"; goto loaderr;
@@ -860,6 +864,11 @@ void configSetCommand(redisClient *c) {
 
         if (yn == -1) goto badfmt;
         server.rdb_compression = yn;
+    } else if (!strcasecmp(c->argv[2]->ptr,"memcompression")) {
+        int yn = yesnotoi(o->ptr);
+
+        if (yn == -1) goto badfmt;
+        server.mem_compression = yn;
     } else if (!strcasecmp(c->argv[2]->ptr,"notify-keyspace-events")) {
         int flags = keyspaceEventsStringToFlags(o->ptr);
 
@@ -1006,6 +1015,7 @@ void configGetCommand(redisClient *c) {
             server.stop_writes_on_bgsave_err);
     config_get_bool_field("daemonize", server.daemonize);
     config_get_bool_field("rdbcompression", server.rdb_compression);
+    config_get_bool_field("memcompression", server.mem_compression);
     config_get_bool_field("rdbchecksum", server.rdb_checksum);
     config_get_bool_field("activerehashing", server.activerehashing);
     config_get_bool_field("repl-disable-tcp-nodelay",
@@ -1721,6 +1731,7 @@ int rewriteConfig(char *path) {
     rewriteConfigNumericalOption(state,"databases",server.dbnum,REDIS_DEFAULT_DBNUM);
     rewriteConfigYesNoOption(state,"stop-writes-on-bgsave-error",server.stop_writes_on_bgsave_err,REDIS_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR);
     rewriteConfigYesNoOption(state,"rdbcompression",server.rdb_compression,REDIS_DEFAULT_RDB_COMPRESSION);
+    rewriteConfigYesNoOption(state,"memcompression",server.mem_compression,REDIS_DEFAULT_MEM_COMPRESSION);
     rewriteConfigYesNoOption(state,"rdbchecksum",server.rdb_checksum,REDIS_DEFAULT_RDB_CHECKSUM);
     rewriteConfigStringOption(state,"dbfilename",server.rdb_filename,REDIS_DEFAULT_RDB_FILENAME);
     rewriteConfigDirOption(state);

src/debug.c

@@ -308,8 +308,10 @@ void debugCommand(redisClient *c) {
         val = dictGetVal(de);
         key = dictGetKey(de);
 
-        if (val->type != REDIS_STRING || !sdsEncodedObject(val)) {
-            addReplyError(c,"Not an sds encoded string.");
+        if (val->type != REDIS_STRING ||
+            (!sdsEncodedObject(val) && val->encoding != REDIS_ENCODING_LZF))
+        {
+            addReplyError(c,"Not an sds/lzf encoded string.");
         } else {
             addReplyStatusFormat(c,
                 "key_sds_len:%lld, key_sds_avail:%lld, "

src/hyperloglog.c

@@ -520,6 +520,7 @@ void pfcountCommand(redisClient *c) {
         addReply(c,shared.czero);
     } else {
         if (isHLLObjectOrReply(c,o) != REDIS_OK) return;
+        o = dbUnshareStringValue(c->db,c->argv[1],o);
 
         /* Check if the cached cardinality is valid. */
         registers = o->ptr;
@@ -535,9 +536,6 @@ void pfcountCommand(redisClient *c) {
             card |= (uint64_t)registers[REDIS_HLL_SIZE-9] << 56;
         } else {
             /* Recompute it and update the cached value. */
-            o = dbUnshareStringValue(c->db,c->argv[1],o);
-            registers = o->ptr;
-
             card = hllCount(registers);
             registers[REDIS_HLL_SIZE-16] = card & 0xff;
             registers[REDIS_HLL_SIZE-15] = (card >> 8) & 0xff;

src/networking.c

@@ -319,6 +319,11 @@ void addReply(redisClient *c, robj *obj) {
         if (_addReplyToBuffer(c,obj->ptr,sdslen(obj->ptr)) != REDIS_OK)
             _addReplyObjectToList(c,obj);
         decrRefCount(obj);
+    } else if (obj->encoding == REDIS_ENCODING_LZF) {
+        obj = getDecodedObject(obj);
+        if (_addReplyToBuffer(c,obj->ptr,sdslen(obj->ptr)) != REDIS_OK)
+            _addReplyObjectToList(c,obj);
+        decrRefCount(obj);
     } else {
         redisPanic("Wrong obj->encoding in addReply()");
     }
@@ -488,7 +493,7 @@ void addReplyBulkLen(redisClient *c, robj *obj) {
 
     if (sdsEncodedObject(obj)) {
         len = sdslen(obj->ptr);
-    } else {
+    } else if (obj->encoding == REDIS_ENCODING_INT) {
         long n = (long)obj->ptr;
 
         /* Compute how many bytes will take this integer as a radix 10 string */
@@ -500,6 +505,9 @@ void addReplyBulkLen(redisClient *c, robj *obj) {
         while((n = n/10) != 0) {
             len++;
         }
+    } else {
+        /* LZF and others not handled explicitly. */
+        len = stringObjectLen(obj);
     }
 
     if (len < REDIS_SHARED_BULKHDR_LEN)

src/object.c

@@ -29,6 +29,7 @@
  */
 
 #include "redis.h"
+#include "lzf.h"    /* LZF compression library */
 #include <math.h>
 #include <ctype.h>
 
@@ -149,6 +150,10 @@ robj *dupStringObject(robj *o) {
         d->encoding = REDIS_ENCODING_INT;
         d->ptr = o->ptr;
         return d;
+    case REDIS_ENCODING_LZF:
+        d = createObject(REDIS_STRING,sdsdup(o->ptr));
+        d->encoding = REDIS_ENCODING_LZF;
+        return d;
     default:
         redisPanic("Wrong encoding.");
         break;
@@ -210,7 +215,8 @@ robj *createZsetZiplistObject(void) {
 }
 
 void freeStringObject(robj *o) {
-    if (o->encoding == REDIS_ENCODING_RAW) {
+    if (o->encoding == REDIS_ENCODING_RAW ||
+        o->encoding == REDIS_ENCODING_LZF) {
         sdsfree(o->ptr);
     }
 }
@@ -327,78 +333,138 @@ int checkType(redisClient *c, robj *o, int type) {
 
 int isObjectRepresentableAsLongLong(robj *o, long long *llval) {
     redisAssertWithInfo(NULL,o,o->type == REDIS_STRING);
-    if (o->encoding == REDIS_ENCODING_INT) {
+    if (intEncodedObject(o)) {
         if (llval) *llval = (long) o->ptr;
         return REDIS_OK;
-    } else {
+    } else if (sdsEncodedObject(o)) {
         return string2ll(o->ptr,sdslen(o->ptr),llval) ? REDIS_OK : REDIS_ERR;
+    } else if (lzfEncodedObject(o)) {
+        int retval;
+        robj *decoded = getDecodedObject(o);
+        retval = string2ll(o->ptr,sdslen(o->ptr),llval) ? REDIS_OK : REDIS_ERR;
+        decrRefCount(decoded);
+        return retval;
+    } else {
+        redisPanic("Unknown encoding.");
     }
 }
 
-/* Try to encode a string object in order to save space */
+/* Try to encode a string object in order to save space. */
+#define LZF_COMPR_STATIC_BUF (1024*32)
 robj *tryObjectEncoding(robj *o) {
     long value;
     sds s = o->ptr;
     size_t len;
 
-    if (o->encoding == REDIS_ENCODING_INT)
-        return o; /* Already encoded */
+    /* Make sure this is a string object, the only type we encode
+     * in this function. Other types use encoded memory efficient
+     * representations but are handled by the commands implementing
+     * the type. */
+    redisAssertWithInfo(NULL,o,o->type == REDIS_STRING);
+
+    /* We try some specialized encoding only for objects that are
+     * RAW or EMBSTR encoded, in other words objects that are still
+     * in represented by an actually array of chars. */
+    if (!sdsEncodedObject(o)) return o;
 
     /* It's not safe to encode shared objects: shared objects can be shared
-     * everywhere in the "object space" of Redis. Encoded objects can only
-     * appear as "values" (and not, for instance, as keys) */
+     * everywhere in the "object space" of Redis and may end in places where
+     * they are not handled. We handle them only as values in the keyspace. */
      if (o->refcount > 1) return o;
 
-    /* Currently we try to encode only strings */
-    redisAssertWithInfo(NULL,o,o->type == REDIS_STRING);
-
     /* Check if we can represent this string as a long integer.
      * Note that we are sure that a string larger than 21 chars is not
-     * representable as a 64 bit integer. */
+     * representable as a 32 nor 64 bit integer. */
     len = sdslen(s);
-    if (len > 21 || !string2l(s,len,&value)) {
-        /* Integer encoding not possible. Check if we can use EMBSTR. */
-        if (sdslen(s) <= REDIS_ENCODING_EMBSTR_SIZE_LIMIT) {
-            robj *emb = createEmbeddedStringObject(s,sdslen(s));
+    if (len <= 21 && string2l(s,len,&value)) {
+        /* This object is encodable as a long. Try to use a shared object.
+         * Note that we avoid using shared integers when maxmemory is used
+         * because every object needs to have a private LRU field for the LRU
+         * algorithm to work well. */
+        if (server.maxmemory == 0 &&
+            value >= 0 &&
+            value < REDIS_SHARED_INTEGERS)
+        {
             decrRefCount(o);
-            return emb;
+            incrRefCount(shared.integers[value]);
+            return shared.integers[value];
         } else {
-            /* We can't encode the object...
-             *
-             * Do the last try, and at least optimize the SDS string inside
-             * the string object to require little space, in case there
-             * is more than 10% of free space at the end of the SDS string.
-             *
-             * We do that only for relatively large strings as this branch
-             * is only entered if the length of the string is greater than
-             * REDIS_ENCODING_EMBSTR_SIZE_LIMIT. */
-            if (o->encoding == REDIS_ENCODING_RAW &&
-                sdsavail(s) > len/10)
-            {
-                o->ptr = sdsRemoveFreeSpace(o->ptr);
-            }
-            /* Return the original object. */
+            if (o->encoding == REDIS_ENCODING_RAW) sdsfree(o->ptr);
+            o->encoding = REDIS_ENCODING_INT;
+            o->ptr = (void*) value;
             return o;
         }
     }
 
-    /* Ok, this object can be encoded...
-     *
-     * Can I use a shared object? Only if the object is inside a given range
-     *
-     * Note that we also avoid using shared integers when maxmemory is used
-     * because every object needs to have a private LRU field for the LRU
-     * algorithm to work well. */
-    if (server.maxmemory == 0 && value >= 0 && value < REDIS_SHARED_INTEGERS) {
+    /* If the string is small and is still RAW encoded,
+     * try the EMBSTR encoding which is more efficient.
+     * In this representation the object and the SDS string are allocated
+     * in the same chunk of memory to save space and cache misses. */
+    if (len <= REDIS_ENCODING_EMBSTR_SIZE_LIMIT) {
+        robj *emb;
+
+        if (o->encoding == REDIS_ENCODING_EMBSTR) return o;
+        emb = createEmbeddedStringObject(s,sdslen(s));
         decrRefCount(o);
-        incrRefCount(shared.integers[value]);
-        return shared.integers[value];
-    } else {
-        if (o->encoding == REDIS_ENCODING_RAW) sdsfree(o->ptr);
-        o->encoding = REDIS_ENCODING_INT;
-        o->ptr = (void*) value;
-        return o;
+        return emb;
     }
+
+    /* Try LZF compression for objects up to server.mem_compression_max_size
+     * and greater than REDIS_ENCODING_EMBSTR_SIZE_LIMIT. */
+    if (server.mem_compression && len <= server.mem_compression_max_size) {
+        /* Allocate four more bytes in our buffer since we need to store
+         * the size of the compressed string as header. */
+        unsigned char compr_static[LZF_COMPR_STATIC_BUF];
+        unsigned char *compr = compr_static;
+        size_t comprlen, outlen;
+
+        /* Save want to save at least 25% of memory for this to make sense. */
+        outlen = len-4-(len/4);
+
+        /* Use an heap allocated buffer if the output is too big for our
+         * static buffer. We use the trick to directly allocating an empty
+         * SDS string so we can use it directly to create the object later. */
+        if (outlen+4 > LZF_COMPR_STATIC_BUF)
+            compr = (unsigned char*) sdsnewlen(NULL,outlen+4);
+        comprlen = lzf_compress(s,len,compr+4,outlen);
+        if (comprlen != 0) {
+            /* Object successfully compressed within the required space. */
+            compr[0] = len & 0xff;
+            compr[1] = (len >> 8) & 0xff;
+            compr[2] = (len >> 16) & 0xff;
+            compr[3] = (len >> 24) & 0xff;
+            if (o->encoding == REDIS_ENCODING_RAW) sdsfree(o->ptr);
+            o->encoding = REDIS_ENCODING_LZF;
+            if (compr == compr_static) {
+                /* When compressing to the static buffer we have to
+                 * generate the SDS string here. */
+                o->ptr = sdsnewlen(compr,comprlen+4);
+            } else {
+                /* Already an SDS, use it. */
+                o->ptr = compr;
+            }
+            return o;
+        }
+        if (compr != compr_static) sdsfree((char*)compr);
+    }
+
+    /* We can't encode the object...
+     *
+     * Do the last try, and at least optimize the SDS string inside
+     * the string object to require little space, in case there
+     * is more than 10% of free space at the end of the SDS string.
+     *
+     * We do that only for relatively large strings as this branch
+     * is only entered if the length of the string is greater than
+     * REDIS_ENCODING_EMBSTR_SIZE_LIMIT. */
+    if (o->encoding == REDIS_ENCODING_RAW &&
+        sdsavail(s) > len/10)
+    {
+        o->ptr = sdsRemoveFreeSpace(o->ptr);
+    }
+
+    /* Return the original object. */
+    return o;
 }
 
 /* Get a decoded version of an encoded object (returned as a new object).
@@ -410,12 +476,20 @@ robj *getDecodedObject(robj *o) {
         incrRefCount(o);
         return o;
     }
-    if (o->type == REDIS_STRING && o->encoding == REDIS_ENCODING_INT) {
+    if (o->type == REDIS_STRING && intEncodedObject(o)) {
         char buf[32];
 
         ll2string(buf,32,(long)o->ptr);
         dec = createStringObject(buf,strlen(buf));
         return dec;
+    } else if (o->type == REDIS_STRING && lzfEncodedObject(o)) {
+        int origlen = stringObjectLen(o);
+        sds orig = sdsnewlen(NULL,origlen);
+        unsigned char *p = o->ptr;
+
+        if (lzf_decompress(p+4,sdslen(o->ptr)-4,orig,origlen) == 0)
+            redisPanic("LZF error during object decoding.");
+        return createObject(REDIS_STRING,orig);
     } else {
         redisPanic("Unknown encoding type");
     }
@@ -436,8 +510,15 @@ int compareStringObjectsWithFlags(robj *a, robj *b, int flags) {
     redisAssertWithInfo(NULL,a,a->type == REDIS_STRING && b->type == REDIS_STRING);
     char bufa[128], bufb[128], *astr, *bstr;
     size_t alen, blen, minlen;
+    int decr = 0;
 
     if (a == b) return 0;
+
+    if (lzfEncodedObject(a) || lzfEncodedObject(b)) {
+        a = getDecodedObject(a);
+        b = getDecodedObject(b);
+        decr = 1;
+    }
     if (sdsEncodedObject(a)) {
         astr = a->ptr;
         alen = sdslen(astr);
@@ -462,6 +543,10 @@ int compareStringObjectsWithFlags(robj *a, robj *b, int flags) {
         if (cmp == 0) return alen-blen;
         return cmp;
     }
+    if (decr) {
+        decrRefCount(a);
+        decrRefCount(b);
+    }
 }
 
 /* Wrapper for compareStringObjectsWithFlags() using binary comparison. */
@@ -479,8 +564,7 @@ int collateStringObjects(robj *a, robj *b) {
  * this function is faster then checking for (compareStringObject(a,b) == 0)
  * because it can perform some more optimization. */
 int equalStringObjects(robj *a, robj *b) {
-    if (a->encoding == REDIS_ENCODING_INT &&
-        b->encoding == REDIS_ENCODING_INT){
+    if (intEncodedObject(a) && intEncodedObject(b)) {
         /* If both strings are integer encoded just check if the stored
          * long is the same. */
         return a->ptr == b->ptr;
@@ -489,13 +573,21 @@ int equalStringObjects(robj *a, robj *b) {
     }
 }
 
+/* Returns the original (uncompressed) size of an LZF encoded object.
+ * Only called by stringObjectLen() that should be the main interface. */
+size_t stringObjectUncompressedLen(robj *o) {
+    unsigned char *p = o->ptr;
+    return p[0] | (p[1]<<8) | (p[2]<<16) | (p[3]<<24);
+}
+
 size_t stringObjectLen(robj *o) {
     redisAssertWithInfo(NULL,o,o->type == REDIS_STRING);
     if (sdsEncodedObject(o)) {
         return sdslen(o->ptr);
+    } else if (o->encoding == REDIS_ENCODING_LZF) {
+        return stringObjectUncompressedLen(o);
     } else {
         char buf[32];
-
         return ll2string(buf,32,(long)o->ptr);
     }
 }
@@ -518,8 +610,14 @@ int getDoubleFromObject(robj *o, double *target) {
                 errno == EINVAL ||
                 isnan(value))
                 return REDIS_ERR;
-        } else if (o->encoding == REDIS_ENCODING_INT) {
+        } else if (intEncodedObject(o)) {
             value = (long)o->ptr;
+        } else if (lzfEncodedObject(o)) {
+            int retval;
+            o = getDecodedObject(o);
+            retval = getDoubleFromObject(o,target);
+            decrRefCount(o);
+            return retval;
         } else {
             redisPanic("Unknown string encoding");
         }
@@ -556,8 +654,14 @@ int getLongDoubleFromObject(robj *o, long double *target) {
             if (isspace(((char*)o->ptr)[0]) || eptr[0] != '\0' ||
                 errno == ERANGE || isnan(value))
                 return REDIS_ERR;
-        } else if (o->encoding == REDIS_ENCODING_INT) {
+        } else if (intEncodedObject(o)) {
             value = (long)o->ptr;
+        } else if (lzfEncodedObject(o)) {
+            int retval;
+            o = getDecodedObject(o);
+            retval = getLongDoubleFromObject(o,target);
+            decrRefCount(o);
+            return retval;
         } else {
             redisPanic("Unknown string encoding");
         }
@@ -594,8 +698,14 @@ int getLongLongFromObject(robj *o, long long *target) {
             if (isspace(((char*)o->ptr)[0]) || eptr[0] != '\0' ||
                 errno == ERANGE)
                 return REDIS_ERR;
-        } else if (o->encoding == REDIS_ENCODING_INT) {
+        } else if (intEncodedObject(o)) {
             value = (long)o->ptr;
+        } else if (lzfEncodedObject(o)) {
+            int retval;
+            o = getDecodedObject(o);
+            retval = getLongLongFromObject(o,target);
+            decrRefCount(o);
+            return retval;
         } else {
             redisPanic("Unknown string encoding");
         }
@@ -644,6 +754,7 @@ char *strEncoding(int encoding) {
     case REDIS_ENCODING_INTSET: return "intset";
     case REDIS_ENCODING_SKIPLIST: return "skiplist";
     case REDIS_ENCODING_EMBSTR: return "embstr";
+    case REDIS_ENCODING_LZF: return "lzf";
     default: return "unknown";
     }
 }

src/rdb.c

@@ -209,11 +209,41 @@ int rdbTryIntegerEncoding(char *s, size_t len, unsigned char *enc) {
     return rdbEncodeInteger(value,enc);
 }
 
-int rdbSaveLzfStringObject(rio *rdb, unsigned char *s, size_t len) {
-    size_t comprlen, outlen;
+/* Save an already compressed object in LZF encoding.
+ *
+ * On success the length of the strored object is returned, otherwise
+ * 0 is returned. */
+int rdbSaveLzfStringObject(rio *rdb, unsigned char *out, size_t len, size_t comprlen) {
     unsigned char byte;
     int n, nwritten = 0;
+
+    /* Data compressed! Let's save it on disk */
+    byte = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_LZF;
+    if ((n = rdbWriteRaw(rdb,&byte,1)) == -1) goto writeerr;
+    nwritten += n;
+    if ((n = rdbSaveLen(rdb,comprlen)) == -1) goto writeerr;
+    nwritten += n;
+    if ((n = rdbSaveLen(rdb,len)) == -1) goto writeerr;
+    nwritten += n;
+    if ((n = rdbWriteRaw(rdb,out,comprlen)) == -1) goto writeerr;
+    nwritten += n;
+    return nwritten;
+
+writeerr:
+    zfree(out);
+    return -1;
+}
+
+/* Try to compress the string at 's' for 'len' bytes using LZF.
+ * If successful save the object with LZF encoding, otherwise
+ * returns 0 if the string can't be compressed, or -1 if the
+ * compressed string can't be saved.
+ *
+ * On success the number of bytes used is returned. */
+int rdbTrySaveLzfStringObject(rio *rdb, unsigned char *s, size_t len) {
+    size_t comprlen, outlen;
     void *out;
+    int retval;
 
     /* We require at least four bytes compression for this to be worth it */
     if (len <= 4) return 0;
@@ -224,26 +254,9 @@ int rdbSaveLzfStringObject(rio *rdb, unsigned char *s, size_t len) {
         zfree(out);
         return 0;
     }
-    /* Data compressed! Let's save it on disk */
-    byte = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_LZF;
-    if ((n = rdbWriteRaw(rdb,&byte,1)) == -1) goto writeerr;
-    nwritten += n;
-
-    if ((n = rdbSaveLen(rdb,comprlen)) == -1) goto writeerr;
-    nwritten += n;
-
-    if ((n = rdbSaveLen(rdb,len)) == -1) goto writeerr;
-    nwritten += n;
-
-    if ((n = rdbWriteRaw(rdb,out,comprlen)) == -1) goto writeerr;
-    nwritten += n;
-
+    retval = rdbSaveLzfStringObject(rdb,out,len,comprlen);
     zfree(out);
-    return nwritten;
-
-writeerr:
-    zfree(out);
-    return -1;
+    return retval;
 }
 
 robj *rdbLoadLzfStringObject(rio *rdb) {
@@ -283,7 +296,7 @@ int rdbSaveRawString(rio *rdb, unsigned char *s, size_t len) {
     /* Try LZF compression - under 20 bytes it's unable to compress even
      * aaaaaaaaaaaaaaaaaa so skip it */
     if (server.rdb_compression && len > 20) {
-        n = rdbSaveLzfStringObject(rdb,s,len);
+        n = rdbTrySaveLzfStringObject(rdb,s,len);
         if (n == -1) return -1;
         if (n > 0) return n;
         /* Return value of 0 means data can't be compressed, save the old way */
@@ -324,6 +337,11 @@ int rdbSaveStringObject(rio *rdb, robj *obj) {
      * object is already integer encoded. */
     if (obj->encoding == REDIS_ENCODING_INT) {
         return rdbSaveLongLongAsStringObject(rdb,(long)obj->ptr);
+    } else if (obj->encoding == REDIS_ENCODING_LZF) {
+        /* Data is already compressed, save it with LZF encoding. */
+        int len = stringObjectLen(obj);
+        unsigned char *p = obj->ptr;
+        return rdbSaveLzfStringObject(rdb,p+4,len,sdslen(obj->ptr)-4);
     } else {
         redisAssertWithInfo(NULL,obj,sdsEncodedObject(obj));
         return rdbSaveRawString(rdb,obj->ptr,sdslen(obj->ptr));

src/redis.c

@@ -1397,6 +1397,8 @@ void initServerConfig() {
     server.aof_filename = zstrdup(REDIS_DEFAULT_AOF_FILENAME);
     server.requirepass = NULL;
     server.rdb_compression = REDIS_DEFAULT_RDB_COMPRESSION;
+    server.mem_compression = REDIS_DEFAULT_MEM_COMPRESSION;
+    server.mem_compression_max_size = REDIS_DEFAULT_MEM_COMPRESSION_MAX_SIZE;
     server.rdb_checksum = REDIS_DEFAULT_RDB_CHECKSUM;
     server.stop_writes_on_bgsave_err = REDIS_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR;
     server.activerehashing = REDIS_DEFAULT_ACTIVE_REHASHING;

src/redis.h

@@ -107,6 +107,8 @@
 #define REDIS_DEFAULT_SYSLOG_ENABLED 0
 #define REDIS_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR 1
 #define REDIS_DEFAULT_RDB_COMPRESSION 1
+#define REDIS_DEFAULT_MEM_COMPRESSION 0
+#define REDIS_DEFAULT_MEM_COMPRESSION_MAX_SIZE (1024*64)
 #define REDIS_DEFAULT_RDB_CHECKSUM 1
 #define REDIS_DEFAULT_RDB_FILENAME "dump.rdb"
 #define REDIS_DEFAULT_SLAVE_SERVE_STALE_DATA 1
@@ -172,7 +174,7 @@
 
 /* Objects encoding. Some kind of objects like Strings and Hashes can be
  * internally represented in multiple ways. The 'encoding' field of the object
- * is set to one of this fields for this object. */
+ * is set to one of this values. */
 #define REDIS_ENCODING_RAW 0     /* Raw representation */
 #define REDIS_ENCODING_INT 1     /* Encoded as integer */
 #define REDIS_ENCODING_HT 2      /* Encoded as hash table */
@@ -182,6 +184,7 @@
 #define REDIS_ENCODING_INTSET 6  /* Encoded as intset */
 #define REDIS_ENCODING_SKIPLIST 7  /* Encoded as skiplist */
 #define REDIS_ENCODING_EMBSTR 8  /* Embedded sds string encoding */
+#define REDIS_ENCODING_LZF 9     /* LZF compressed string. */
 
 /* Defines related to the dump file format. To store 32 bits lengths for short
  * keys requires a lot of space, so we check the most significant 2 bits of
@@ -809,8 +812,10 @@ struct redisServer {
     size_t set_max_intset_entries;
     size_t zset_max_ziplist_entries;
     size_t zset_max_ziplist_value;
+    int mem_compression;    /* In memory LZF compression. */
+    size_t mem_compression_max_size; /* Try to compress up to this size. */
     time_t unixtime;        /* Unix time sampled every cron cycle. */
-    long long mstime;       /* Like 'unixtime' but with milliseconds resolution. */
+    long long mstime;       /* Like unixtime but in milliseconds. */
     /* Pubsub */
     dict *pubsub_channels;  /* Map channels to list of subscribed clients */
     list *pubsub_patterns;  /* A list of pubsub_patterns */
@@ -1080,6 +1085,9 @@ int compareStringObjects(robj *a, robj *b);
 int collateStringObjects(robj *a, robj *b);
 int equalStringObjects(robj *a, robj *b);
 unsigned long long estimateObjectIdleTime(robj *o);
+#define rawEncodedObject(objptr) (objptr->encoding == REDIS_ENCODING_RAW)
+#define lzfEncodedObject(objptr) (objptr->encoding == REDIS_ENCODING_LZF)
+#define intEncodedObject(objptr) (objptr->encoding == REDIS_ENCODING_INT)
 #define sdsEncodedObject(objptr) (objptr->encoding == REDIS_ENCODING_RAW || objptr->encoding == REDIS_ENCODING_EMBSTR)
 
 /* Synchronous I/O with timeout */

src/slowlog.c

@@ -63,15 +63,17 @@ slowlogEntry *slowlogCreateEntry(robj **argv, int argc, long long duration) {
         } else {
             /* Trim too long strings as well... */
             if (argv[j]->type == REDIS_STRING &&
-                sdsEncodedObject(argv[j]) &&
-                sdslen(argv[j]->ptr) > SLOWLOG_ENTRY_MAX_STRING)
+                (sdsEncodedObject(argv[j]) || lzfEncodedObject(argv[j])) &&
+                stringObjectLen(argv[j]) > SLOWLOG_ENTRY_MAX_STRING)
             {
-                sds s = sdsnewlen(argv[j]->ptr, SLOWLOG_ENTRY_MAX_STRING);
+                robj *o = getDecodedObject(argv[j]);
+                sds s = sdsnewlen(o->ptr, SLOWLOG_ENTRY_MAX_STRING);
 
                 s = sdscatprintf(s,"... (%lu more bytes)",
                     (unsigned long)
-                    sdslen(argv[j]->ptr) - SLOWLOG_ENTRY_MAX_STRING);
+                    stringObjectLen(argv[j]) - SLOWLOG_ENTRY_MAX_STRING);
                 se->argv[j] = createObject(REDIS_STRING,s);
+                decrRefCount(o);
             } else {
                 se->argv[j] = argv[j];
                 incrRefCount(argv[j]);

src/t_string.c

@@ -242,6 +242,7 @@ void getrangeCommand(redisClient *c) {
     if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptybulk)) == NULL ||
         checkType(c,o,REDIS_STRING)) return;
 
+    if (lzfEncodedObject(o)) o = dbUnshareStringValue(c->db,c->argv[1],o);
     if (o->encoding == REDIS_ENCODING_INT) {
         str = llbuf;
         strlen = ll2string(llbuf,sizeof(llbuf),(long)o->ptr);