RSS aware maxmemory: algorith #1 implemented.

src/config.c

@@ -226,6 +226,11 @@ void loadServerConfigFromString(char *config) {
             }
         } else if (!strcasecmp(argv[0],"maxmemory") && argc == 2) {
             server.maxmemory = memtoll(argv[1],NULL);
+            server.maxmemory_enforced = (double) server.maxmemory / server.maxmemory_frag_guess;
+        } else if (!strcasecmp(argv[0],"rss-aware-maxmemory") && argc==2) {
+            if ((server.rss_aware_maxmemory = yesnotoi(argv[1])) == -1) {
+                err = "argument must be 'yes' or 'no'"; goto loaderr;
+            }
         } else if (!strcasecmp(argv[0],"maxmemory-policy") && argc == 2) {
             if (!strcasecmp(argv[1],"volatile-lru")) {
                 server.maxmemory_policy = REDIS_MAXMEMORY_VOLATILE_LRU;
@@ -631,12 +636,18 @@ void configSetCommand(redisClient *c) {
         if (getLongLongFromObject(o,&ll) == REDIS_ERR ||
             ll < 0) goto badfmt;
         server.maxmemory = ll;
+        server.maxmemory_enforced = (double) server.maxmemory / server.maxmemory_frag_guess;
         if (server.maxmemory) {
             if (server.maxmemory < zmalloc_used_memory()) {
                 redisLog(REDIS_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in keys eviction and/or inability to accept new write commands depending on the maxmemory-policy.");
             }
             freeMemoryIfNeeded();
         }
+    } else if (!strcasecmp(c->argv[2]->ptr,"rss-aware-maxmemory")) {
+        int yn = yesnotoi(o->ptr);
+
+        if (yn == -1) goto badfmt;
+        server.rss_aware_maxmemory = yn;
     } else if (!strcasecmp(c->argv[2]->ptr,"maxclients")) {
         int orig_value = server.maxclients;
 
@@ -1093,6 +1104,8 @@ void configGetCommand(redisClient *c) {
             server.aof_rewrite_incremental_fsync);
     config_get_bool_field("aof-load-truncated",
             server.aof_load_truncated);
+    config_get_bool_field("rss-aware-maxmemory",
+            server.rss_aware_maxmemory);
 
     /* Everything we can't handle with macros follows. */
 

src/config.h

@@ -48,11 +48,13 @@
 #define HAVE_PROC_STAT 1
 #define HAVE_PROC_MAPS 1
 #define HAVE_PROC_SMAPS 1
+#define HAVE_RSS_REPORTING 1
 #endif
 
 /* Test for task_info() */
 #if defined(__APPLE__)
 #define HAVE_TASKINFO 1
+#define HAVE_RSS_REPORTING 1
 #endif
 
 /* Test for backtrace() */

src/redis.c

@@ -1433,8 +1433,11 @@ void initServerConfig(void) {
     server.maxclients = REDIS_MAX_CLIENTS;
     server.bpop_blocked_clients = 0;
     server.maxmemory = REDIS_DEFAULT_MAXMEMORY;
+    server.maxmemory_frag_guess = REDIS_DEFAULT_MAXMEMORY_FRAG_GUESS;
+    server.maxmemory_enforced = (double) REDIS_DEFAULT_MAXMEMORY / server.maxmemory_frag_guess;
     server.maxmemory_policy = REDIS_DEFAULT_MAXMEMORY_POLICY;
     server.maxmemory_samples = REDIS_DEFAULT_MAXMEMORY_SAMPLES;
+    server.rss_aware_maxmemory = REDIS_DEFAULT_RSS_AWARE_MAXMEMORY;
     server.hash_max_ziplist_entries = REDIS_HASH_MAX_ZIPLIST_ENTRIES;
     server.hash_max_ziplist_value = REDIS_HASH_MAX_ZIPLIST_VALUE;
     server.list_max_ziplist_entries = REDIS_LIST_MAX_ZIPLIST_ENTRIES;
@@ -3154,7 +3157,7 @@ void evictionPoolPopulate(dict *sampledict, dict *keydict, struct evictionPoolEn
 }
 
 int freeMemoryIfNeeded(void) {
-    size_t mem_used, mem_tofree, mem_freed;
+    size_t mem_used, mem_tofree, mem_freed, mem_target = server.maxmemory;
     int slaves = listLength(server.slaves);
     mstime_t latency;
 
@@ -3180,14 +3183,72 @@ int freeMemoryIfNeeded(void) {
         mem_used -= aofRewriteBufferSize();
     }
 
+    /* If we use RSS aware maxmemory, update the target memory using
+     * the current fragmentation figure. */
+#ifdef HAVE_RSS_REPORTING
+    if (server.rss_aware_maxmemory &&
+        server.maxmemory_policy != REDIS_MAXMEMORY_NO_EVICTION)
+    {
+        static unsigned long iterations = 0;
+        static unsigned long sampling_stage = 1;
+        static float last_observed_frag = 0;
+
+        /* For some time, we analyze what happens during memory pressure, when
+         * objects are evicted and reallocated. */
+        if (mem_used > server.maxmemory_enforced) {
+            unsigned long sample_cycles = 1000000;
+
+            /* Every sample_cycle cycles we sample the fragmentation, and
+             * compare it with the previos one. If it is no longer raising,
+             * we take it as a guess of the fragmentation with this workload. */
+            if (sampling_stage && iterations < sample_cycles) {
+                iterations++;
+                if (iterations == sample_cycles) {
+                    float current_frag = zmalloc_get_fragmentation_ratio(server.resident_set_size);
+                    if (last_observed_frag == 0) {
+                        /* First sample we get. */
+                        last_observed_frag = current_frag;
+                    } else {
+                        if (current_frag <= last_observed_frag) {
+                            size_t enforced_new;
+
+                            sampling_stage = 0;
+                            server.maxmemory_frag_guess = current_frag;
+                            /* Update the global fragmentation guess and use
+                             * it (also used it for successive
+                             * "CONFIG SET maxmemory" commands). */
+                            if (server.maxmemory_frag_guess < 1)
+                                server.maxmemory_frag_guess = 1;
+                            else if (server.maxmemory_frag_guess > 2)
+                                server.maxmemory_frag_guess = 2;
+
+                            /* Only set the new limit if it is higher than our
+                             * initial guess, otherwise it is futile: RSS will
+                             * not go backward anyway. */
+                            enforced_new = (double) server.maxmemory /
+                                           server.maxmemory_frag_guess;
+                            if (enforced_new > server.maxmemory_enforced)
+                                server.maxmemory_enforced = enforced_new;
+                            redisLog(REDIS_NOTICE,"RSS aware maxmemory, fragmentation looks stable at: %f", server.maxmemory_frag_guess);
+                        }
+                        last_observed_frag = current_frag;
+                    }
+                    iterations = 0;
+                }
+            }
+        }
+        mem_target = server.maxmemory_enforced;
+    }
+#endif
+
     /* Check if we are over the memory limit. */
-    if (mem_used <= server.maxmemory) return REDIS_OK;
+    if (mem_used <= mem_target) return REDIS_OK;
 
     if (server.maxmemory_policy == REDIS_MAXMEMORY_NO_EVICTION)
         return REDIS_ERR; /* We need to free memory, but policy forbids. */
 
     /* Compute how much memory we need to free. */
-    mem_tofree = mem_used - server.maxmemory;
+    mem_tofree = mem_used - mem_target;
     mem_freed = 0;
     latencyStartMonitor(latency);
     while (mem_freed < mem_tofree) {

src/redis.h

@@ -121,6 +121,8 @@ typedef long long mstime_t; /* millisecond time type. */
 #define REDIS_DEFAULT_REPL_DISABLE_TCP_NODELAY 0
 #define REDIS_DEFAULT_MAXMEMORY 0
 #define REDIS_DEFAULT_MAXMEMORY_SAMPLES 5
+#define REDIS_DEFAULT_MAXMEMORY_FRAG_GUESS 1.4
+#define REDIS_DEFAULT_RSS_AWARE_MAXMEMORY 0
 #define REDIS_DEFAULT_AOF_FILENAME "appendonly.aof"
 #define REDIS_DEFAULT_AOF_NO_FSYNC_ON_REWRITE 0
 #define REDIS_DEFAULT_AOF_LOAD_TRUNCATED 1
@@ -839,6 +841,10 @@ struct redisServer {
     unsigned long long maxmemory;   /* Max number of memory bytes to use */
     int maxmemory_policy;           /* Policy for key eviction */
     int maxmemory_samples;          /* Pricision of random sampling */
+    /* RSS aware maxmemory additional state. */
+    int rss_aware_maxmemory;                /* Non zero if enabled. */
+    unsigned long long maxmemory_enforced;  /* Currently enforced maxmemory. */
+    float maxmemory_frag_guess;             /* Guessed fragmentation. */
     /* Blocked clients */
     unsigned int bpop_blocked_clients; /* Number of clients blocked by lists */
     list *unblocked_clients; /* list of clients to unblock before next loop */