redis-cli does no longer try to auto detect if it is used inside a pipe. To read last argument from stdandard input there is to use the -x option. This will make it playing better inside cron scripts and in general when stdin is hacked.

When creating the readable event results in an error (this happens when
the server hits OS limits), the client was not added to the list of
clients when freeClient was called. This results in an assertion error.
It is better to check this condition first and free the client
immediately when this condition occurs. Port of 00a90feb.

.gitignore

@@ -6,8 +6,13 @@ redis-cli
 redis-server
 redis-benchmark
 redis-check-dump
+redis-check-aof
 doc-tools
 mkrelease.sh
 release
 myredis.conf
 misc/*
+src/release.h
+appendonly.aof
+SHORT_TERM_TODO
+redis.conf.*

BETATESTING.txt

@@ -1,9 +0,0 @@
-This is a stable release, for beta testing make sure to download the latest source code from Git:
-
-    git clone git://github.com/antirez/redis.git
-
-It's also possibe to download the latest source code as a tarball:
-
-    http://github.com/antirez/redis/tree/master
-
-(use the download button)

INSTALL

@@ -0,0 +1,19 @@
+To compile Redis, do the following:
+
+    cd src; make
+
+The compilation will produce a redis-server binary.
+Copy this file where you want.
+
+Run the server using the following command line:
+
+    /path/to/redis-server
+
+This will start a Redis server with the default configuration.
+
+Otherwise if you want to provide your configuration use:
+
+    /path/to/redis-server /path/to/redis.conf
+
+You can find an example redis.conf file in the root directory
+of this source distribution.

Makefile

@@ -1,103 +1,14 @@
-# Redis Makefile
-# Copyright (C) 2009 Salvatore Sanfilippo <antirez at gmail dot com>
-# This file is released under the BSD license, see the COPYING file
+# Top level makefile, the real shit is at src/Makefile
 
-uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
-OPTIMIZATION?=-O2
-ifeq ($(uname_S),SunOS)
-  CFLAGS?= -std=c99 -pedantic $(OPTIMIZATION) -Wall -W -D__EXTENSIONS__ -D_XPG6
-  CCLINK?= -ldl -lnsl -lsocket -lm -lpthread
-else
-  CFLAGS?= -std=c99 -pedantic $(OPTIMIZATION) -Wall -W $(ARCH) $(PROF)
-  CCLINK?= -lm -pthread
-endif
-CCOPT= $(CFLAGS) $(CCLINK) $(ARCH) $(PROF)
-DEBUG?= -g -rdynamic -ggdb 
+TARGETS=32bit noopt test
 
-OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o
-BENCHOBJ = ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o
-CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o
-CHECKDUMPOBJ = redis-check-dump.o lzf_c.o lzf_d.o
+all:
+	cd src && $(MAKE) $@
 
-PRGNAME = redis-server
-BENCHPRGNAME = redis-benchmark
-CLIPRGNAME = redis-cli
-CHECKDUMPPRGNAME = redis-check-dump
+install: dummy
+	cd src && $(MAKE) $@
 
-all: redis-server redis-benchmark redis-cli redis-check-dump
+$(TARGETS) clean:
+	cd src && $(MAKE) $@
 
-# Deps (use make dep to generate this)
-adlist.o: adlist.c adlist.h zmalloc.h
-ae.o: ae.c ae.h zmalloc.h config.h ae_kqueue.c
-ae_epoll.o: ae_epoll.c
-ae_kqueue.o: ae_kqueue.c
-ae_select.o: ae_select.c
-anet.o: anet.c fmacros.h anet.h
-dict.o: dict.c fmacros.h dict.h zmalloc.h
-lzf_c.o: lzf_c.c lzfP.h
-lzf_d.o: lzf_d.c lzfP.h
-pqsort.o: pqsort.c
-redis-benchmark.o: redis-benchmark.c fmacros.h ae.h anet.h sds.h adlist.h \
-  zmalloc.h
-redis-cli.o: redis-cli.c fmacros.h anet.h sds.h adlist.h zmalloc.h
-redis.o: redis.c fmacros.h config.h redis.h ae.h sds.h anet.h dict.h \
-  adlist.h zmalloc.h lzf.h pqsort.h zipmap.h staticsymbols.h
-sds.o: sds.c sds.h zmalloc.h
-zipmap.o: zipmap.c zmalloc.h
-zmalloc.o: zmalloc.c config.h
-
-redis-server: $(OBJ)
-	$(CC) -o $(PRGNAME) $(CCOPT) $(DEBUG) $(OBJ)
-	@echo ""
-	@echo "Hint: To run the test-redis.tcl script is a good idea."
-	@echo "Launch the redis server with ./redis-server, then in another"
-	@echo "terminal window enter this directory and run 'make test'."
-	@echo ""
-
-redis-benchmark: $(BENCHOBJ)
-	$(CC) -o $(BENCHPRGNAME) $(CCOPT) $(DEBUG) $(BENCHOBJ)
-
-redis-cli: $(CLIOBJ)
-	$(CC) -o $(CLIPRGNAME) $(CCOPT) $(DEBUG) $(CLIOBJ)
-
-redis-check-dump: $(CHECKDUMPOBJ)
-	$(CC) -o $(CHECKDUMPPRGNAME) $(CCOPT) $(DEBUG) $(CHECKDUMPOBJ)
-
-.c.o:
-	$(CC) -c $(CFLAGS) $(DEBUG) $(COMPILE_TIME) $<
-
-clean:
-	rm -rf $(PRGNAME) $(BENCHPRGNAME) $(CLIPRGNAME) $(CHECKDUMPPRGNAME) *.o *.gcda *.gcno *.gcov
-
-dep:
-	$(CC) -MM *.c
-
-staticsymbols:
-	tclsh utils/build-static-symbols.tcl > staticsymbols.h
-
-test:
-	tclsh test-redis.tcl
-
-bench:
-	./redis-benchmark
-
-log:
-	git log '--pretty=format:%ad %s' --date=short > Changelog
-
-32bit:
-	@echo ""
-	@echo "WARNING: if it fails under Linux you probably need to install libc6-dev-i386"
-	@echo ""
-	make ARCH="-m32"
-
-gprof:
-	make PROF="-pg"
-
-gcov:
-	make PROF="-fprofile-arcs -ftest-coverage"
-
-noopt:
-	make OPTIMIZATION=""
-
-32bitgprof:
-	make PROF="-pg" ARCH="-arch i386"
+dummy:

README

@@ -1 +1,68 @@
-Check the 'doc' directory. doc/README.html is a good starting point :)
+Where to find complete Redis documentation?
+-------------------------------------------
+
+This README is just a fast "quick start" document. You can find more detailed
+documentation here:
+
+1) http://code.google.com/p/redis
+2) Check the 'doc' directory. doc/README.html is a good starting point :)
+
+Building Redis
+--------------
+
+It is as simple as:
+
+    % make
+
+Redis is just a single binary, but if you want to install it you can use
+the "make install" target that will copy the binary in /usr/local/bin
+for default.
+
+You can run a 32 bit Redis binary using:
+
+    % make 32bit
+
+After you build Redis is a good idea to test it, using:
+
+    % make test
+
+Running Redis
+-------------
+
+To run Redis with the default configuration just type:
+
+    % cd src
+    % ./redis-server
+    
+If you want to provide your redis.conf, you have to run it using an additional
+parameter (the path of the configuration file):
+
+    % cd src
+    % ./redis-server /path/to/redis.conf
+
+Playing with Redis
+------------------
+
+You can use redis-cli to play with Redis. Start a redis-server instance,
+then in another terminal try the following:
+
+    % cd src
+    % ./redis-cli
+    redis> ping
+    PONG
+    redis> set foo bar
+    OK
+    redis> get foo
+    "bar"
+    redis> incr mycounter
+    (integer) 1
+    redis> incr mycounter
+    (integer) 2
+    redis> 
+
+You can find the list of all the available commands here:
+
+    http://code.google.com/p/redis/wiki/CommandReference
+
+Enjoy!
+

TODO

@@ -1,56 +1,31 @@
 Redis TODO and Roadmap
+----------------------
 
-VERSION 2.0 TODO
-================
-
-* BRPOPLPUSH
-* List ops like L/RPUSH L/RPOP should return the new list length.
-* Save dataset / fsync() on SIGTERM
-* MULTI/EXEC should support the "EXEC FSYNC" form?
-* BLPOP & C. tests (write a non blocking Tcl client as first step)
-* Once ZRANK is implemented, change the implementation of ZCOUNT to use the augmented skiplist in order to be much faster.
-* Write doc for ZCOUNT, and for open / closed intervals of sorted sets range operations.
-
-Virtual Memory sub-TODO:
-* Check if the page selection algorithm is working well
-* Divide swappability of objects by refcount
-* Use multiple open FDs against the VM file, one for thread.
-* it should be possible to give the vm-max-memory option in megabyte, gigabyte, ..., just using 2GB, 100MB, and so forth.
-* Try to understand what can be moved into I/O threads that currently is instead handled by the main thread. For instance swapping file table scannig to find contiguous page could be a potential candidate (but I'm not convinced it's a good idea, better to improve the algorithm, for instance double the fast forward at every step?).
-* Possibly decrRefCount() against swapped objects can be moved into I/O threads, as it's a slow operation against million elements list, and in general consumes CPU time that can be consumed by other threads (and cores).
-* EXISTS should avoid loading the object if possible without too make the code too specialized.
-* vm-min-age <seconds> option
-* Make sure objects loaded from the VM are specially encoded when possible.
-* Check what happens performance-wise if instead to create threads again and again the same threads are reused forever. Note: this requires a way to disable this clients in the child, but waiting for empty new jobs queue can be enough.
-* Sets of integers are slow to load, for a number of reasons. Fix it. (use slow_sets.rdb file for debugging). (p.s. this was now partially fixed).
-* On EXEC try to block the client until relevant keys are loaded.
-
-* Hashes (GET/SET/DEL/INCRBY/EXISTS/FIELDS/LEN/MSET/MGET). Special encoding for hashes with less than N elements.
-* Write documentation for APPEND
-* Implement LEN, PEEK, POKE, SETBIT, GETBIT
-
-VERSION 2.2 TODO (Fault tolerant sharding)
-===========================================
-
-* Redis-cluster, a fast intermediate layer (proxy) that implements consistent hashing and fault tollerant nodes handling.
-
-Interesting readings about this:
-
-    - http://ayende.com/Blog/archive/2009/04/06/designing-rhino-dht-a-fault-tolerant-dynamically-distributed-hash.aspx
-
-VERSION 2.4 TODO (Optimizations and latency)
+VERSION 2.2 TODO (Optimizations and latency)
 ============================================
 
-* Lower the CPU usage.
-* Lower the RAM usage everywhere possible.
-* Use epool and alike to rewrite ae.c for Linux and other platforms suppporting fater-than-select() mutiplexing APIs.
-* Implement an UDP interface for low-latency GET/SET operations.
+* Support for syslog(3).
+* Change the implementation of ZCOUNT to use the augmented skiplist in order to be much faster.
+* Add an explicit test for MULTI/EXEC reloaded in the AOF.
+* Command table -> hash table, with support for command renaming
+
+VM TODO
+=======
+
+* Use multiple open FDs against the VM file, one for thread.
+* Check what happens performance-wise if instead of creating threads again and again the same threads are reused forever. Note: this requires a way to disable this clients in the child, but waiting for empty new jobs queue can be enough.
+
+STRING COMMANDS
+===============
+
+* Implement STRLEN, PEEK, POKE, SETBIT, GETBIT
 
 OTHER IMPORTANT THINGS THAT WILL BE ADDED BUT I'M NOT SURE WHEN
 ===============================================================
 
 BIG ONES:
 
+* BRPOPLPUSH
 * Specially encoded memory-saving integer sets.
 * A command to export a JSON dump (there should be mostly working patch needing major reworking).
 * Specially encoded sets of integers (this includes a big refactoring providing an higher level layer for Sets manipulation)
@@ -64,6 +39,7 @@ SMALL ONES:
 * Don't save empty lists / sets / zsets on disk with snapshotting.
 * Remove keys when a list / set / zset reaches length of 0.
 * An option to exec a command slave-side if the master connection is lost: even cooler: if the script returns "0" the slave elects itself as master, otherwise continue trying to reconnect.
+* PING the master from time to time to check if it's gone.
 
 THE "MAYBE" TODO LIST: things that may or may not get implemented
 =================================================================
@@ -71,18 +47,20 @@ THE "MAYBE" TODO LIST: things that may or may not get implemented
 Most of this can be seen just as proposals, the fact they are in this list
 it's not a guarantee they'll ever get implemented ;)
 
-* Move dict.c from hash table to skip list, in order to avoid the blocking resize operation needed for the hash table.
-* FORK command (fork()s executing the commands received by the current
-  client in the new process). Hint: large SORTs can use more cores,
-  copy-on-write will avoid memory problems.
-* DUP command? DUP srckey dstkey, creates an exact clone of srckey value in dstkey.
 * SORT: Don't copy the list into a vector when BY argument is constant.
 * Write the hash table size of every db in the dump, so that Redis can resize the hash table just one time when loading a big DB.
-* LOCK / TRYLOCK / UNLOCK as described many times in the google group
-* Replication automated tests
 * Byte Array type (BA prefixed commands): BASETBIT BAGETBIT BASETU8 U16 U32 U64 S8 S16 S32 S64, ability to atomically INCRBY all the base types. BARANGE to get a range of bytes as a bulk value, BASETRANGE to set a range of bytes.
-* zmalloc() should avoid to add a private header for archs where there is some other kind of libc-specific way to get the size of a malloced block. Already done for Mac OS X.
 * Read-only mode.
-* Pattern-matching replication.
-* Add an option to relax the delete-expiring-keys-on-write semantic *denying* replication and AOF when this is on? Can be handy sometimes, when using Redis for non persistent state, but can create problems. For instance should rename and move also "move" the timeouts? How does this affect other commands?
+* Kill the delete-on-write behavior of expires, replicating DELs
 * Multiple BY in SORT.
+
+KNOWN BUGS
+==========
+
+* LRANGE and other commands are using 32 bit integers for ranges, and overflows are not detected. So LRANGE mylist 0 23498204823094823904823904 will have random effects.
+
+REDIS CLI TODO
+==============
+
+* Computer parsable output generation
+* Memoize return values so that they can be used later as arguments, like $1

design-documents/REDIS-CLUSTER

@@ -1,29 +1,101 @@
 Redis Cluster Design Proposal (work in progress)
 
+28 Nov 2010: Ver 1.0 - initial version
+22 APr 2010: Ver 1.1 - more details and rationales
+
+Overview
+========
+
+Redis is a fast key-value store supporting complex aggregate data types as
+values. For instance keys can be bound to lists with many elements, sets,
+sub-dictionaries (hashes) and so forth.
+
+While Redis is very fast, currently it lacks scalability in the form of ability
+to transparently run across different nodes. This is desirable mainly for the
+following three rasons:
+
+A) Fault tolerance. Some node may go off line without affecting the operations.
+B) Holding bigger datasets without using a single box with a lot of RAM.
+C) Scaling writes.
+
+Since a single Redis instance supports 140,000 operations per second in a good
+Linux box costing less than $1000, the need for Redis Cluster arises more
+from "A" and "B". Scaling writes can also be useful in very high load
+environments. Scaling reads is already easily accomplished using Redis built-in
+replication.
+
+Design goals
+============
+
+Designing a DHT in 2010 is hard as there is too much bias towards good designs
+that are already well tested in practice, like the Amazon Dynamo design.
+Still a Dynamo alike DHT may not be the best fit for Redis.
+
+Redis is very simple and fast at its core, so Redis cluster should try to
+follow the same guidelines. The first problem with a Dynamo-alike DHT is that
+Redis supports complex data types. Merging complex values like lsits, where
+in the case of a netsplit may diverge in very complex ways, is not going to
+be easy. The "most recent data" wins is not applicable and all the resolution
+business should be in the application.
+
+Even a simple application can end up with complex schema of keys and complex
+values. Writing code in order to resolve conflicts is not going to be
+programmer friendly.
+
+So the author of this document claims that Redis does not need to resist to
+netsplits, but it is enough to resist to M-1 nodes going offline, where
+M is the number of nodes storing every key-value pair.
+
+For instance in a three nodes cluster I may configure the cluster in order to
+store every key into two instances (M=2). Such a cluster can resist to a single
+node going offline without interruption of the service.
+
+When more than M-1 nodes are off line the cluster should detect such a condition
+and refusing any further query. The system administrator should check why
+M-1 nodes are offline and bring them back again if possible.
+
+Once resisting to big net splits is no longer a requirement as there is no
+conflict resolution stage, since at least an original node responsible of
+holding every possible key must be online for the cluster to work, there is
+also no need for a design where every node can act as an independent entity
+receiving queries and forwarding this queries to other nodes as needed.
+
+Instead a more decoupled approach can be used, in the form of a Redis Proxy
+node (or multiple Proxy nodes) that is contacted by clients, and
+is responsible of forwarding queries and replies back and forth from data nodes.
+
+Data nodes can be just vanilla redis-server instances.
+
 Network layout
 ==============
 
-    - N different Data Nodes. Every node is identified by ip:port.
-    - A single Configuration Node.
-    - M different Proxy Nodes (redis-cluster).
-    - A single Handling Node.
+ - One ore more Data Nodes. Every node is identified by ip:port.
+ - A single Configuration Node.
+ - One more more Proxy Nodes (redis-cluster nodes).
+ - A single Handling Node.
+
+Data Nodes and the Configuration Node are just vanilla redis-server instances.
 
 Configuration Node
 ==================
 
-    - Contains information about all the Data nodes in the cluster.
-    - Contains information about all the Proxy nodes in the cluster.
-    - Maps the keyspace to different nodes.
+ - Contains information about all the Data nodes in the cluster.
+ - Contains information about all the Proxy nodes in the cluster.
+ - Contains information about what Data Node holds a given sub-space of keys.
 
 The keyspace is divided into 1024 different "hashing slots".
+(1024 is just an example, this value should be configurable)
 
-Given a key perform SHA1(key) and use the last 10 bits of the result to get a 10 bit number representing the key slot (from 0 to 1023).
+Given a key perform SHA1(key) and use the last 10 bits of the result to get a 10 bit number representing the "key slot" (from 0 to 1023).
 
-The Configuration node maps every slot of the keyspace to K different Data Nodes.
+The Configuration node maps every slot of the keyspace to M different Data Nodes (every key is stored into M nodes, configurable).
 
 The Configuration node can be modified by a single client at a time. Locking is performed using SETNX.
 
-The Configuration node should be replicated as there is a single configuration node for the whole network.
+The Configuration node should be replicated as there is a single configuration node for the whole network. It is the only single point of failure of the system.
+When a Configuration node fails the cluster does not stop operating, but is not
+able to recover if there is some exceptional condition to handle, like a Data
+Node going off line or the addition of a new Data Node to the cluster.
 
 The Configuration node is a standard Redis server, like every other Data node.
 
@@ -37,23 +109,52 @@ Proxy Nodes
 
 Proxy nodes get requests from clients and route this requests to the right Redis nodes.
 
-When a proxy node is started it needs to know the Configuration node address in order to load the infomration about the Data nodes and the mapping between the key space and the nodes.
+Proxy nodes take persistent connections to all the Data Nodes and the
+Configuration Node. This connections are keep alive with PING requests from time
+to time if there is no traffic. This way Proxy Nodes can understand asap if
+there is a problem in some Data Node or in the Configuration Node.
 
-On startup a Proxy node will also register itself in the Configuration node, and will make sure to refresh it's configuration every N seconds (via an EXPIREing key) so that it's possible to detect when a Proxy node fails.
+When a Proxy Node is started it needs to know the Configuration node address in order to load the infomration about the Data nodes and the mapping between the key space and the nodes.
 
-The Proxy node also is in charge of signaling failing Data nodes to the Configuration node, so that the Handling node can take appropriate actions.
+On startup a Proxy Node will also register itself in the Configuration node, and will make sure to refresh it's configuration every N seconds (via an EXPIREing key) so that it's possible to detect when a Proxy node fails.
+
+Clients can submit queries to any Proxy Node, so well designed clients may ask
+at startup the list of Proxy Nodes querying the Configuration Node. Then if
+a query fails against a given Proxy Node it can be retried against the next.
+
+The Proxy Node is also in charge of signaling failing Data nodes to the Configuration node, so that the Handling Node can take appropriate actions.
 
 When a new Data node joins or leaves the cluster, and in general when the cluster configuration changes, all the Proxy nodes will receive a notification and will reload the configuration from the Configuration node.
 
+Proxy Nodes - how queries are submited
+======================================
+
+This is how a query is processed:
+
+1) A client sends a query to a Proxy Node, using the Redis protocol like if it was a plain Redis Node.
+2) The Proxy Node inspects the command arguments to detect the key. The key is hashed. The Proxy Node has the table mapping a given key to M nodes, and persistent connections to all the nodes.
+
+At this point the process is different in case of read or write queries:
+
+WRITE QUERY:
+
+3a) The Proxy Node forwards the query to M Data Nodes at the same time, waiting for replies.
+3b) Once all the replies are received the Proxy Node checks that the replies are consistent. For instance all the M nodes need to reply with OK and so forth. If the query fails in a subset of nodes but succeeds in other nodes, the failing nodes are considered unreliable and are put off line notifying the configuration node.
+3c) The reply is transfered back to the client.
+
+READ QUERY:
+
+3d) The Proxy Node forwards the query to a single random client, passing the reply back to the client.
+
 Handling Node
 =============
 
 The handling node is a special Redis client with the following role:
 
-    - Handles the cluster configuration stored in the Config node.
-    - Is in charge for adding and removing nodes dynamically from the net.
-    - Relocates keys on nodes additions / removal.
-    - Signal a configuration change to Proxy nodes.
+ - Handles the cluster configuration stored in the Config node.
+ - Is in charge for adding and removing nodes dynamically from the net.
+ - Relocates keys on nodes additions / removal.
+ - Signal a configuration change to Proxy nodes.
 
 More details on hashing slots
 ============================
@@ -110,6 +211,4 @@ Then the slot is moved (1/1024 of all the keys). The Configuration server is mod
 Implementation details
 ======================
 
-Every Proxy node should take persistent connections to all the Data nodes.
-
 To run the Handling node and the Configuration node in the same physical computer is probably a good idea.

design-documents/REDIS-CLUSTER-2

@@ -0,0 +1,343 @@
+Redis Cluster - Alternative 1
+
+28 Apr 2010: Ver 1.0 - initial version
+
+Overview
+========
+
+The motivations and design goals of Redis Cluster are already outlined in the
+first design document of Redis Cluster. This document is just an attempt to
+provide a completely alternative approach in order to explore more ideas.
+
+In this document the alternative explored is a cluster where communication is
+performed directly from client to the target node, without intermediate layer.
+
+The intermediate layer can be used, in the form of a proxy, in order to provide
+the same functionality to clients not able to directly use the cluster protocol.
+So in a first stage clients can use a proxy to implement the hash ring, but
+later this clients can switch to a native implementation, following a
+specification that the Redis project will provide.
+
+In this new design fault tolerance is achieved by replicating M-1 times every
+data node instead of storing the same key M times across nodes.
+
+From the point of view of CAP our biggest sacrifice is about "P", that is
+resistance to partitioning. Only M-1 nodes can go down for the cluster still
+be functional. Also when possible "A" is somewhat sacrificed for "L", that
+is, Latency. Not really in the CAP equation but a very important parameter.
+
+Network layout
+==============
+
+In this alternative design the network layout is simple as there are only
+clients talking directly to N data nodes. So we can imagine to have:
+
+- K Redis clients, directly talking to the data nodes.
+- N Redis data nodes, that are, normal Redis instances.
+
+Data nodes are replicate M-1 times (so there are a total of M copies for
+every node). If M is one, the system is not fault tolerant. If M is 2 one
+data node can go off line without affecting the operations. And so forth.
+
+Hash slots
+==========
+
+The key space is divided into 1024 slots.
+
+Given a key, the SHA1 function is applied to it.
+The first 10 bytes of the SHA1 digest are interpreted as an unsigned integer
+from 0 to 1023. This is the hash slot of the key.
+
+Data nodes
+==========
+
+Data nodes are normal Redis instances, but a few additional commands are
+provided.
+
+HASHRING ADD ... list of hash slots ...
+HASHRING DEL ... list of hash slots ...
+HASHRING REHASHING slot
+HASHRING SLOTS => returns the list of configured slots
+HSAHRING KEYS ... list of hash slots ...
+
+By default Redis instances are configured to accept operations about all
+the hash slots. With this commands it's possible to configure a Redis instance
+to accept only a subset of the key space.
+
+If an operation is performed against a key hashing to a slot that is not
+configured to be accepted, the Redis instance will reply with:
+
+  "-ERR wrong hash slot"
+
+More details on the HASHRING command and sub commands will be showed later
+in this document.
+
+Additionally three other commands are added:
+
+DUMP key
+RESTORE key <dump data>
+MIGRATE key host port
+
+DUMP is used to output a very compact binary representation of the data stored at key.
+
+RESTORE re-creates a value (storing it at key) starting from the output produced by DUMP.
+
+MIGRATE is like a server-side DUMP+RESTORE command. This atomic command moves one key from the connected instance to another instance, returning the status code of the operation (+OK or an error).
+
+The protocol described in this draft only uses the MIGRATE command, but this in turn will use RESTORE internally when connecting to another server, and DUMP is provided for symmetry.
+
+Querying the cluster
+====================
+
+1) Reading the cluster config
+-----------------------------
+
+Clients of the cluster are required to have the cluster configuration loaded
+into memory. The cluster configuration is the sum of the following info:
+
+- Number of data nodes in the cluster, for instance, 10
+- A map between hash slots and nodes, so for instnace:
+  hash slot 1 -> node 0
+  hash slot 2 -> node 5
+  hash slot 3 -> node 3
+  ... and so forth ...
+- Physical address of nodes, and their replicas.
+  node 0 addr -> 192.168.1.100
+  node 0 replicas -> 192.168.1.101, 192.168.1.105
+- Configuration version: the SHA1 of the whole configuration
+
+The configuration is stored in every single data node of the cluster.
+
+A client without the configuration in memory is require, as a first step, to
+read the config. In order to do so the client requires to have a list of IPs
+that are with good probability data nodes of the cluster.
+
+The client will try to get the config from all this nodes. If no node is found
+responding, an error is reported to the user.
+
+2) Caching and refreshing the configuration
+-------------------------------------------
+
+A node is allowed to cache the configuration in memory or in a different way
+(for instance storing the configuration into a file), but every client is
+required to check if the configuration changed at max every 10 seconds, asking
+for the configuration version key with a single GET call, and checking if the
+configuration version matches the one loaded in memory.
+
+Also a client is required to refresh the configuration every time a node
+replies with:
+
+  "-ERR wrong hash slot"
+
+As this means that hash slots were reassigned in some way.
+
+Checking the configuration every 10 seconds is not required in theory but is
+a good protection against errors and failures that may happen in real world
+environments. It is also very cheap to perform, as a GET operation from time
+to time is going to have no impact in the overall performance.
+
+3) Read query
+-------------
+
+To perform a read query the client hashes the key argument from the command
+(in the intiial version of Redis Cluster only single-key commands are
+allowed). Using the in memory configuration it maps the hash key to the
+node ID.
+
+If the client is configured to support read-after-write consistency, then
+the "master" node for this hash slot is queried.
+
+Otherwise the client picks a random node from the master and the replicas
+available.
+
+4) Write query
+--------------
+
+A write query is exactly like a read query, with the difference that the
+write always targets the master node, instead of the replicas.
+
+Creating a cluster
+==================
+
+In order to create a new cluster, the redis-cluster command line utility is
+used. It gets a list of available nodes and replicas, in order to write the
+initial configuration in all the nodes.
+
+At this point the cluster is usable by clients.
+
+Adding nodes to the cluster
+===========================
+
+The command line utility redis-cluster is used in order to add a node to the
+cluster:
+
+1) The cluster configuration is loaded.
+2) A fair number of hash slots are assigned to the new data node.
+3) Hash slots moved to the new node are marked as "REHASHING" in the old
+   nodes, using the HASHRING command:
+
+    HASHRING SETREHASHING 1 192.168.1.103 6380
+
+The above command set the hash slot "1" in rehashing state, with the
+"forwarding address" to 192.168.1.103:6380. As a result if this node receives
+a query about a key hashing to hash slot 1, that *is not present* in the
+current data set, it replies with:
+
+    "-MIGRATED 192.168.1.103:6380"
+
+The client can then reissue the query against the new node.
+
+Instead even if the hash slot is marked as rehashing but the requested key
+is still there, the query is processed. This allows for non blocking
+rehashing.
+
+Note that no additional memory is used by Redis in order to provide such a
+feature.
+
+4) While the Hash slot is marked as "REHASHING", redis-cluster asks this node
+the list of all the keys matching the specified hash slot. Then all the keys
+are moved to the new node using the MIGRATE command.
+5) Once all the keys are migrated, the hash slot is deleted from the old
+node configuration with "HASHRING DEL 1". And the configuration is update.
+
+Using this algorithm all the hash slots are migrated one after the other to the new node. In practical implementation before to start the migration the
+redis-cluster utility should write a log into the configuration so that
+in case of crash or any other problem the utility is able to recover from
+were it left.
+
+Fault tolerance
+===============
+
+Fault tolerance is reached replicating every data node M-1 times, so that we
+have one master and M-1 replicas for a total of M nodes holding the same
+hash slots. Up to M-1 nodes can go down without affecting the cluster.
+
+The tricky part about fault tolerance is detecting when a node is failing and
+signaling it to all the other clients.
+
+When a master node is failing in a permanent way, promoting the first slave
+is easy:
+1) At some point a client will notice there are problems accessing a given node. It will try to refresh the config, but will notice that the config is already up to date.
+2) In order to make sure the problem is not about the client connectivity itself, it will try to reach other nodes as well. If more than M-1 nodes appear to be down, it's either a client networking problem or alternatively the cluster can't be fixed as too many nodes are down anyway. So no action is taken, but an error is reported.
+3) If instead only 1 or at max M-1 nodes appear to be down, the client promotes a slave as master and writes the new configuration to all the data nodes.
+
+All the other clients will see the data node not working, and as a first step will try to refresh the configuration. They will successful refresh the configuration and the cluster will work again.
+
+Every time a slave is promoted, the information is written in a log that is actually a Redis list, in all the data nodes, so that system administration tools can detect what happened in order to send notifications to the admin.
+
+Intermittent problems
+---------------------
+
+In the above scenario a master was failing in a permanent way. Now instead
+let's think to a case where a network cable is not working well so a node
+appears to be a few seconds up and a few seconds down.
+
+When this happens recovering can be much harder, as a client may notice the
+problem and will promote a slave to master as a result, but then the host
+will be up again and the other clients will not see the problem, writing to
+the old master for at max 10 seconds (after 10 seconds all the clients are
+required to perform a few GETs to check the configuration version of the
+cluster and update if needed).
+
+One way to fix this problem is to delegate the fail over mechanism to a
+failover agent. When clients notice problems will not take any active action
+but will just log the problem into a redis list in all the reachable nodes,
+wait, check for configuration change, and retry.
+
+The failover agent constantly monitor this logs: if some client is reporting
+a failing node, it can take appropriate actions, checking if the failure is
+permanent or not. If it's not he can send a SHUTDOWN command to the failing
+master if possible. The failover agent can also consider better the problem
+checking if the failing mode is advertised by all the clients or just a single
+one, and can check itself if there is a real problem before to proceed with
+the fail over.
+
+Redis proxy
+===========
+
+In order to make the switch to the clustered version of Redis simpler, and
+because the client-side protocol is non trivial to implement compared to the
+usual Redis client lib protocol (where a minimal lib can be as small as
+100 lines of code), a proxy will be provided to implement the cluster protocol
+as a proxy.
+
+Every client will talk to a redis-proxy node that is responsible of using
+the new protocol and forwarding back the replies.
+
+In the long run the aim is to switch all the major client libraries to the
+new protocol in a native way.
+
+Supported commands
+==================
+
+Because with this design we talk directly to data nodes and there is a single
+"master" version of every value (that's the big gain dropping "P" from CAP!)
+almost all the redis commands can be supported by the clustered version
+including MULTI/EXEC and multi key commands as long as all the keys will hash
+to the same hash slot. In order to guarantee this, key tags can be used,
+where when a specific pattern is present in the key name, only that part is
+hashed in order to obtain the hash index.
+
+Random remarks
+==============
+
+- It's still not clear how to perform an atomic election of a slave to master.
+- In normal conditions (all the nodes working) this new design is just
+  K clients talking to N nodes without intermediate layers, no routes:
+  this means it is horizontally scalable with O(1) lookups.
+- The cluster should optionally be able to work with manual fail over
+  for environments where it's desirable to do so. For instance it's possible
+  to setup periodic checks on all the nodes, and switch IPs when needed
+  or other advanced configurations that can not be the default as they
+  are too environment dependent.
+
+A few ideas about client-side slave election
+============================================
+
+Detecting failures in a collaborative way
+-----------------------------------------
+
+In order to take the node failure detection and slave election a distributed
+effort, without any "control program" that is in some way a single point
+of failure (the cluster will not stop when it stops, but errors are not
+corrected without it running), it's possible to use a few consensus-alike
+algorithms.
+
+For instance all the nodes may take a list of errors detected by clients.
+
+If Client-1 detects some failure accessing Node-3, for instance a connection
+refused error or a timeout, it logs what happened with LPUSH commands against
+all the other nodes. This "error messages" will have a timestamp and the Node
+id. Something like:
+
+    LPUSH __cluster__:errors 3:1272545939
+
+So if the error is reported many times in a small amount of time, at some
+point a client can have enough hints about the need of performing a
+slave election.
+
+Atomic slave election
+---------------------
+
+In order to avoid races when electing a slave to master (that is in order to
+avoid that some client can still contact the old master for that node in
+the 10 seconds timeframe), the client performing the election may write
+some hint in the configuration, change the configuration SHA1 accordingly and
+wait for more than 10 seconds, in order to be sure all the clients will
+refresh the configuration before a new access.
+
+The config hint may be something like:
+
+"we are switching to a new master, that is x.y.z.k:port, in a few seconds"
+
+When a client updates the config and finds such a flag set, it starts to
+continuously refresh the config until a change is noticed (this will take
+at max 10-15 seconds).
+
+The client performing the election will wait that famous 10 seconds time frame
+and finally will update the config in a definitive way setting the new
+slave as mater. All the clients at this point are guaranteed to have the new
+config either because they refreshed or because in the next query their config
+is already expired and they'll update the configuration.
+
+EOF

design-documents/VIRTUAL-MEMORY

@@ -1,11 +0,0 @@
-- Use N working childs (fork at startup) in order to implement async I/O.
-- The swap file is opened at startup and unlink(2)-ed
-- Swap file free/used blocks bitmap is taken in memory
-- When a child is saving on background or rewriting the append only log the swap file gets frozen (no writes from the parent).
-- When Redis is low on memory keys not recently used and big enough will be transfered on Disk by one of the child processes doing async I/O. Only when the transfer finishes the parent will mark the value as swapped out and will free the associated value (if didn't changed in the meantime).
-- When Redis is going to process a command will first check that all the keys involved are in memory. If not will send a request to an async I/O child in order to load this keys in memory. When the operation finished Redis will "resume" the client operation (just the client structure will hold the arguments of the suspended command, Redis will execute the command and unmask the read/write events in the client socket).
-- async I/O childs and parent communicate via pipes, so while Redis is blocked in the event loop can be resumed by an async child I/O just writing a message in the pipe.
-- Every Redis type should have a function to guess the max space needed to serialized an object.
-- The swap file is divided into blocks.
-- Even if Redis unblock the command when not all the keys are loaded, or if a key was swapped out in the mean time, this is not a critical condition as anyway the key lookup process will (this time synchronously) load the keys in memory if needed. This should happen very rarely or possibly never, so a bug in the async loading stage will not cause a bug but just a performance hit.
-- The blocks allocation algorithm should try to avoid fragmentation and cache misses.

doc/AppendCommand.html

@@ -0,0 +1,48 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>AppendCommand: Contents</b><br>&nbsp;&nbsp;<a href="#APPEND _key_ _value_">APPEND _key_ _value_</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Examples">Examples</a>
+                </div>
+                
+                <h1 class="wikiname">AppendCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;#sidebar <a href="StringCommandsSidebar.html">StringCommandsSidebar</a><h1><a name="APPEND _key_ _value_">APPEND _key_ _value_</a></h1>
+<i>Time complexity: O(1). The amortized time complexity is O(1) assuming the appended value is small and the already present value is of any size, since the dynamic string library used by Redis will double the free space available on every reallocation.</i><blockquote>If the <i>key</i> already exists and is a string, this command appends theprovided value at the end of the string.If the <i>key</i> does not exist it is created and set as an empty string, soAPPEND will be very similar to SET in this special case.</blockquote>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Integer reply</a>, specifically the total length of the string after the append operation.<h2><a name="Examples">Examples</a></h2><pre class="codeblock python" name="code">
+redis&gt; exists mykey
+(integer) 0
+redis&gt; append mykey &quot;Hello &quot;
+(integer) 6
+redis&gt; append mykey &quot;World&quot;
+(integer) 11
+redis&gt; get mykey
+&quot;Hello World&quot;
+</pre>
+
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/AppendOnlyFileHowto.html

@@ -16,7 +16,7 @@
             <div id="pagecontent">
                 <div class="index">
 <!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
-<b>AppendOnlyFileHowto: Contents</b><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#General Information">General Information</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Log rewriting">Log rewriting</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Wait... but how does this work?">Wait... but how does this work?</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#How durable is the append only file?">How durable is the append only file?</a>
+<b>AppendOnlyFileHowto: Contents</b><br>&nbsp;&nbsp;<a href="#Append Only File HOWTO">Append Only File HOWTO</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#General Information">General Information</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Log rewriting">Log rewriting</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Wait... but how does this work?">Wait... but how does this work?</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#How durable is the append only file?">How durable is the append only file?</a>
                 </div>
                 
                 <h1 class="wikiname">AppendOnlyFileHowto</h1>
@@ -26,11 +26,13 @@
                 </div>
 
                 <div class="narrow">
-                    &iuml;&raquo;&iquest;= Append Only File HOWTO =<h2><a name="General Information">General Information</a></h2>Append only file is an alternative durability option for Redis. What this mean? Let's start with some fact:<br/><br/><ul><li> For default Redis saves snapshots of the dataset on disk, in a binary file called dump.rdb (by default at least). For instance you can configure Redis to save the dataset every 60 seconds if there are at least 100 changes in the dataset, or every 1000 seconds if there is at least a single change in the dataset. This is known as &quot;Snapshotting&quot;.</li><li> Snapshotting is not very durable. If your computer running Redis stops, your power line fails, or you write killall -9 redis-server for a mistake, the latest data written on Redis will get lost. There are applications where this is not a big deal. There are applications where this is not acceptable and Redis <b>was</b> not an option for this applications.</li></ul>
+                    &iuml;&raquo;&iquest;#sidebar <a href="RedisGuides.html">RedisGuides</a>
+<h1><a name="Append Only File HOWTO">Append Only File HOWTO</a></h1><h2><a name="General Information">General Information</a></h2>Append only file is an alternative durability option for Redis. What this mean? Let's start with some fact:<br/><br/><ul><li> For default Redis saves snapshots of the dataset on disk, in a binary file called dump.rdb (by default at least). For instance you can configure Redis to save the dataset every 60 seconds if there are at least 100 changes in the dataset, or every 1000 seconds if there is at least a single change in the dataset. This is known as &quot;Snapshotting&quot;.</li><li> Snapshotting is not very durable. If your computer running Redis stops, your power line fails, or you write killall -9 redis-server for a mistake, the latest data written on Redis will get lost. There are applications where this is not a big deal. There are applications where this is not acceptable and Redis <b>was</b> not an option for this applications.</li></ul>
 What is the solution? To use append only file as alternative to snapshotting. How it works?<br/><br/><ul><li> It is an 1.1 only feature.</li><li> You have to turn it on editing the configuration file. Just make sure you have &quot;appendonly yes&quot; somewhere.</li><li> Append only files work this way: every time Redis receive a command that changes the dataset (for instance a SET or LPUSH command) it appends this command in the append only file. When you restart Redis it will first <b>re-play</b> the append only file to rebuild the state.</li></ul>
-<h2><a name="Log rewriting">Log rewriting</a></h2>As you can guess... the append log file gets bigger and bigger, every time there is a new operation changing the dataset. Even if you set always the same key &quot;mykey&quot; to the values of &quot;1&quot;, &quot;2&quot;, &quot;3&quot;, ... up to 10000000000 in the end you'll have just a single key in the dataset, just a few bytes! but how big will be the append log file? Very very big.<br/><br/>So Redis supports an interesting feature: it is able to rebuild the append log file, in background, without to stop processing client commands. The key is the command <a href="BGREWRITEAOF.html">BGREWRITEAOF</a>. This command basically is able to use the dataset in memory in order to rewrite the shortest sequence of commands able to rebuild the exact dataset that is currently in memory.<br/><br/>So from time to time when the log gets too big, try this command. It's safe as if it fails you will not lost your old log (but you can make a backup copy given that currently 1.1 is still in beta!).<h2><a name="Wait... but how does this work?">Wait... but how does this work?</a></h2>Basically it uses the same fork() copy-on-write trick that snapshotting already uses. This is how the algorithm works:<br/><br/><ul><li> Redis forks, so now we have a child and a parent.</li><li> The child starts writing the new append log file in a temporary file.</li><li> The parent accumulates all the new changes in an in-memory buffer.</li><li> When the child finished to rewrite the file, the parent gets a signal, and append the in-memory buffer at the end of the file generated by the child.</li><li> Profit! Now Redis atomically renames the old file into the new one, and starts appending new data into the new file.</li></ul>
+<h2><a name="Log rewriting">Log rewriting</a></h2>As you can guess... the append log file gets bigger and bigger, every time there is a new operation changing the dataset. Even if you set always the same key &quot;mykey&quot; to the values of &quot;1&quot;, &quot;2&quot;, &quot;3&quot;, ... up to 10000000000 in the end you'll have just a single key in the dataset, just a few bytes! but how big will be the append log file? Very very big.<br/><br/>So Redis supports an interesting feature: it is able to rebuild the append log file, in background, without to stop processing client commands. The key is the command <a href="BGREWRITEAOF.html">BGREWRITEAOF</a>. This command basically is able to use the dataset in memory in order to rewrite the shortest sequence of commands able to rebuild the exact dataset that is currently in memory.<br/><br/>So from time to time when the log gets too big, try this command. It's safe as if it fails you will not lost your old log (but you can make a backup copy given that currently 1.1 is still in beta!).<h2><a name="Wait... but how does this work?">Wait... but how does this work?</a></h2>Basically it uses the same fork() copy-on-write trick that snapshotting already uses. This is how the algorithm works:<br/><br/><ul><li> Redis forks, so now we have a child and a parent.</li><li> The child starts writing the new append log file in a temporary file.</li><li> The parent accumulates all the new changes in an in-memory buffer (but at the same time it writes the new changes in the <b>old</b> append only file, so if the rewriting fails, we are safe).</li><li> When the child finished to rewrite the file, the parent gets a signal, and append the in-memory buffer at the end of the file generated by the child.</li><li> Profit! Now Redis atomically renames the old file into the new one, and starts appending new data into the new file.</li></ul>
 <h2><a name="How durable is the append only file?">How durable is the append only file?</a></h2>Check redis.conf, you can configure how many times Redis will fsync() data on disk. There are three options:<br/><br/><ul><li> Fsync() every time a new command is appended to the append log file. Very very slow, very safe.</li><li> Fsync() one time every second. Fast enough, and you can lose 1 second of data if there is a disaster.</li><li> Never fsync(), just put your data in the hands of the Operating System. The faster and unsafer method.</li></ul>
 Warning: by default Redis will fsync() after <b>every command</b>! This is because the Redis authors want to ship a default configuration that is the safest pick. But the best compromise for most datasets is to fsync() one time every second.
+
                 </div>
         
             </div>

doc/BlpopCommand.html

@@ -0,0 +1,51 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>BlpopCommand: Contents</b><br>&nbsp;&nbsp;<a href="#BLPOP _key1_ _key2_ ... _keyN_ _timeout_ (Redis &gt;">BLPOP _key1_ _key2_ ... _keyN_ _timeout_ (Redis &gt;</a><br>&nbsp;&nbsp;<a href="#BRPOP _key1_ _key2_ ... _keyN_ _timeout_ (Redis &gt;">BRPOP _key1_ _key2_ ... _keyN_ _timeout_ (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Non blocking behavior">Non blocking behavior</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Blocking behavior">Blocking behavior</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Multiple clients blocking for the same keys">Multiple clients blocking for the same keys</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#blocking POP inside a MULTI/EXEC transaction">blocking POP inside a MULTI/EXEC transaction</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+                </div>
+                
+                <h1 class="wikiname">BlpopCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;#sidebar <a href="ListCommandsSidebar.html">ListCommandsSidebar</a><h1><a name="BLPOP _key1_ _key2_ ... _keyN_ _timeout_ (Redis &gt;">BLPOP _key1_ _key2_ ... _keyN_ _timeout_ (Redis &gt;</a></h1> 1.3.1) =
+<h1><a name="BRPOP _key1_ _key2_ ... _keyN_ _timeout_ (Redis &gt;">BRPOP _key1_ _key2_ ... _keyN_ _timeout_ (Redis &gt;</a></h1> 1.3.1) =
+<i>Time complexity: O(1)</i><blockquote>BLPOP (and BRPOP) is a blocking list pop primitive. You can see this commandsas blocking versions of <a href="LpopCommand.html">LPOP</a> and <a href="LpopCommand.html">RPOP</a> able toblock if the specified keys don't exist or contain empty lists.</blockquote>
+<blockquote>The following is a description of the exact semantic. We describe BLPOP butthe two commands are identical, the only difference is that BLPOP pops theelement from the left (head) of the list, and BRPOP pops from the right (tail).</blockquote>
+<h2><a name="Non blocking behavior">Non blocking behavior</a></h2><blockquote>When BLPOP is called, if at least one of the specified keys contain a nonempty list, an element is popped from the head of the list and returned tothe caller together with the name of the key (BLPOP returns a two elementsarray, the first element is the key, the second the popped value).</blockquote>
+<blockquote>Keys are scanned from left to right, so for instance if youissue <b>BLPOP list1 list2 list3 0</b> against a dataset where <b>list1</b> does notexist but <b>list2</b> and <b>list3</b> contain non empty lists, BLPOP guaranteesto return an element from the list stored at <b>list2</b> (since it is the firstnon empty list starting from the left).</blockquote>
+<h2><a name="Blocking behavior">Blocking behavior</a></h2><blockquote>If none of the specified keys exist or contain non empty lists, BLPOPblocks until some other client performs a <a href="RpushCommand.html">LPUSH</a> oran <a href="RpushCommand.html">RPUSH</a> operation against one of the lists.</blockquote>
+<blockquote>Once new data is present on one of the lists, the client finally returnswith the name of the key unblocking it and the popped value.</blockquote>
+<blockquote>When blocking, if a non-zero timeout is specified, the client will unblockreturning a nil special value if the specified amount of seconds passedwithout a push operation against at least one of the specified keys.</blockquote>
+<blockquote>The timeout argument is interpreted as an integer value. A timeout of zero means instead to block forever.</blockquote>
+<h2><a name="Multiple clients blocking for the same keys">Multiple clients blocking for the same keys</a></h2><blockquote>Multiple clients can block for the same key. They are put intoa queue, so the first to be served will be the one that started to waitearlier, in a first-blpopping first-served fashion.</blockquote>
+<h2><a name="blocking POP inside a MULTI/EXEC transaction">blocking POP inside a MULTI/EXEC transaction</a></h2><blockquote>BLPOP and BRPOP can be used with pipelining (sending multiple commands and reading the replies in batch), but it does not make sense to use BLPOP or BRPOP inside a MULTI/EXEC block (a Redis transaction).</blockquote>
+<blockquote>The behavior of BLPOP inside MULTI/EXEC when the list is empty is to return a multi-bulk nil reply, exactly what  happens when the timeout is reached. If you like science fiction, think at it like if inside MULTI/EXEC the time will  flow at infinite speed :) </blockquote>
+<h2><a name="Return value">Return value</a></h2><blockquote>BLPOP returns a two-elements array via a multi bulk reply in order to returnboth the unblocking key and the popped value.</blockquote>
+<blockquote>When a non-zero timeout is specified, and the BLPOP operation timed out,the return value is a nil multi bulk reply. Most client values will returnfalse or nil accordingly to the programming language used.</blockquote>
+<a href="ReplyTypes.html">Multi bulk reply</a>
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/ConfigCommand.html

@@ -0,0 +1,76 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>ConfigCommand: Contents</b><br>&nbsp;&nbsp;<a href="#CONFIG GET _pattern_ (Redis &gt;">CONFIG GET _pattern_ (Redis &gt;</a><br>&nbsp;&nbsp;<a href="#CONFIG SET _parameter_ _value_ (Redis &gt;">CONFIG SET _parameter_ _value_ (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#CONFIG GET _pattern_">CONFIG GET _pattern_</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#CONFIG SET _parameter_ _value_">CONFIG SET _parameter_ _value_</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Parameters value format">Parameters value format</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#See Also">See Also</a>
+                </div>
+                
+                <h1 class="wikiname">ConfigCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;#sidebar <a href="ControlCommandsSidebar.html">ControlCommandsSidebar</a><h1><a name="CONFIG GET _pattern_ (Redis &gt;">CONFIG GET _pattern_ (Redis &gt;</a></h1> 2.0)=
+<h1><a name="CONFIG SET _parameter_ _value_ (Redis &gt;">CONFIG SET _parameter_ _value_ (Redis &gt;</a></h1> 2.0)=<br/><br/><blockquote>The CONFIG command is able to retrieve or alter the configuration of a runningRedis server. Not all the configuration parameters are supported.</blockquote>
+<blockquote>CONFIG has two sub commands, GET and SET. The GET command is used to readthe configuration, while the SET command is used to alter the configuration.</blockquote>
+<h2><a name="CONFIG GET _pattern_">CONFIG GET _pattern_</a></h2><blockquote>CONFIG GET returns the current configuration parameters. This sub commandonly accepts a single argument, that is glob style pattern. All theconfiguration parameters matching this parameter are reported as alist of key-value pairs. Example:</blockquote><pre class="codeblock python" name="code">
+$ redis-cli config get '*'
+1. &quot;dbfilename&quot;
+2. &quot;dump.rdb&quot;
+3. &quot;requirepass&quot;
+4. (nil)
+5. &quot;masterauth&quot;
+6. (nil)
+7. &quot;maxmemory&quot;
+8. &quot;0\n&quot;
+9. &quot;appendfsync&quot;
+10. &quot;everysec&quot;
+11. &quot;save&quot;
+12. &quot;3600 1 300 100 60 10000&quot;
+
+$ redis-cli config get 'm*'
+1. &quot;masterauth&quot;
+2. (nil)
+3. &quot;maxmemory&quot;
+4. &quot;0\n&quot;
+</pre>The return type of the command is a <a href="ReplyTypes.html">bulk reply</a>.<h2><a name="CONFIG SET _parameter_ _value_">CONFIG SET _parameter_ _value_</a></h2><blockquote>CONFIG SET is used in order to reconfigure the server, setting a specificconfiguration parameter to a new value.</blockquote>
+<blockquote>The list of configuration parameters supported by CONFIG SET can beobtained issuing a <code name="code" class="python">CONFIG GET *</code> command.</blockquote>
+<blockquote>The configuration set using CONFIG SET is immediately loaded by the Redisserver that will start acting as specified starting from the next command.</blockquote>
+<blockquote>Example:</blockquote><pre class="codeblock python python" name="code">
+$ ./redis-cli 
+redis&gt; set x 10
+OK
+redis&gt; config set maxmemory 200
+OK
+redis&gt; set y 20
+(error) ERR command not allowed when used memory &gt; 'maxmemory'
+redis&gt; config set maxmemory 0
+OK
+redis&gt; set y 20
+OK
+</pre><h2><a name="Parameters value format">Parameters value format</a></h2><blockquote>The value of the configuration parameter is the same as the one of thesame parameter in the Redis configuration file, with the following exceptions:</blockquote>
+<ul><li> The <code name="code" class="python">save</code> paramter is a list of space-separated integers. Every pair of integers specify the time and number of changes limit to trigger a save. For instance the command <code name="code" class="python">CONFIG SET save &quot;3600 10 60 10000&quot;</code> will configure the server to issue a background saving of the RDB file every 3600 seconds if there are at least 10 changes in the dataset, and every 60 seconds if there are at least 10000 changes. To completely disable automatic snapshots just set the parameter as an empty string.</li><li> All the integer parameters representing memory are returned and accepted only using bytes as unit.</li></ul>
+<h2><a name="See Also">See Also</a></h2>The <a href="InfoCommand.html">INFO</a> command can be used in order to read configuriaton parameters that are not available in the CONFIG command.
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/DelCommand.html

@@ -27,12 +27,11 @@
 
                 <div class="narrow">
                     &iuml;&raquo;&iquest;#sidebar <a href="GenericCommandsSidebar.html">GenericCommandsSidebar</a><h1><a name="DEL _key1_ _key2_ ... _keyN_">DEL _key1_ _key2_ ... _keyN_</a></h1>
-<i>Time complexity: O(1)</i><blockquote>Remove the specified keys. If a given key does not existno operation is performed for this key. The commnad returns the number ofkeys removed.</blockquote>
+<i>Time complexity: O(1)</i><blockquote>Remove the specified keys. If a given key does not existno operation is performed for this key. The command returns the number ofkeys removed.</blockquote>
 <h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Integer reply</a>, specifically:<br/><br/><pre class="codeblock python" name="code">
 an integer greater than 0 if one or more keys were removed
 0 if none of the specified key existed
 </pre>
-
                 </div>
         
             </div>

doc/EventLibray.html

@@ -0,0 +1,44 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>EventLibray: Contents</b><br>&nbsp;&nbsp;<a href="#Event Library">Event Library</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Why is an Event Library needed at all?">Why is an Event Library needed at all?</a>
+                </div>
+                
+                <h1 class="wikiname">EventLibray</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    
+<h1><a name="Event Library">Event Library</a></h1><h2><a name="Why is an Event Library needed at all?">Why is an Event Library needed at all?</a></h2>Let us figure it out through a series of Q&amp;As.<br/><br/>Q: What do you expect a network server to be doing all the time? &lt;br/&gt;
+A: Watch for inbound connections on the port its listening and accept them.<br/><br/>Q: Calling <a href="http://man.cx/accept%282%29" target="_blank">accept</a> yields a descriptor. What do I do with it?&lt;br/&gt;
+A: Save the descriptor and do a non-blocking read/write operation on it.<br/><br/>Q: Why does the read/write have to be non-blocking?&lt;br/&gt;
+A: If the file operation ( even a socket in Unix is a file ) is blocking how could the server for example accept other connection requests when its blocked in a file I/O operation.<br/><br/>Q: I guess I have to do many such non-blocking operations on the socket to see when it's ready. Am I right?&lt;br/&gt;
+A: Yes. That is what an event library does for you. Now you get it.<br/><br/>Q: How do Event Libraries do what they do?&lt;br/&gt;
+A: They use the operating system's <a href="http://www.devshed.com/c/a/BrainDump/Linux-Files-and-the-Event-Poll-Interface/" target="_blank">polling</a> facility along with timers.<br/><br/>Q: So are there any open source event libraries that do what you just described? &lt;br/&gt;
+A: Yes. Libevent and Libev are two such event libraries that I can recall off the top of my head.<br/><br/>Q: Does Redis use such open source event libraries for handling socket I/O?&lt;br/&gt;
+A: No. For various <a href="http://groups.google.com/group/redis-db/browse_thread/thread/b52814e9ef15b8d0/" target="_blank">reasons</a> Redis uses its own event library.
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/ExpireCommand.html

@@ -16,7 +16,7 @@
             <div id="pagecontent">
                 <div class="index">
 <!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
-<b>ExpireCommand: Contents</b><br>&nbsp;&nbsp;<a href="#EXPIRE _key_ _seconds_">EXPIRE _key_ _seconds_</a><br>&nbsp;&nbsp;<a href="#EXPIREAT _key_ _unixtime_ (Redis &gt;">EXPIREAT _key_ _unixtime_ (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#How the expire is removed from a key">How the expire is removed from a key</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Restrictions with write operations against volatile keys">Restrictions with write operations against volatile keys</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Setting the timeout again on already volatile keys">Setting the timeout again on already volatile keys</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Enhanced Lazy Expiration algorithm">Enhanced Lazy Expiration algorithm</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Version 1.0">Version 1.0</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Version 1.1">Version 1.1</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+<b>ExpireCommand: Contents</b><br>&nbsp;&nbsp;<a href="#EXPIRE _key_ _seconds_">EXPIRE _key_ _seconds_</a><br>&nbsp;&nbsp;<a href="#EXPIREAT _key_ _unixtime_ (Redis &gt;">EXPIREAT _key_ _unixtime_ (Redis &gt;</a><br>&nbsp;&nbsp;<a href="#PERSIST _key_">PERSIST _key_</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#How the expire is removed from a key">How the expire is removed from a key</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Restrictions with write operations against volatile keys">Restrictions with write operations against volatile keys</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Restrictions for write operations with volatile keys as sources">Restrictions for write operations with volatile keys as sources</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Setting the timeout again on already volatile keys">Setting the timeout again on already volatile keys</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Enhanced Lazy Expiration algorithm">Enhanced Lazy Expiration algorithm</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Version 1.0">Version 1.0</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Version 1.1">Version 1.1</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#FAQ: Can you explain better why Redis &lt; 2.1.3 deletes keys with an EXPIRE on write operations?">FAQ: Can you explain better why Redis &lt; 2.1.3 deletes keys with an EXPIRE on write operations?</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#FAQ: How this limitations were solved in Redis versions &gt; 2.1.3?">FAQ: How this limitations were solved in Redis versions &gt; 2.1.3?</a>
                 </div>
                 
                 <h1 class="wikiname">ExpireCommand</h1>
@@ -28,12 +28,15 @@
                 <div class="narrow">
                     &iuml;&raquo;&iquest;#sidebar <a href="GenericCommandsSidebar.html">GenericCommandsSidebar</a><h1><a name="EXPIRE _key_ _seconds_">EXPIRE _key_ _seconds_</a></h1>
 <h1><a name="EXPIREAT _key_ _unixtime_ (Redis &gt;">EXPIREAT _key_ _unixtime_ (Redis &gt;</a></h1> 1.1)=
+<h1><a name="PERSIST _key_">PERSIST _key_</a></h1>
 <i>Time complexity: O(1)</i><blockquote>Set a timeout on the specified key. After the timeout the key will beautomatically delete by the server. A key with an associated timeout issaid to be <i>volatile</i> in Redis terminology.</blockquote>
-<blockquote>Voltile keys are stored on disk like the other keys, the timeout is persistenttoo like all the other aspects of the dataset. Saving a dataset containingthe dataset and stopping the server does not stop the flow of time as Redisregisters on disk when the key will no longer be available as Unix time, andnot the remaining seconds.</blockquote>
+<blockquote>Voltile keys are stored on disk like the other keys, the timeout is persistenttoo like all the other aspects of the dataset. Saving a dataset containingexpires and stopping the server does not stop the flow of time as Redisstores on disk the time when the key will no longer be available as Unixtime, and not the remaining seconds.</blockquote>
 <blockquote>EXPIREAT works exctly like EXPIRE but instead to get the number of secondsrepresenting the Time To Live of the key as a second argument (that is arelative way of specifing the TTL), it takes an absolute one in the form ofa UNIX timestamp (Number of seconds elapsed since 1 Gen 1970).</blockquote>
-<blockquote>EXPIREAT was introduced in order to implement [Persistence append only saving mode] so that EXPIRE commands are automatically translated into EXPIREAT commands for the append only file. Of course EXPIREAT can alsoused by programmers that need a way to simply specify that a given key should expire at a given time in the future.</blockquote>
-<h2><a name="How the expire is removed from a key">How the expire is removed from a key</a></h2><blockquote>When the key is set to a new value using the SET command, the INCR commandor any other command that modify the value stored at key the timeout isremoved from the key and the key becomes non volatile.</blockquote>
-<h2><a name="Restrictions with write operations against volatile keys">Restrictions with write operations against volatile keys</a></h2><blockquote>Write operations like LPUSH, LSET and every other command that has theeffect of modifying the value stored at a volatile key have a special semantic:basically a volatile key is destroyed when it is target of a write operation.See for example the following usage pattern:</blockquote>
+<blockquote>EXPIREAT was introduced in order to implement <a href="AppendOnlyFileHowto.html">the Append Only File persistence mode</a>so that EXPIRE commands are automatically translated into EXPIREAT commands for the append only file. Of course EXPIREAT can alsoused by programmers that need a way to simply specify that a given key should expire at a given time in the future.</blockquote>
+<blockquote>Since Redis 2.1.3 you can update the value of the timeout of a key alreadyhaving an expire set. It is also possible to undo the expire at allturning the key into a normal key using the PERSIST command.</blockquote>
+<h2><a name="How the expire is removed from a key">How the expire is removed from a key</a></h2><blockquote>When the key is set to a new value using the SET command, or when a keyis destroied via DEL, the timeout is removed from the key.</blockquote>
+<h2><a name="Restrictions with write operations against volatile keys">Restrictions with write operations against volatile keys</a></h2><blockquote>IMPORTANT: Since Redis 2.1.3 or greater, there are no restrictions aboutthe operations you can perform against volatile keys, however older versionsof Redis, including the current stable version 2.0.0, has the followinglimitations:</blockquote>
+<blockquote>Write operations like LPUSH, LSET and every other command that has theeffect of modifying the value stored at a volatile key have a special semantic:basically a volatile key is destroyed when it is target of a write operation.See for example the following usage pattern:</blockquote>
 <pre class="codeblock python" name="code">
 % ./redis-cli lpush mylist foobar /Users/antirez/hack/redis
 OK
@@ -45,8 +48,13 @@ OK
 OK
 % ./redis-cli lrange mylist 0 -1  /Users/antirez/hack/redis
 1. newelement
-</pre><blockquote>What happened here is that lpush against the key with a timeout set deletedthe key before to perform the operation. There is so a simple rule, writeoperations against volatile keys will destroy the key before to perform theoperation. Why Redis uses this behavior? In order to retain an importantproperty: a server that receives a given number of commands in the samesequence will end with the same dataset in memory. Without the delete-on-writesemantic what happens is that the state of the server depends on the timeof the commands to. This is not a desirable property in a distributed databasethat supports replication.</blockquote>
-<h2><a name="Setting the timeout again on already volatile keys">Setting the timeout again on already volatile keys</a></h2><blockquote>Trying to call EXPIRE against a key that already has an associated timeoutwill not change the timeout of the key, but will just return 0. If insteadthe key does not have a timeout associated the timeout will be set and EXPIREwill return 1.</blockquote>
+</pre><blockquote>What happened here is that LPUSH against the key with a timeout set deletedthe key before to perform the operation. There is so a simple rule, writeoperations against volatile keys will destroy the key before to perform theoperation. Why Redis uses this behavior? In order to retain an importantproperty: a server that receives a given number of commands in the samesequence will end with the same dataset in memory. Without the delete-on-writesemantic what happens is that the state of the server depends on the timethe commands were issued. This is not a desirable property in a distributed databasethat supports replication.</blockquote>
+<h2><a name="Restrictions for write operations with volatile keys as sources">Restrictions for write operations with volatile keys as sources</a></h2>Even when the volatile key is not modified as part of a write operation, if it is 
+read in a composite write operation (such as SINTERSTORE) it will be cleared at the 
+start of the operation.  This is done to avoid concurrency issues in replication.  
+Imagine a key that is about to expire and the composite operation is run against it. 
+On a slave node, this key might already be expired, which leaves you with a 
+desync in your dataset.<h2><a name="Setting the timeout again on already volatile keys">Setting the timeout again on already volatile keys</a></h2><blockquote>Trying to call EXPIRE against a key that already has an associated timeoutwill not change the timeout of the key, but will just return 0. If insteadthe key does not have a timeout associated the timeout will be set and EXPIREwill return 1.</blockquote>
 <h2><a name="Enhanced Lazy Expiration algorithm">Enhanced Lazy Expiration algorithm</a></h2><blockquote>Redis does not constantly monitor keys that are going to be expired.Keys are expired simply when some client tries to access a key, andthe key is found to be timed out.</blockquote>
 <blockquote>Of course this is not enough as there are expired keys that will neverbe accessed again. This keys should be expired anyway, so once everysecond Redis test a few keys at random among keys with an  expire set.All the keys that are already expired are deleted from the keyspace. </blockquote>
 <h3><a name="Version 1.0">Version 1.0</a></h3><blockquote>Each time a fixed number of keys where tested (100 by default). So ifyou had a client setting keys with a very short expire faster than 100for second the memory continued to grow. When you stopped to insertnew keys the memory started to be freed, 100 keys every second in thebest conditions. Under a peak Redis continues to use more and more RAMeven if most keys are expired in each sweep.</blockquote>
@@ -56,8 +64,29 @@ OK
 <blockquote>This means that at any given moment the maximum amount of keys alreadyexpired that are using memory is at max equal to max setting operations per second divided by 4.</blockquote>
 <h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Integer reply</a>, specifically:<br/><br/><pre class="codeblock python python" name="code">
 1: the timeout was set.
-0: the timeout was not set since the key already has an associated timeout, or the key does not exist.
+0: the timeout was not set since the key already has an associated timeout
+   (this may happen only in Redis versions &lt; 2.1.3, Redis &gt;= 2.1.3 will
+   happily update the timeout), or the key does not exist.
+</pre><h2><a name="FAQ: Can you explain better why Redis &lt; 2.1.3 deletes keys with an EXPIRE on write operations?">FAQ: Can you explain better why Redis &lt; 2.1.3 deletes keys with an EXPIRE on write operations?</a></h2>
+Ok let's start with the problem:
+<pre class="codeblock python python python" name="code">
+redis&gt; set a 100
+OK
+redis&gt; expire a 360
+(integer) 1
+redis&gt; incr a
+(integer) 1
 </pre>
+I set a key to the value of 100, then set an expire of 360 seconds, and then incremented the key (before the 360 timeout expired of course). The obvious result would be: 101, instead the key is set to the value of 1. Why?
+There is a very important reason involving the Append Only File and Replication. Let's rework a bit hour example adding the notion of time to the mix:
+<pre class="codeblock python python python python" name="code">
+SET a 100
+EXPIRE a 5
+... wait 10 seconds ...
+INCR a
+</pre>
+Imagine a Redis version that does not implement the &quot;Delete keys with an expire set on write operation&quot; semantic.
+Running the above example with the 10 seconds pause will lead to 'a' being set to the value of 1, as it no longer exists when INCR is called 10 seconds later.<br/><br/>Instead if we drop the 10 seconds pause, the result is that 'a' is set to 101.<br/><br/>And in the practice timing changes! For instance the client may wait 10 seconds before INCR, but the sequence written in the Append Only File (and later replayed-back as fast as possible when Redis is restarted) will not have the pause. Even if we add a timestamp in the AOF, when the time difference is smaller than our timer resolution, we have a race condition.<br/><br/>The same happens with master-slave replication. Again, consider the example above: the client will use the same sequence of commands without the 10 seconds pause, but the replication link will slow down for a few seconds due to a network problem. Result? The master will contain 'a' set to 101, the slave 'a' set to 1.<br/><br/>The only way to avoid this but at the same time have reliable non time dependent timeouts on keys is to destroy volatile keys when a write operation is attempted against it.<br/><br/>After all Redis is one of the rare fully persistent databases that will give you EXPIRE. This comes to a cost :)<h2><a name="FAQ: How this limitations were solved in Redis versions &gt; 2.1.3?">FAQ: How this limitations were solved in Redis versions &gt; 2.1.3?</a></h2>Since Redis 2.1.3 there are no longer restrictions in the use you can do of write commands against volatile keys, still the replication and AOF file are guaranteed to be fully consistent.<br/><br/>In order to obtain a correct behavior without sacrificing consistency now when a key expires, a DEL operation is synthesized in both the AOF file and against all the attached slaves. This way the expiration process is centralized in the master instance, and there is no longer a chance of consistency errors.<br/><br/>However while the slaves while connected to a master will not expire keys independently, they'll still take the full state of the expires existing in the dataset, so when a slave is elected to a master it will be able to expire the keys independently, fully acting as a master.
 
                 </div>
         

doc/GenericCommandsSidebar.html

@@ -26,7 +26,7 @@
                 </div>
 
                 <div class="narrow">
-                    &iuml;&raquo;&iquest;== Generic Commands ==<br/><br/><ul><li> <a href="ExistsCommand.html">EXISTS</a></li><li> <a href="DelCommand.html">DEL</a></li><li> <a href="TypeCommand.html">TYPE</a></li><li> <a href="KeysCommand.html">KEYS</a></li><li> <a href="RandomkeyCommand.html">RANDOMKEY</a></li><li> <a href="RenameCommand.html">RENAME</a></li><li> <a href="RenamenxCommand.html">RENAMENX</a></li><li> <a href="DbsizeCommand.html">DBSIZE</a></li><li> <a href="ExpireCommand.html">EXPIRE</a></li><li> <a href="TtlCommand.html">TTL</a></li><li> <a href="SelectCommand.html">SELECT</a></li><li> <a href="MoveCommand.html">MOVE</a></li><li> <a href="FlushdbCommand.html">FLUSHDB</a></li><li> <a href="FlushallCommand.html">FLUSHALL</a></li></ul>
+                    &iuml;&raquo;&iquest;== Generic Commands ==<br/><br/><ul><li> <a href="ExistsCommand.html">EXISTS</a></li><li> <a href="DelCommand.html">DEL</a></li><li> <a href="TypeCommand.html">TYPE</a></li><li> <a href="KeysCommand.html">KEYS</a></li><li> <a href="RandomkeyCommand.html">RANDOMKEY</a></li><li> <a href="RenameCommand.html">RENAME</a></li><li> <a href="RenamenxCommand.html">RENAMENX</a></li><li> <a href="DbsizeCommand.html">DBSIZE</a></li><li> <a href="ExpireCommand.html">EXPIRE</a></li><li> <a href="ExpireCommand.html">PERSIST</a></li><li> <a href="TtlCommand.html">TTL</a></li><li> <a href="SelectCommand.html">SELECT</a></li><li> <a href="MoveCommand.html">MOVE</a></li><li> <a href="FlushdbCommand.html">FLUSHDB</a></li><li> <a href="FlushallCommand.html">FLUSHALL</a></li><li> <a href="MultiExecCommand.html">Redis Transactions</a></li></ul>
                 </div>
         
             </div>

doc/HackingStrings.html

@@ -0,0 +1,83 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>HackingStrings: Contents</b><br>&nbsp;&nbsp;<a href="#Hacking Strings">Hacking Strings</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Creating Redis Strings">Creating Redis Strings</a>
+                </div>
+                
+                <h1 class="wikiname">HackingStrings</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    
+<h1><a name="Hacking Strings">Hacking Strings</a></h1>The implementation of Redis strings is contained in <b></b>sds.c<b></b> ( sds stands for Simple Dynamic Strings ).<br/><br/>The C structure <i>sdshdr</i> declared in <b>sds.h</b> represents a Redis string:<br/><br/><pre class="codeblock python" name="code">
+struct sdshdr {
+    long len;
+    long free;
+    char buf[];
+};
+</pre>The <i>buf</i> character array stores the actual string.<br/><br/>The <i>len</i> field stores the length of <i>buf</i>. This makes obtaining the length
+of a Redis string an O(1) operation.<br/><br/>The <i>free</i> field stores the number of additional bytes available for use.<br/><br/>Together the <i>len</i> and <i>free</i> field can be thought of as holding the metadata of the 
+<i>buf</i> character array.<h2><a name="Creating Redis Strings">Creating Redis Strings</a></h2>A new data type named <code name="code" class="python">sds</code> is defined in <b>sds.h</b> to be a synonymn for a character pointer:<br/><br/><pre class="codeblock python python" name="code">
+typedef char *sds;
+</pre><code name="code" class="python">sdsnewlen</code> function defined in <b>sds.c</b> creates a new Redis String: <br/><br/><pre class="codeblock python python python" name="code">
+sds sdsnewlen(const void *init, size_t initlen) {
+    struct sdshdr *sh;
+
+    sh = zmalloc(sizeof(struct sdshdr)+initlen+1);
+#ifdef SDS_ABORT_ON_OOM
+    if (sh == NULL) sdsOomAbort();
+#else
+    if (sh == NULL) return NULL;
+#endif
+    sh-&gt;len = initlen;
+    sh-&gt;free = 0;
+    if (initlen) {
+        if (init) memcpy(sh-&gt;buf, init, initlen);
+        else memset(sh-&gt;buf,0,initlen);
+    }
+    sh-&gt;buf[initlen] = '\0';
+    return (char*)sh-&gt;buf;
+}
+</pre>Remember a Redis string is a variable of type <code name="code" class="python">struct sdshdr</code>. But <code name="code" class="python">sdsnewlen</code> returns a character pointer!!<br/><br/>That's a trick and needs some explanation.<br/><br/>Suppose I create a Redis string using <code name="code" class="python">sdsnewlen</code> like below:<br/><br/><pre class="codeblock python python python python" name="code">
+sdsnewlen(&quot;redis&quot;, 5);
+</pre>This creates a new variable of type <code name="code" class="python">struct sdshdr</code> allocating memory for <i>len</i> and <i>free</i>
+fields as well as for the <i>buf</i> character array.<br/><br/><pre class="codeblock python python python python python" name="code">
+sh = zmalloc(sizeof(struct sdshdr)+initlen+1); // initlen is length of init argument.
+</pre>After <code name="code" class="python">sdsnewlen</code> succesfully creates a Redis string the result is something like:<br/><br/><pre class="codeblock python python python python python python" name="code">
+-----------
+|5|0|redis|
+-----------
+^   ^
+sh  sh-&gt;buf 
+</pre><code name="code" class="python">sdsnewlen</code> returns sh-&gt;buf to the caller.<br/><br/>What do you do if you need to free the Redis string pointed by <code name="code" class="python">sh</code>?<br/><br/>You want the pointer <code name="code" class="python">sh</code> but you only have the pointer <code name="code" class="python">sh-&gt;buf</code>.<br/><br/>Can you get the pointer <code name="code" class="python">sh</code> from <code name="code" class="python">sh-&gt;buf</code>?<br/><br/>Yes. Pointer arithmetic. Notice from the above ASCII art that if you subtract 
+the size of two longs from <code name="code" class="python">sh-&gt;buf</code> you get the pointer <code name="code" class="python">sh</code>. <br/><br/>The sizeof two longs happens to be the size of <code name="code" class="python">struct sdshdr</code>.<br/><br/>Look at <code name="code" class="python">sdslen</code> function and see this trick at work:<br/><br/><pre class="codeblock python python python python python python python" name="code">
+size_t sdslen(const sds s) {
+    struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr)));
+    return sh-&gt;len;
+}
+</pre>Knowing this trick you could easily go through the rest of the functions in <b>sds.c</b>.<br/><br/>The Redis string implementation is hidden behind an interface that accepts only character pointers. The users of Redis strings need not care about how its implemented and treat Redis strings as a character pointer.
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/HashCommandsSidebar.html

@@ -0,0 +1,36 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>HashCommandsSidebar: Contents</b>
+                </div>
+                
+                <h1 class="wikiname">HashCommandsSidebar</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;== Hash Commands ==<br/><br/><ul><li> <a href="HsetCommand.html">HSET</a></li><li> <a href="HgetCommand.html">HGET</a></li><li> <a href="HsetnxCommand.html">HSETNX</a></li><li> <a href="HmsetCommand.html">HMSET</a></li><li> <a href="HmgetCommand.html">HMGET</a></li><li> <a href="HincrbyCommand.html">HINCRBY</a></li><li> <a href="HexistsCommand.html">HEXISTS</a></li><li> <a href="HdelCommand.html">HDEL</a></li><li> <a href="HlenCommand.html">HLEN</a></li><li> <a href="HgetallCommand.html">HKEYS</a></li><li> <a href="HgetallCommand.html">HVALS</a></li><li> <a href="HgetallCommand.html">HGETALL</a></li></ul>
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/Hashes.html

@@ -0,0 +1,37 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>Hashes: Contents</b><br>&nbsp;&nbsp;<a href="#Redis Hash Type">Redis Hash Type</a><br>&nbsp;&nbsp;<a href="#Implementation details">Implementation details</a>
+                </div>
+                
+                <h1 class="wikiname">Hashes</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;#sidebar <a href="HashCommandsSidebar.html">HashCommandsSidebar</a><h1><a name="Redis Hash Type">Redis Hash Type</a></h1>Redis Hashes are unordered maps of <a href="String.html">Redis Strings</a> between fields and values. It is possible to add, remove, test for existence of fields in O(1) amortized time. It is also possible to enumerate all the keys, values, or both, in O(N) (where N is the number of fields inside the hash).<br/><br/>Redis Hashes are interesting because they are very well suited to represent objects. For instance web applications users can be represented by a Redis Hash containing fields such username, encrpypted_password, lastlogin, and so forth.<br/><br/>Another very important property of Redis Hashes is that they use very little memory for hashes composed of a small number of fields (configurable, check redis.conf for details), compared to storing every field as a top level Redis key. This is obtained using a different specialized representation for small hashes. See the implementation details paragraph below for more information.<br/><br/>Commands operating on hashes try to make a good use of the return value in order to signal the application about previous existence of fields. For instance the <a href="HsetCommand.html">HSET</a> command will return 1 if the field set was not already present in the hash, otherwise will return 0 (and the user knows this was just an update operation).<br/><br/>The max number of fields in a set is 232-1 (4294967295, more than 4 billion of members per hash).<h1><a name="Implementation details">Implementation details</a></h1>The obvious internal representation of hashes is indeed an hash table, as the name of the data structure itself suggests. Still the drawback of this representation is that there is a lot of space overhead for hash table metadata.<br/><br/>Because one of the most interesting uses of Hashes is object encoding, and objects are often composed of a few fields each, Redis uses a different internal representation for small hashes (for Redis to consider a hash small, this must be composed a limited number of fields, and each field and value can't exceed a given number of bytes. All this is user-configurable).<br/><br/>Small hashes are thus encoded using a data structure called zipmap (is not something you can find in a CS book, the name is a Redis invention), that is a very memory efficient data structure to represent string to string maps, at the cost of being O(N) instead of O(1) for most operations. Since the constant times of this data structure are very small, and the zipmaps are converted into real hash tables once they are big enough, the amortized time of Redis hashes is still O(1), and in the practice small zipmaps are not slower than small hash tables because they are designed for good cache locality and fast access.<br/><br/>The result is that small hashes are both memory efficient and fast, while bigger hashes are fast but not as memory efficient than small hashes.
+
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/HdelCommand.html

@@ -0,0 +1,39 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>HdelCommand: Contents</b><br>&nbsp;&nbsp;<a href="#HDEL _key_ _field_ (Redis &gt;">HDEL _key_ _field_ (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+                </div>
+                
+                <h1 class="wikiname">HdelCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;#sidebar <a href="HashCommandsSidebar.html">HashCommandsSidebar</a><h1><a name="HDEL _key_ _field_ (Redis &gt;">HDEL _key_ _field_ (Redis &gt;</a></h1> 1.3.10)=
+<i>Time complexity: O(1)</i><blockquote>Remove the specified <i>field</i> from an hash stored at <i>key</i>.</blockquote>
+<blockquote>If the <i>field</i> was present in the hash it is returned and 1 is returned, otherwise 0 is returned and no operation is performed.</blockquote>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Integer reply</a>
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/HexistsCommand.html

@@ -0,0 +1,39 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>HexistsCommand: Contents</b><br>&nbsp;&nbsp;<a href="#HEXISTS _key_ _field_ (Redis &gt;">HEXISTS _key_ _field_ (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+                </div>
+                
+                <h1 class="wikiname">HexistsCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;#sidebar <a href="HashCommandsSidebar.html">HashCommandsSidebar</a><h1><a name="HEXISTS _key_ _field_ (Redis &gt;">HEXISTS _key_ _field_ (Redis &gt;</a></h1> 1.3.10)=
+<i>Time complexity: O(1)</i><blockquote>Return 1 if the hash stored at <i>key</i> contains the specified <i>field</i>.</blockquote>
+<blockquote>Return 0 if the <i>key</i> is not found or the <i>field</i> is not present.</blockquote>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Integer reply</a>
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/HgetCommand.html

@@ -0,0 +1,39 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>HgetCommand: Contents</b><br>&nbsp;&nbsp;<a href="#HGET _key_ _field_ (Redis &gt;">HGET _key_ _field_ (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+                </div>
+                
+                <h1 class="wikiname">HgetCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;#sidebar <a href="HashCommandsSidebar.html">HashCommandsSidebar</a><h1><a name="HGET _key_ _field_ (Redis &gt;">HGET _key_ _field_ (Redis &gt;</a></h1> 1.3.10)=
+<i>Time complexity: O(1)</i><blockquote>If <i>key</i> holds a hash, retrieve the value associated to the specified <i>field</i>.</blockquote>
+<blockquote>If the <i>field</i> is not found or the <i>key</i> does not exist, a special 'nil' value is returned.</blockquote>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Bulk reply</a>
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/HgetallCommand.html

@@ -0,0 +1,40 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>HgetallCommand: Contents</b><br>&nbsp;&nbsp;<a href="#HKEYS _key_ (Redis &gt;">HKEYS _key_ (Redis &gt;</a><br>&nbsp;&nbsp;<a href="#HVALS _key_ (Redis &gt;">HVALS _key_ (Redis &gt;</a><br>&nbsp;&nbsp;<a href="#HGETALL _key_ (Redis &gt;">HGETALL _key_ (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+                </div>
+                
+                <h1 class="wikiname">HgetallCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;#sidebar <a href="HashCommandsSidebar.html">HashCommandsSidebar</a><h1><a name="HKEYS _key_ (Redis &gt;">HKEYS _key_ (Redis &gt;</a></h1> 1.3.10)=
+<h1><a name="HVALS _key_ (Redis &gt;">HVALS _key_ (Redis &gt;</a></h1> 1.3.10)=
+<h1><a name="HGETALL _key_ (Redis &gt;">HGETALL _key_ (Redis &gt;</a></h1> 1.3.10)=
+<i>Time complexity: O(N), where N is the total number of entries</i><blockquote>HKEYS returns all the fields names contained into a hash, HVALS all the associated values, while HGETALL returns both the fields and values in the form of <i>field1</i>, <i>value1</i>, <i>field2</i>, <i>value2</i>, ..., <i>fieldN</i>, <i>valueN</i>.</blockquote>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Multi Bulk Reply</a>
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/HincrbyCommand.html

@@ -0,0 +1,45 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>HincrbyCommand: Contents</b><br>&nbsp;&nbsp;<a href="#HINCRBY _key_ _field_ _value_ (Redis &gt;">HINCRBY _key_ _field_ _value_ (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Examples">Examples</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+                </div>
+                
+                <h1 class="wikiname">HincrbyCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    
+<h1><a name="HINCRBY _key_ _field_ _value_ (Redis &gt;">HINCRBY _key_ _field_ _value_ (Redis &gt;</a></h1> 1.3.10)=
+<i>Time complexity: O(1)</i><blockquote>Increment the number stored at <i>field</i> in the hash at <i>key</i> by <i>value</i>. If <i>key</i> does not exist, a new key holding a hash is created. If <i>field</i> does not exist or holds a string, the value is set to 0 before applying the operation.</blockquote>
+<blockquote>The range of values supported by HINCRBY is limited to 64 bit signed integers.</blockquote><h2><a name="Examples">Examples</a></h2>
+Since the <i>value</i> argument is signed you can use this command to perform both increments and decrements:<br/><br/><pre class="codeblock python" name="code">
+HINCRBY key field 1 (increment by one)
+HINCRBY key field -1 (decrement by one, just like the DECR command)
+HINCRBY key field -10 (decrement by 10)
+</pre>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Integer reply</a> The new value at <i>field</i> after the increment operation.
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/HlenCommand.html

@@ -0,0 +1,38 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>HlenCommand: Contents</b><br>&nbsp;&nbsp;<a href="#HLEN _key_ (Redis &gt;">HLEN _key_ (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+                </div>
+                
+                <h1 class="wikiname">HlenCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;#sidebar <a href="HashCommandsSidebar.html">HashCommandsSidebar</a><h1><a name="HLEN _key_ (Redis &gt;">HLEN _key_ (Redis &gt;</a></h1> 1.3.10)=
+<i>Time complexity: O(1)</i><blockquote>Return the number of entries (fields) contained in the hash stored at <i>key</i>. If the specified <i>key</i> does not exist, 0 is returned assuming an empty hash. </blockquote>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Integer reply</a>
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/HmsetCommand.html

@@ -0,0 +1,40 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>HmsetCommand: Contents</b><br>&nbsp;&nbsp;<a href="#HMSET _key_ _field1_ _value1_ ... _fieldN_ _valueN_ (Redis &gt;">HMSET _key_ _field1_ _value1_ ... _fieldN_ _valueN_ (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+                </div>
+                
+                <h1 class="wikiname">HmsetCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    
+<h1><a name="HMSET _key_ _field1_ _value1_ ... _fieldN_ _valueN_ (Redis &gt;">HMSET _key_ _field1_ _value1_ ... _fieldN_ _valueN_ (Redis &gt;</a></h1> 1.3.10) =
+<i>Time complexity: O(N) (with N being the number of fields)</i><blockquote>Set the respective fields to the respective values. HMSET replaces old values with new values.</blockquote>
+<blockquote>If <i>key</i> does not exist, a new key holding a hash is created.</blockquote>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Status code reply</a> Always +OK because HMSET can't fail
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/HsetCommand.html

@@ -0,0 +1,40 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>HsetCommand: Contents</b><br>&nbsp;&nbsp;<a href="#HSET _key_ _field_ _value_ (Redis &gt;">HSET _key_ _field_ _value_ (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+                </div>
+                
+                <h1 class="wikiname">HsetCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;#sidebar <a href="HashCommandsSidebar.html">HashCommandsSidebar</a><h1><a name="HSET _key_ _field_ _value_ (Redis &gt;">HSET _key_ _field_ _value_ (Redis &gt;</a></h1> 1.3.10)=
+<i>Time complexity: O(1)</i><blockquote>Set the specified hash <i>field</i> to the specified <i>value</i>.</blockquote>
+<blockquote>If <i>key</i> does not exist, a new key holding a hash is created.</blockquote>
+<blockquote>If the field already exists, and the HSET just produced an update of thevalue, 0 is returned, otherwise if a new field is created 1 is returned.</blockquote>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Integer reply</a>
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/HsetnxCommand.html

@@ -0,0 +1,41 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>HsetnxCommand: Contents</b><br>&nbsp;&nbsp;<a href="#HSETNX _key_ _field_ _value_ (Redis &gt;">HSETNX _key_ _field_ _value_ (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+                </div>
+                
+                <h1 class="wikiname">HsetnxCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    
+<h1><a name="HSETNX _key_ _field_ _value_ (Redis &gt;">HSETNX _key_ _field_ _value_ (Redis &gt;</a></h1> 1.3.10)=
+<i>Time complexity: O(1)</i><blockquote>Set the specified hash <i>field</i> to the specified <i>value</i>, if <i>field</i> does not exist yet.</blockquote>
+<blockquote>If <i>key</i> does not exist, a new key holding a hash is created.</blockquote>
+<blockquote>If the field already exists, this operation has no effect and returns 0.Otherwise, the field is set to <i>value</i> and the operation returns 1.</blockquote>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Integer reply</a>
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/IncrCommand.html

@@ -33,8 +33,7 @@
 <i>Time complexity: O(1)</i><blockquote>Increment or decrement the number stored at <i>key</i> by one. If the key doesnot exist or contains a value of a wrong type, set the key to thevalue of &quot;0&quot; before to perform the increment or decrement operation.</blockquote>
 <blockquote>INCRBY and DECRBY work just like INCR and DECR but instead toincrement/decrement by 1 the increment/decrement is <i>integer</i>.</blockquote>
 <blockquote>INCR commands are limited to 64 bit signed integers.</blockquote>
-<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Integer reply</a>, this commands will reply with the new value of <i>key</i> after the increment or decrement.
-
+Note: this is actually a string operation, that is, in Redis there are not &quot;integer&quot; types. Simply the string stored at the key is parsed as a base 10 64 bit signed integer, incremented, and then converted back as a string.<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Integer reply</a>, this commands will reply with the new value of <i>key</i> after the increment or decrement.
                 </div>
         
             </div>

doc/IntroductionToRedisDataTypes.html

@@ -16,7 +16,7 @@
             <div id="pagecontent">
                 <div class="index">
 <!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
-<b>IntroductionToRedisDataTypes: Contents</b><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Redis keys">Redis keys</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#The string type">The string type</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#The List type">The List type</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#First steps with Redis lists">First steps with Redis lists</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Pushing IDs instead of the actual data in Redis lists">Pushing IDs instead of the actual data in Redis lists</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Redis Sets">Redis Sets</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#A digression. How to get unique identifiers for strings">A digression. How to get unique identifiers for strings</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Sorted sets">Sorted sets</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Operating on ranges">Operating on ranges</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Back to the reddit example">Back to the reddit example</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Updating the scores of a sorted set">Updating the scores of a sorted set</a>
+<b>IntroductionToRedisDataTypes: Contents</b><br>&nbsp;&nbsp;<a href="#A fifteen minutes introduction to Redis data types">A fifteen minutes introduction to Redis data types</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Redis keys">Redis keys</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#The string type">The string type</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#The List type">The List type</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#First steps with Redis lists">First steps with Redis lists</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Pushing IDs instead of the actual data in Redis lists">Pushing IDs instead of the actual data in Redis lists</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Redis Sets">Redis Sets</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#A digression. How to get unique identifiers for strings">A digression. How to get unique identifiers for strings</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Sorted sets">Sorted sets</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Operating on ranges">Operating on ranges</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Back to the reddit example">Back to the reddit example</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Updating the scores of a sorted set">Updating the scores of a sorted set</a>
                 </div>
                 
                 <h1 class="wikiname">IntroductionToRedisDataTypes</h1>
@@ -26,7 +26,8 @@
                 </div>
 
                 <div class="narrow">
-                    &iuml;&raquo;&iquest;= A fifteen minutes introduction to Redis data types =<br/><br/>As you already probably know Redis is not a plain key-value store, actually it is a <b>data structures server</b>, supporting different kind of values. That is, you can't just set strings as values of keys. All the following data types are supported as values:<br/><br/><ul><li> Binary-safe strings.</li><li> Lists of binary-safe strings.</li><li> Sets of binary-safe strings, that are collection of unique unsorted elements. You can think at this as a Ruby hash where all the keys are set to the 'true' value.</li><li> Sorted sets, similar to Sets but where every element is associated to a floating number score. The elements are taken sorted by score. You can think at this as Ruby hashes where the key is the element and the value is the score, but where elements are always taken in order without requiring a sorting operation.</li></ul>
+                    &iuml;&raquo;&iquest;#sidebar <a href="RedisGuides.html">RedisGuides</a>
+<h1><a name="A fifteen minutes introduction to Redis data types">A fifteen minutes introduction to Redis data types</a></h1>As you already probably know Redis is not a plain key-value store, actually it is a <b>data structures server</b>, supporting different kind of values. That is, you can't just set strings as values of keys. All the following data types are supported as values:<br/><br/><ul><li> Binary-safe strings.</li><li> Lists of binary-safe strings.</li><li> Sets of binary-safe strings, that are collection of unique unsorted elements. You can think at this as a Ruby hash where all the keys are set to the 'true' value.</li><li> Sorted sets, similar to Sets but where every element is associated to a floating number score. The elements are taken sorted by score. You can think at this as Ruby hashes where the key is the element and the value is the score, but where elements are always taken in order without requiring a sorting operation.</li></ul>
 It's not always trivial to grasp how this data types work and what to use in order to solve a given problem from the <a href="CommandReference.html">Redis command reference</a>, so this document is a crash course to Redis data types and their most used patterns.<br/><br/>For all the examples we'll use the <b>redis-cli</b> utility, that's a simple but handy command line utility to issue commands against the Redis server.<h2><a name="Redis keys">Redis keys</a></h2>Before to start talking about the different kind of values supported by Redis it is better to start saying that keys are not binary safe strings in Redis, but just strings not containing a space or a newline character. For instance &quot;foo&quot; or &quot;123456789&quot; or &quot;foo_bar&quot; are valid keys, while &quot;hello world&quot; or &quot;hello\n&quot; are not.<br/><br/>Actually there is nothing inside the Redis internals preventing the use of binary keys, it's just a matter of protocol, and actually the new protocol introduced with Redis 1.2 (1.2 betas are 1.1.x) in order to implement commands like MSET, is totally binary safe. Still for now consider this as an hard limit as the database is only tested with &quot;normal&quot; keys.<br/><br/>A few other rules about keys:<br/><br/><ul><li> Too long keys are not a good idea, for instance a key of 1024 bytes is not a good idea not only memory-wise, but also because the lookup of the key in the dataset may require several costly key-comparisons.</li><li> Too short keys are not a good idea. There is no point in writing &quot;u:1000:pwd&quot; as key if you can write instead &quot;user:1000:password&quot;, the latter is more readable and the added space is very little compared to the space used by the key object itself.</li><li> Try to stick with a schema. For instance &quot;object-type:id:field&quot; can be a nice idea, like in &quot;user:1000:password&quot;. I like to use dots for multi-words fields, like in &quot;comment:1234:reply.to&quot;.</li></ul>
 <h2><a name="The string type">The string type</a></h2>This is the simplest Redis type. If you use only this type, Redis will be something like a memcached server with persistence.<br/><br/>Let's play a bit with the string type:<br/><br/><pre class="codeblock python" name="code">
 $ ./redis-cli set mykey &quot;my binary safe value&quot;
@@ -42,7 +43,7 @@ $ ./redis-cli incr counter
 (integer) 102
 $ ./redis-cli incrby counter 10
 (integer) 112
-</pre>The <a href="IncrCommand.html">INCR</a> command parses the string value as an integer, increments it by one, and finally sets the obtained value as the new string value. There are other similar commands like <a href="IncrCommand.html">INCRBY</a>, <a href="IncrCommand.html">DECR</a> and <a href="IncrCommand.html">DECRBY</a>. Actually internally it's always the same command, acting in a slightly different way.<br/><br/>What means that INCR is atomic? That even multiple clients issuing INCR against the same key will never incur into a race condition. For instance it can't never happen that client 1 read &quot;10&quot;, client 2 read &quot;10&quot; at the same time, both increment to 11, and set the new value of 11. The final value will always be of 12 ad the read-increment-set operation is performed while all the other clients are not executing a command at the same time.<br/><br/>Another interesting operation on string is the <a href="GetsetCommand.html">GETSET</a> command, that does just what its name suggests: Set a key to a new value, returning the old value, as result. Why this is useful? Example: you have a system that increments a Redis key using the <a href="IncrCommand.html">INCR</a> command every time your web site receives a new visit. You want to collect this information one time every hour, without loosing a single key. You can GETSET the key assigning it the new value of &quot;0&quot; and reading the old value back.<h2><a name="The List type">The List type</a></h2>To explain the List data type it's better to start with a little of theory, as the term <b>List</b> is often used in an improper way by information technology folks. For instance &quot;Python Lists&quot; are not what the name may suggest (Linked Lists), but them are actually Arrays (the same data type is called Array in Ruby actually).<br/><br/>From a very general point of view a List is just a sequence of ordered elements: 10,20,1,2,3 is a list, but when a list of items is implemented using an Array and when instead a <b>Linked List</b> is used for the implementation, the properties change a lot.<br/><br/>Redis lists are implemented via Linked Lists, this means that even if you have million of elements inside a list, the operation of adding a new element in the head or in the tail of the list is performed <b>in constant time</b>. Adding a new element with the <a href="LpopCommand.html">LPOP</a> command to the head of a ten elements list is the same speed as adding an element to the head of a 10 million elements list.<br/><br/>What's the downside? That accessing an element <b>by index</b> is very fast in lists implemented with an Array and not so fast in lists implemented by linked lists.<br/><br/>Redis Lists are implemented with linked lists because for a database system is crucial to be able to add elements to a very long list in a very fast way. Another strong advantage is, as you'll see in a moment, that Redis Lists can be taken at constant length in constant time.<h3><a name="First steps with Redis lists">First steps with Redis lists</a></h3>The <a href="RpushCommand.html">LPUSH</a> command add a new element into a list, on the left (on head), while the <a href="RpushCommand.html">RPUSH</a> command add a new element into alist, ot the right (on tail). Finally the <a href="LrangeCommand.html">LRANGE</a> command extract ranges of elements from lists:<br/><br/><pre class="codeblock python python python" name="code">
+</pre>The <a href="IncrCommand.html">INCR</a> command parses the string value as an integer, increments it by one, and finally sets the obtained value as the new string value. There are other similar commands like <a href="IncrCommand.html">INCRBY</a>, <a href="IncrCommand.html">DECR</a> and <a href="IncrCommand.html">DECRBY</a>. Actually internally it's always the same command, acting in a slightly different way.<br/><br/>What means that INCR is atomic? That even multiple clients issuing INCR against the same key will never incur into a race condition. For instance it can't never happen that client 1 read &quot;10&quot;, client 2 read &quot;10&quot; at the same time, both increment to 11, and set the new value of 11. The final value will always be of 12 ad the read-increment-set operation is performed while all the other clients are not executing a command at the same time.<br/><br/>Another interesting operation on string is the <a href="GetsetCommand.html">GETSET</a> command, that does just what its name suggests: Set a key to a new value, returning the old value, as result. Why this is useful? Example: you have a system that increments a Redis key using the <a href="IncrCommand.html">INCR</a> command every time your web site receives a new visit. You want to collect this information one time every hour, without loosing a single key. You can GETSET the key assigning it the new value of &quot;0&quot; and reading the old value back.<h2><a name="The List type">The List type</a></h2>To explain the List data type it's better to start with a little of theory, as the term <b>List</b> is often used in an improper way by information technology folks. For instance &quot;Python Lists&quot; are not what the name may suggest (Linked Lists), but them are actually Arrays (the same data type is called Array in Ruby actually).<br/><br/>From a very general point of view a List is just a sequence of ordered elements: 10,20,1,2,3 is a list, but when a list of items is implemented using an Array and when instead a <b>Linked List</b> is used for the implementation, the properties change a lot.<br/><br/>Redis lists are implemented via Linked Lists, this means that even if you have million of elements inside a list, the operation of adding a new element in the head or in the tail of the list is performed <b>in constant time</b>. Adding a new element with the <a href="LpushCommand.html">LPUSH</a> command to the head of a ten elements list is the same speed as adding an element to the head of a 10 million elements list.<br/><br/>What's the downside? That accessing an element <b>by index</b> is very fast in lists implemented with an Array and not so fast in lists implemented by linked lists.<br/><br/>Redis Lists are implemented with linked lists because for a database system is crucial to be able to add elements to a very long list in a very fast way. Another strong advantage is, as you'll see in a moment, that Redis Lists can be taken at constant length in constant time.<h3><a name="First steps with Redis lists">First steps with Redis lists</a></h3>The <a href="RpushCommand.html">LPUSH</a> command add a new element into a list, on the left (on head), while the <a href="RpushCommand.html">RPUSH</a> command add a new element into alist, ot the right (on tail). Finally the <a href="LrangeCommand.html">LRANGE</a> command extract ranges of elements from lists:<br/><br/><pre class="codeblock python python python" name="code">
 $ ./redis-cli rpush messages &quot;Hello how are you?&quot;
 OK
 $ ./redis-cli rpush messages &quot;Fine thanks. I'm having fun with Redis&quot;
@@ -142,6 +143,7 @@ $ ./redis-cli zrangebyscore hackers -inf 1950
 $ ./redis-cli zremrangebyscore hackers 1940 1960
 (integer) 2
 </pre><a href="ZremrangebyscoreCommand.html">ZREMRANGEBYSCORE</a> is not the best command name, but it can be very useful, and returns the number of removed elements.<h3><a name="Back to the reddit example">Back to the reddit example</a></h3>For the last time, back to the Reddit example. Now we have a decent plan to populate a sorted set in order to generate the home page. A sorted set can contain all the news that are not older than a few days (we remove old entries from time to time using ZREMRANGEBYSCORE). A background job gets all the elements from this sorted set, get the user votes and the time of the news, and compute the score to populate the <b>reddit.home.page</b> sorted set with the news IDs and associated scores. To show the home page we have just to perform a blazingly fast call to ZRANGE.<br/><br/>From time to time we'll remove too old news from the <b>reddit.home.page</b> sorted set as well in order for our system to work always against a limited set of news.<h3><a name="Updating the scores of a sorted set">Updating the scores of a sorted set</a></h3>Just a final note before to finish this tutorial. Sorted sets scores can be updated at any time. Just calling again ZADD against an element already included in the sorted set will update its score (and position) in O(log(N)), so sorted sets are suitable even when there are tons of updates.<br/><br/>This tutorial is in no way complete, this is just the basics to get started with Redis, read the <a href="CommandReference.html">Command Reference</a> to discover a lot more.<br/><br/>Thanks for reading. Salvatore.
+
                 </div>
         
             </div>

doc/KeysCommand.html

@@ -32,7 +32,8 @@
 <blockquote>the slow commands that may ruin the DB performance if not usedwith care*.</blockquote>
 <blockquote>In other words this command is intended only for debugging and *special* operations like  creating a script to change the DB schema. Don't use it in your normal code. Use Redis <a href="Sets.html">Sets</a> in order to group together a subset of objects.</blockquote>
 Glob style patterns examples:
-<blockquote>* h?llo will match hello hallo hhllo* h*llo will match hllo heeeello* h<code name="code" class="python">[</code>ae<code name="code" class="python">]</code>llo will match hello and hallo, but not hillo</blockquote>Use \ to escape special chars if you want to match them verbatim.<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Bulk reply</a>, specifically a string in the form of space separated list of keys. Note that most client libraries will return an Array of keys and not a single string with space separated keys (that is, split by &quot; &quot; is performed in the client library usually).</b></blockquote>
+<blockquote>* h?llo will match hello hallo hhllo* h*llo will match hllo heeeello* h<code name="code" class="python">[</code>ae<code name="code" class="python">]</code>llo will match hello and hallo, but not hillo</blockquote>Use \ to escape special chars if you want to match them verbatim.<h2><a name="Return value">Return value</a></h2>
+<a href="ReplyTypes.html">Multi bulk reply</a></b></blockquote>
                 </div>
         
             </div>

doc/LindexCommand.html

@@ -28,10 +28,9 @@
                 <div class="narrow">
                     &iuml;&raquo;&iquest;#sidebar <a href="ListCommandsSidebar.html">ListCommandsSidebar</a><h1><a name="LINDEX _key_ _index_">LINDEX _key_ _index_</a></h1>
 <i>Time complexity: O(n) (with n being the length of the list)</i><blockquote>Return the specified element of the list stored at the specifiedkey. 0 is the first element, 1 the second and so on. Negative indexesare supported, for example -1 is the last element, -2 the penultimateand so on.</blockquote>
-<blockquote>If the value stored at key is not of list type an error is returned.If the index is out of range an empty string is returned.</blockquote>
+<blockquote>If the value stored at key is not of list type an error is returned.If the index is out of range a 'nil' reply is returned.</blockquote>
 <blockquote>Note that even if the average time complexity is O(n) asking forthe first or the last element of the list is O(1).</blockquote>
 <h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Bulk reply</a>, specifically the requested element.
-
                 </div>
         
             </div>

doc/ListCommandsSidebar.html

@@ -26,7 +26,7 @@
                 </div>
 
                 <div class="narrow">
-                    &iuml;&raquo;&iquest;== List Commands ==<br/><br/><ul><li> <a href="RpushCommand.html">RPUSH</a></li><li> <a href="RpushCommand.html">LPUSH</a></li><li> <a href="LlenCommand.html">LLEN</a></li><li> <a href="LrangeCommand.html">LRANGE</a></li><li> <a href="LtrimCommand.html">LTRIM</a></li><li> <a href="LindexCommand.html">LINDEX</a></li><li> <a href="LsetCommand.html">LSET</a></li><li> <a href="LremCommand.html">LREM</a></li><li> <a href="LpopCommand.html">LPOP</a></li><li> <a href="LpopCommand.html">RPOP</a></li><li> <a href="RpoplpushCommand.html">RPOPLPUSH</a></li><li> <a href="SortCommand.html">SORT</a></li></ul>
+                    &iuml;&raquo;&iquest;== List Commands ==<br/><br/><ul><li> <a href="RpushCommand.html">RPUSH</a></li><li> <a href="RpushCommand.html">LPUSH</a></li><li> <a href="LlenCommand.html">LLEN</a></li><li> <a href="LrangeCommand.html">LRANGE</a></li><li> <a href="LtrimCommand.html">LTRIM</a></li><li> <a href="LindexCommand.html">LINDEX</a></li><li> <a href="LsetCommand.html">LSET</a></li><li> <a href="LremCommand.html">LREM</a></li><li> <a href="LpopCommand.html">LPOP</a></li><li> <a href="LpopCommand.html">RPOP</a></li><li> <a href="BlpopCommand.html">BLPOP</a></li><li> <a href="BlpopCommand.html">BRPOP</a></li><li> <a href="RpoplpushCommand.html">RPOPLPUSH</a></li><li> <a href="SortCommand.html">SORT</a></li></ul>
                 </div>
         
             </div>

doc/LrangeCommand.html

@@ -16,7 +16,7 @@
             <div id="pagecontent">
                 <div class="index">
 <!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
-<b>LrangeCommand: Contents</b><br>&nbsp;&nbsp;<a href="#LRANGE _key_ _start_ _end_">LRANGE _key_ _start_ _end_</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+<b>LrangeCommand: Contents</b><br>&nbsp;&nbsp;<a href="#LRANGE _key_ _start_ _end_">LRANGE _key_ _start_ _end_</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Consistency with range functions in various programming languages">Consistency with range functions in various programming languages</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Out-of-range indexes">Out-of-range indexes</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
                 </div>
                 
                 <h1 class="wikiname">LrangeCommand</h1>
@@ -27,12 +27,17 @@
 
                 <div class="narrow">
                     &iuml;&raquo;&iquest;#sidebar <a href="ListCommandsSidebar.html">ListCommandsSidebar</a><h1><a name="LRANGE _key_ _start_ _end_">LRANGE _key_ _start_ _end_</a></h1>
-<i>Time complexity: O(n) (with n being the length of the range)</i><blockquote>Return the specified elements of the list stored at the specifiedkey. Start and end are zero-based indexes. 0 is the first elementof the list (the list head), 1 the next element and so on.</blockquote>
-<blockquote>For example LRANGE foobar 0 2 will return the first three elementsof the list.</blockquote>
-<blockquote>_start_ and <i>end</i> can also be negative numbers indicating offsetsfrom the end of the list. For example -1 is the last element ofthe list, -2 the penultimate element and so on.</blockquote>
-<blockquote>Indexes out of range will not produce an error: if start is overthe end of the list, or start <code name="code" class="python">&gt;</code> end, an empty list is returned.If end is over the end of the list Redis will threat it just likethe last element of the list.</blockquote>
-<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Multi bulk reply</a>, specifically a list of elements in the specified range.
-
+<i>Time complexity: O(start+n) (with n being the length of the range and start being the start offset)</i>Return the specified elements of the list stored at the specified
+key. Start and end are zero-based indexes. 0 is the first element
+of the list (the list head), 1 the next element and so on.<br/><br/>For example LRANGE foobar 0 2 will return the first three elements
+of the list.<br/><br/><i>start</i> and <i>end</i> can also be negative numbers indicating offsets
+from the end of the list. For example -1 is the last element of
+the list, -2 the penultimate element and so on.<h2><a name="Consistency with range functions in various programming languages">Consistency with range functions in various programming languages</a></h2>Note that if you have a list of numbers from 0 to 100, LRANGE 0 10 will return
+11 elements, that is, rightmost item is included. This <b>may or may not</b> be consistent with
+behavior of range-related functions in your programming language of choice (think Ruby's Range.new, Array#slice or Python's range() function).<br/><br/>LRANGE behavior is consistent with one of Tcl.<h2><a name="Out-of-range indexes">Out-of-range indexes</a></h2>Indexes out of range will not produce an error: if start is over
+the end of the list, or start <code name="code" class="python">&gt;</code> end, an empty list is returned.
+If end is over the end of the list Redis will threat it just like
+the last element of the list.<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Multi bulk reply</a>, specifically a list of elements in the specified range.
                 </div>
         
             </div>

doc/LsetCommand.html

@@ -28,8 +28,7 @@
                 <div class="narrow">
                     &iuml;&raquo;&iquest;#sidebar <a href="ListCommandsSidebar.html">ListCommandsSidebar</a><h1><a name="LSET _key_ _index_ _value_">LSET _key_ _index_ _value_</a></h1>
 <i>Time complexity: O(N) (with N being the length of the list)</i><blockquote>Set the list element at <i>index</i> (see LINDEX for information about the_index_ argument) with the new <i>value</i>. Out of range indexes willgenerate an error. Note that setting the first or last elements ofthe list is O(1).</blockquote>
-<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Status code reply</a>
-
+<blockquote>Similarly to other list commands accepting indexes, the index can be negative to access elements starting from the end of the list. So -1 is the last element, -2 is the penultimate, and so forth.</blockquote><h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Status code reply</a>
                 </div>
         
             </div>

doc/MultiExecCommand.html

@@ -0,0 +1,167 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>MultiExecCommand: Contents</b><br>&nbsp;&nbsp;<a href="#WATCH key1 key2 ... keyN (Redis &gt;">WATCH key1 key2 ... keyN (Redis &gt;</a><br>&nbsp;&nbsp;<a href="#UNWATCH">UNWATCH</a><br>&nbsp;&nbsp;<a href="#MULTI">MULTI</a><br>&nbsp;&nbsp;<a href="#COMMAND_1 ...">COMMAND_1 ...</a><br>&nbsp;&nbsp;<a href="#COMMAND_2 ...">COMMAND_2 ...</a><br>&nbsp;&nbsp;<a href="#COMMAND_N ...">COMMAND_N ...</a><br>&nbsp;&nbsp;<a href="#EXEC or DISCARD">EXEC or DISCARD</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Usage">Usage</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#The DISCARD command">The DISCARD command</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Check and Set (CAS) transactions using WATCH">Check and Set (CAS) transactions using WATCH</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#WATCH explained">WATCH explained</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#WATCH used to implement ZPOP">WATCH used to implement ZPOP</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+                </div>
+                
+                <h1 class="wikiname">MultiExecCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;#sidebar <a href="GenericCommandsSidebar.html">GenericCommandsSidebar</a><h1><a name="WATCH key1 key2 ... keyN (Redis &gt;">WATCH key1 key2 ... keyN (Redis &gt;</a></h1> 2.1.0)=
+<h1><a name="UNWATCH">UNWATCH</a></h1>
+<h1><a name="MULTI">MULTI</a></h1>
+<h1><a name="COMMAND_1 ...">COMMAND_1 ...</a></h1>
+<h1><a name="COMMAND_2 ...">COMMAND_2 ...</a></h1>
+<h1><a name="COMMAND_N ...">COMMAND_N ...</a></h1>
+<h1><a name="EXEC or DISCARD">EXEC or DISCARD</a></h1>MULTI, EXEC, DISCARD and WATCH commands are the fundation of Redis Transactions.
+A Redis Transaction allows the execution of a group of Redis commands in a single
+step, with two important guarantees:<br/><br/><ul><li> All the commands in a transaction are serialized and executed sequentially. It can never happen that a request issued by another client is served <b>in the middle</b> of the execution of a Redis transaction. This guarantees that the commands are executed as a single atomic operation.</li><li> Either all of the commands or none are processed. The EXEC command triggers the execution of all the commands in the transaction, so if a client loses the connection to the server in the context of a transaction before calling the MULTI command none of the operations are performed, instead if the EXEC command is called, all the operations are performed. An exception to this rule is when the Append Only File is enabled: every command that is part of a Redis transaction will log in the AOF as long as the operation is completed, so if the Redis server crashes or is killed by the system administrator in some hard way it is possible that only a partial number of operations are registered.</li></ul>
+Since Redis 2.1.0, it's also possible to add a further guarantee to the above two, in the form of optimistic locking of a set of keys in a way very similar to a CAS (check and set) operation. This is documented later in this manual page.<h2><a name="Usage">Usage</a></h2>A Redis transaction is entered using the MULTI command. The command always
+replies with OK. At this point the user can issue multiple commands. Instead
+to execute this commands Redis will &quot;queue&quot; them. All the commands are
+executed once EXEC is called.<br/><br/>Calling DISCARD instead will flush the transaction queue and will exit
+the transaction.<br/><br/>The following is an example using the Ruby client:
+<pre class="codeblock python" name="code">
+?&gt; r.multi
+=&gt; &quot;OK&quot;
+&gt;&gt; r.incr &quot;foo&quot;
+=&gt; &quot;QUEUED&quot;
+&gt;&gt; r.incr &quot;bar&quot;
+=&gt; &quot;QUEUED&quot;
+&gt;&gt; r.incr &quot;bar&quot;
+=&gt; &quot;QUEUED&quot;
+&gt;&gt; r.exec
+=&gt; [1, 1, 2]
+</pre>
+As it is possible to see from the session above, MULTI returns an &quot;array&quot; of
+replies, where every element is the reply of a single command in the
+transaction, in the same order the commands were queued.<br/><br/>When a Redis connection is in the context of a MULTI request, all the commands
+will reply with a simple string &quot;QUEUED&quot; if they are correct from the
+point of view of the syntax and arity (number of arguments) of the commaand.
+Some command is still allowed to fail during execution time.<br/><br/>This is more clear if at protocol level: in the following example one command
+will fail when executed even if the syntax is right:
+<pre class="codeblock python python" name="code">
+Trying 127.0.0.1...
+Connected to localhost.
+Escape character is '^]'.
+MULTI
++OK
+SET a 3 
+abc
++QUEUED
+LPOP a
++QUEUED
+EXEC
+*2
++OK
+-ERR Operation against a key holding the wrong kind of value
+</pre>
+MULTI returned a two elements bulk reply in witch one of this is a +OK
+code and one is a -ERR reply. It's up to the client lib to find a sensible
+way to provide the error to the user.<br/><br/><blockquote>IMPORTANT: even when a command will raise an error, all the other commandsin the queue will be processed. Redis will NOT stop the processing ofcommands once an error is found.</blockquote>
+Another example, again using the write protocol with telnet, shows how
+syntax errors are reported ASAP instead:
+<pre class="codeblock python python python" name="code">
+MULTI
++OK
+INCR a b c
+-ERR wrong number of arguments for 'incr' command
+</pre>
+This time due to the syntax error the &quot;bad&quot; INCR command is not queued
+at all.<h2><a name="The DISCARD command">The DISCARD command</a></h2>DISCARD can be used in order to abort a transaction. No command will be
+executed, and the state of the client is again the normal one, outside
+<blockquote>of a transaction. Example using the Ruby client:</blockquote><pre class="codeblock python python python python" name="code">
+?&gt; r.set(&quot;foo&quot;,1)
+=&gt; true
+&gt;&gt; r.multi
+=&gt; &quot;OK&quot;
+&gt;&gt; r.incr(&quot;foo&quot;)
+=&gt; &quot;QUEUED&quot;
+&gt;&gt; r.discard
+=&gt; &quot;OK&quot;
+&gt;&gt; r.get(&quot;foo&quot;)
+=&gt; &quot;1&quot;
+</pre><h2><a name="Check and Set (CAS) transactions using WATCH">Check and Set (CAS) transactions using WATCH</a></h2>WATCH is used in order to provide a CAS (Check and Set) behavior to
+Redis Transactions.<br/><br/>WATCHed keys are monitored in order to detect changes against this keys.
+If at least a watched key will be modified before the EXEC call, the
+whole transaction will abort, and EXEC will return a nil object
+(A Null Multi Bulk reply) to notify that the transaction failed.<br/><br/>For example imagine we have the need to atomically increment the value
+of a key by 1 (I know we have INCR, let's suppose we don't have it).<br/><br/>The first try may be the following:
+<pre class="codeblock python python python python python" name="code">
+val = GET mykey
+val = val + 1
+SET mykey $val
+</pre>
+This will work reliably only if we have a single client performing the operation in a given time.
+If multiple clients will try to increment the key about at the same time
+there will be a race condition. For instance client A and B will read the
+old value, for instance, 10. The value will be incremented to 11 by both
+the clients, and finally SET as the value of the key. So the final value
+will be &quot;11&quot; instead of &quot;12&quot;.<br/><br/>Thanks to WATCH we are able to model the problem very well:
+<pre class="codeblock python python python python python python" name="code">
+WATCH mykey
+val = GET mykey
+val = val + 1
+MULTI
+SET mykey $val
+EXEC
+</pre>
+Using the above code, if there are race conditions and another client
+modified the result of <i>val</i> in the time between our call to WATCH and
+our call to EXEC, the transaction will fail.<br/><br/>We'll have just to re-iterate the operation hoping this time we'll not get
+a new race. This form of locking is called <b>optimistic locking</b> and is
+a very powerful form of locking as in many problems there are multiple
+clients accessing a much bigger number of keys, so it's very unlikely that
+there are collisions: usually operations don't need to be performed
+multiple times.<h2><a name="WATCH explained">WATCH explained</a></h2>So what is WATCH really about? It is a command that will make the EXEC
+conditional: we are asking Redis to perform the transaction only if no
+other client modified any of the WATCHed keys. Otherwise the transaction is not
+entered at all. (Note that if you WATCH a volatile key and Redis expires the key after you WATCHed it, EXEC will still work. <a href="http://code.google.com/p/redis/issues/detail?id=270" target="_blank">More</a>.)<br/><br/>WATCH can be called multiple times. Simply all the WATCH calls will
+have the effects to watch for changes starting from the call, up to the
+moment EXEC is called.<br/><br/>When EXEC is called, either if it will fail or succeed, all keys are
+UNWATCHed. Also when a client connection is closed, everything gets
+UNWATCHed.<br/><br/>It is also possible to use the UNWATCH command (without arguments) in order
+to flush all the watched keys. Sometimes this is useful as we
+optimistically lock a few keys, since possibly we need to perform a transaction
+to alter those keys, but after reading the current content of the keys
+we don't want to proceed. When this happens we just call UNWATCH so that
+the connection can already be used freely for new transactions.<h2><a name="WATCH used to implement ZPOP">WATCH used to implement ZPOP</a></h2>A good example to illustrate how WATCH can be used to create new atomic
+operations otherwise not supported by Redis is to implement ZPOP, that is
+a command that pops the element with the lower score from a sorted set
+in an atomic way. This is the simplest implementation:
+<pre class="codeblock python python python python python python python" name="code">
+WATCH zset
+ele = ZRANGE zset 0 0
+MULTI
+ZREM zset ele
+EXEC
+</pre>
+If EXEC fails (returns a nil value) we just re-iterate the operation.<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Multi bulk reply</a>, specifically:<br/><br/><pre class="codeblock python python python python python python python python" name="code">
+The result of a MULTI/EXEC command is a multi bulk reply where every element is the return value of every command in the atomic transaction.
+</pre>If a MULTI/EXEC transaction is aborted because of WATCH detected modified keys, a <a href="ReplyTypes.html">Null Multi Bulk reply</a> is returned.
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/PublishSubscribe.html

@@ -0,0 +1,116 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>PublishSubscribe: Contents</b><br>&nbsp;&nbsp;<a href="#UNSUBSCRIBE channel_1 channel_2 ... channel_N">UNSUBSCRIBE channel_1 channel_2 ... channel_N</a><br>&nbsp;&nbsp;<a href="#UNSUBSCRIBE (unsubscribe from all channels)">UNSUBSCRIBE (unsubscribe from all channels)</a><br>&nbsp;&nbsp;<a href="#PSUBSCRIBE pattern_1 pattern_2 ... pattern_N">PSUBSCRIBE pattern_1 pattern_2 ... pattern_N</a><br>&nbsp;&nbsp;<a href="#PUNSUBSCRIBE pattern_1 pattern_2 ... pattern_N">PUNSUBSCRIBE pattern_1 pattern_2 ... pattern_N</a><br>&nbsp;&nbsp;<a href="#PUNSUBSCRIBE (unsubscribe from all patterns)">PUNSUBSCRIBE (unsubscribe from all patterns)</a><br>&nbsp;&nbsp;<a href="#PUBLISH channel message">PUBLISH channel message</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Format of pushed messages">Format of pushed messages</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Unsubscribing from all the channels at once">Unsubscribing from all the channels at once</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Wire protocol example">Wire protocol example</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#PSUBSCRIBE and PUNSUBSCRIBE: pattern matching subscriptions">PSUBSCRIBE and PUNSUBSCRIBE: pattern matching subscriptions</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Messages matching both a pattern and a channel subscription">Messages matching both a pattern and a channel subscription</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#The meaning of the count of subscriptions with pattern matching">The meaning of the count of subscriptions with pattern matching</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#More details on the PUBLISH command">More details on the PUBLISH command</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Programming Example">Programming Example</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Client library implementations hints">Client library implementations hints</a>
+                </div>
+                
+                <h1 class="wikiname">PublishSubscribe</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;=SUBSCRIBE channel_1 channel_2 ... channel_N=
+<h1><a name="UNSUBSCRIBE channel_1 channel_2 ... channel_N">UNSUBSCRIBE channel_1 channel_2 ... channel_N</a></h1>
+<h1><a name="UNSUBSCRIBE (unsubscribe from all channels)">UNSUBSCRIBE (unsubscribe from all channels)</a></h1>
+<h1><a name="PSUBSCRIBE pattern_1 pattern_2 ... pattern_N">PSUBSCRIBE pattern_1 pattern_2 ... pattern_N</a></h1>
+<h1><a name="PUNSUBSCRIBE pattern_1 pattern_2 ... pattern_N">PUNSUBSCRIBE pattern_1 pattern_2 ... pattern_N</a></h1>
+<h1><a name="PUNSUBSCRIBE (unsubscribe from all patterns)">PUNSUBSCRIBE (unsubscribe from all patterns)</a></h1>
+<h1><a name="PUBLISH channel message">PUBLISH channel message</a></h1>Time complexity: subscribe is O(1), unsubscribe is O(N) where N is the number of clients already subscribed to a channel, publish is O(N+M) where N is the number of clients subscribed to the receiving channel, and M is the total number of subscribed patterns (by any client). Psubscribe is O(N) where N is the number of patterns the Psubscribing client is already subscribed to. Punsubscribe is O(N+M) where N is the number of patterns the Punsubscribing client is already subscribed and M is the number of total patterns subscribed in the system (by any client).<br/><br/><blockquote>SUBSCRIBE, UNSUBSCRIBE and PUBLISH commands implement the<a href="http://en.wikipedia.org/wiki/Publish/subscribe" target="_blank">Publish/Subscribe messaging paradigm</a> where (citing Wikipedia) senders (publishers) are not programmed to send their messages to specific receivers (subscribers). Rather, published messages are characterized into channels, without knowledge of what (if any) subscribers there may be. Subscribers express interest in one or more channels, and only receive messages that are of interest, without knowledge of what (if any) publishers there are. This decoupling of publishers and subscribers can allow for greater scalability and a more dynamic network topology.</blockquote>
+<blockquote>For instance in order to subscribe to the channels foo and bar the clientwill issue the SUBSCRIBE command followed by the names of the channels.</blockquote><pre class="codeblock python" name="code">
+SUBSCRIBE foo bar
+</pre>
+<blockquote>All the messages sent by other clients to this channels will be pushed bythe Redis server to all the subscribed clients, in the form of a threeelements bulk reply, where the first element is the message type, thesecond the originating channel, and the third argument the message payload.</blockquote>
+<blockquote>A client subscribed to 1 or more channels should NOT issue other commandsother than SUBSCRIBE and UNSUBSCRIBE, but can subscribe or unsubscribeto other channels dynamically.</blockquote>
+<blockquote>The reply of the SUBSCRIBE and UNSUBSCRIBE operations are sent in the formof messages, so that the client can just read a coherent stream of messageswhere the first element indicates the kind of message.</blockquote><h2><a name="Format of pushed messages">Format of pushed messages</a></h2>
+<blockquote>Messages are in the form of multi bulk replies with three elements.The first element is the kind of message:</blockquote><ul><li> &quot;subscribe&quot;: means that we successfully subscribed to the channel given as second element of the multi bulk reply. The third argument represents the number of channels we are currently subscribed to.</li><li> &quot;unsubscribe&quot;: means that we successfully unsubscribed from the channel given as second element of the multi bulk reply. The third argument represents the number of channels we are currently subscribed to. If this latest argument is zero, we are no longer subscribed to any channel, and the client can issue any kind of Redis command as we are outside the Pub/sub state.</li><li> &quot;message&quot;: it is a message received as result of a PUBLISH command issued by another client. The second element is the name of the originating channel, and the third the actual message payload.</li></ul><h2><a name="Unsubscribing from all the channels at once">Unsubscribing from all the channels at once</a></h2>
+If the UNSUBSCRIBE command is issued without additional arguments, it is equivalent to unsubscribing to all the channels we are currently subscribed. A message for every unsubscribed channel will be received.
+<h2><a name="Wire protocol example">Wire protocol example</a></h2>
+<pre class="codeblock python python" name="code">
+SUBSCRIBE first second
+*3
+$9
+subscribe
+$5
+first
+:1
+*3
+$9
+subscribe
+$6
+second
+:2
+</pre>
+at this point from another client we issue a PUBLISH operation against the channel named &quot;second&quot;. This is what the first client receives:
+<pre class="codeblock python python python" name="code">
+*3
+$7
+message
+$6
+second
+$5
+Hello
+</pre>
+Now the client unsubscribes itself from all the channels using the UNSUBSCRIBE command without additional arguments:
+<pre class="codeblock python python python python" name="code">
+UNSUBSCRIBE
+*3
+$11
+unsubscribe
+$6
+second
+:1
+*3
+$11
+unsubscribe
+$5
+first
+:0
+</pre>
+<h2><a name="PSUBSCRIBE and PUNSUBSCRIBE: pattern matching subscriptions">PSUBSCRIBE and PUNSUBSCRIBE: pattern matching subscriptions</a></h2>
+Redis Pub/Sub implementation supports pattern matching. Clients may subscribe to glob style patterns in order to receive all the messages sent to channel names matching a given pattern.<br/><br/>For instance the command:
+<pre class="codeblock python python python python python" name="code">
+PSUBSCRIBE news.*
+</pre>
+Will receive all the messages sent to the channel news.art.figurative and news.music.jazz and so forth. All the glob style patterns as valid, so multiple wild cards are supported.<br/><br/>Messages received as a result of pattern matching are sent in a different format:
+<ul><li> The type of the message is &quot;pmessage&quot;: it is a message received as result of a PUBLISH command issued by another client, matching a pattern matching subscription. The second element is the original pattern matched, the third element is the name of the originating channel, and the last element the actual message payload.</li></ul>
+Similarly to SUBSCRIBE and UNSUBSCRIBE, PSUBSCRIBE and PUNSUBSCRIBE commands are acknowledged by the system sending a message of type &quot;psubscribe&quot; and &quot;punsubscribe&quot; using the same format as the &quot;subscribe&quot; and &quot;unsubscribe&quot; message format.
+<h2><a name="Messages matching both a pattern and a channel subscription">Messages matching both a pattern and a channel subscription</a></h2>
+A client may receive a single message multiple time if it's subscribed to multiple patterns matching a published message, or it is subscribed to both patterns and channels matching the message. Like in the following example:
+<pre class="codeblock python python python python python python" name="code">
+SUBSCRIBE foo
+PSUBSCRIBE f*
+</pre>
+In the above example, if a message is sent to the <b>foo</b> channel, the client will receive two messages, one of type &quot;message&quot; and one of type &quot;pmessage&quot;.
+<h2><a name="The meaning of the count of subscriptions with pattern matching">The meaning of the count of subscriptions with pattern matching</a></h2>
+In <b>subscribe</b>, <b>unsubscribe</b>, <b>psubscribe</b> and <b>punsubscribe</b> message types, the last argument is the count of subscriptions still active. This number is actually the total number of channels and patterns the client is still subscribed to. So the client will exit the Pub/Sub state only when this count will drop to zero as a result of unsubscription from all the channels and patterns.
+<h2><a name="More details on the PUBLISH command">More details on the PUBLISH command</a></h2>
+The Publish command is a bulk command where the first argument is the target class, and the second argument the data to send. It returns an Integer Reply representing the number of clients that received the message (that is, the number of clients that were listening for this class).
+<h2><a name="Programming Example">Programming Example</a></h2>
+Pieter Noordhuis provided a great example using Event-machine and Redis to create <a href="http://chat.redis-db.com" target="_blank">a multi user high performance web chat</a>, with source code included of course!
+<h2><a name="Client library implementations hints">Client library implementations hints</a></h2>
+Because all the messages received contain the original subscription causing the message delivery (the channel in the case of &quot;message&quot; type, and the original pattern in the case of &quot;pmessage&quot; type) clinet libraries may bind the original subscription to callbacks (that can be anonymous functions, blocks, function pointers, and so forth), using an hash table.<br/><br/>When a message is received an O(1) lookup can be done in order to deliver the message to the registered callback.
+
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/QuickStart.html

@@ -16,7 +16,7 @@
             <div id="pagecontent">
                 <div class="index">
 <!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
-<b>QuickStart: Contents</b><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Obtain the latest version">Obtain the latest version</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Compile">Compile</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Run the server">Run the server</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Play with the built in client">Play with the built in client</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Further reading">Further reading</a>
+<b>QuickStart: Contents</b><br>&nbsp;&nbsp;<a href="#Quick Start">Quick Start</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Obtain the latest version">Obtain the latest version</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Compile">Compile</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Run the server">Run the server</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Play with the built in client">Play with the built in client</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Further reading">Further reading</a>
                 </div>
                 
                 <h1 class="wikiname">QuickStart</h1>
@@ -26,7 +26,8 @@
                 </div>
 
                 <div class="narrow">
-                    &iuml;&raquo;&iquest;= Quick Start =<br/><br/>This quickstart is a five minutes howto on how to get started with Redis. For more information on Redis check <a href="http://code.google.com/p/redis/wiki/index" target="_blank">Redis Documentation Index</a>.<h2><a name="Obtain the latest version">Obtain the latest version</a></h2>The latest stable source distribution of Redis can be obtained <a href="http://code.google.com/p/redis/downloads/list" target="_blank">at this location as a tarball</a>.<br/><br/><pre class="codeblock python" name="code">
+                    &iuml;&raquo;&iquest;#sidebar <a href="RedisGuides.html">RedisGuides</a>
+<h1><a name="Quick Start">Quick Start</a></h1>This quickstart is a five minutes howto on how to get started with Redis. For more information on Redis check <a href="http://code.google.com/p/redis/wiki/index" target="_blank">Redis Documentation Index</a>.<h2><a name="Obtain the latest version">Obtain the latest version</a></h2>The latest stable source distribution of Redis can be obtained <a href="http://code.google.com/p/redis/downloads/list" target="_blank">at this location as a tarball</a>.<br/><br/><pre class="codeblock python" name="code">
 $ wget http://redis.googlecode.com/files/redis-1.02.tar.gz
 </pre>The unstable source code, with more features but not ready for production, can be downloaded using git:<br/><br/><pre class="codeblock python python" name="code">
 $ git clone git://github.com/antirez/redis.git

doc/README.html

@@ -26,11 +26,42 @@
                 </div>
 
                 <div class="narrow">
-                    &iuml;&raquo;&iquest;= Introduction =<br/><br/>Redis is a database. To be specific, Redis is a database implementing a dictionary, where every key is associated with a value. For example I can set the key &quot;surname_1992&quot; to the string &quot;Smith&quot;.
-What makes Redis different from many other key-value stores, is that every single value has a type. The following types are supported:<br/><br/><ul><li> <a href="Strings.html">Strings</a></li><li> <a href="Lists.html">Lists</a></li><li> <a href="Sets.html">Sets</a></li><li> <a href="SortedSets.html">Sorted Set</a> (since version 1.1)</li></ul>
-The type of a value determines what operations (called commands) are available for the value itself. 
-For example you can append elements to a list stored at the key &quot;mylist&quot; using the LPUSH or RPUSH command in O(1). Later you'll be able to get a range of elements with LRANGE or trim the list with LTRIM. Sets are very flexible too, it is possible to add and remove elements from Sets (unsorted collections of strings), and then ask for server-side intersection, union, difference of Sets. Each command is performed through server-side atomic operations.
-Please refer to the <a href="CommandReference.html">Command Reference</a> to see the full list of operations associated to these data types.<br/><br/>In other words, you can look at Redis as a data structures server. A Redis user is virtually provided with an interface to  <a href="http://en.wikipedia.org/wiki/Abstract_data_type" target="_blank">Abstract Data Types</a>, saving her from the responsibility to implement concrete data structures and algorithms. Indeed both algorithms and data structures in Redis are properly choosed in order to obtain the best performance.<h1><a name="All data in memory, but saved on disk">All data in memory, but saved on disk</a></h1>Redis loads and mantains the whole dataset into memory, but the dataset is persistent, since at the same time it is saved on disk, so that when the server is restarted data can be loaded back in memory.<br/><br/>There are two kind of persistence supported: the first one is called snapshotting. In this mode Redis, from time to time, writes a dump on disk asynchronously. The dataset is loaded from the dump every time the server is (re)started.<br/><br/>Redis can be configured to save the dataset when a certain number of changes is reached and after a given number of seconds elapses. For example, you can configure Redis to save after 1000 changes and at most 60 seconds since the last save. You can specify any combination for these numbers.<br/><br/>Because data is written asynchronously, when a system crash occurs, the last few queries can get lost (that is acceptable in many applications but not in all).  In order to make this a non issue Redis supports another, safer persistence mode, called <a href="AppendOnlyFileHowto.html">Append Only File</a>, where every command received altering the dataset (so not a read-only command, but a write command) is written on an append only file ASAP. This commands are <i>replayed</i> when the server is restarted in order to rebuild the dataset in memory.<br/><br/>Redis Append Only File supports a very handy feature: the server is able to safely rebuild the append only file in background in a non-blocking fashion when it gets too long. You can find <a href="AppendOnlyFileHowto.html">more details in the Append Only File HOWTO</a>.<h1><a name="Master-Slave replication made trivial">Master-Slave replication made trivial</a></h1>Whatever will be the persistence mode you'll use Redis supports master-slave replications if you want to stay really safe or if you need to scale to huge amounts of reads.<br/><br/><b>Redis Replication is trivial to setup</b>. So trivial that all you need to do in order to configure a Redis server to be a slave of another one, with automatic synchronization if the link will go down and so forth, is the following config line: <code name="code" class="python">slaveof 192.168.1.100 6379</code>. <a href="ReplicationHowto.html">We provide a Replication Howto</a> if you want to know more about this feature.<h1><a name="It's persistent but supports expires">It's persistent but supports expires</a></h1>Redis can be used as a <b>memcached on steroids</b> because is as fast as memcached but with a number of features more. Like memcached, Redis also supports setting timeouts to keys so that this key will be automatically removed when a given amount of time passes.<h1><a name="Beyond key-value databases">Beyond key-value databases</a></h1>All these features allow to use Redis as the sole DB for your scalable application without the need of any relational database. <a href="TwitterAlikeExample.html">We wrote a simple Twitter clone in PHP + Redis</a> to show a real world example, the link points to an article explaining the design and internals in very simple words.<h1><a name="Multiple databases support">Multiple databases support</a></h1>Redis supports multiple databases with commands to atomically move keys from one database to the other. By default DB 0 is selected for every new connection, but using the SELECT command it is possible to select a different database. The MOVE operation can move an item from one DB to another atomically. This can be used as a base for locking free algorithms together with the 'RANDOMKEY' commands.<h1><a name="Know more about Redis!">Know more about Redis!</a></h1>To really get a feeling about what Redis is and how it works please try reading <a href="IntroductionToRedisDataTypes.html">A fifteen minutes introduction to Redis data types</a>.<br/><br/>To know a bit more about how Redis works <i>internally</i> continue reading.<h1><a name="Redis Tutorial">Redis Tutorial</a></h1>(note, you can skip this section if you are only interested in &quot;formal&quot; doc.)<br/><br/>Later in this document you can find detailed information about Redis commands,
+                    &iuml;&raquo;&iquest;= Introduction =<br/><br/>Redis is an extremely fast and powerful key-value store database and server implemented in ANSI C.  Redis offers many different ways to do one straightforward thing: store a value (&quot;antirez&quot;) to a key (&quot;redis&quot;).  While the format of keys must always be simple strings, the power is with the values, which support the following data types:<br/><br/><ul><li> <a href="Strings.html">Strings</a></li><li> <a href="Lists.html">Lists</a></li><li> <a href="Sets.html">Sets</a></li><li> <a href="SortedSets.html">Sorted Sets (zsets)</a></li><li> <a href="Hashes.html">Hashes</a></li></ul>
+Each value type has an associated list of commands which can operate on them, and the <a href="CommandReference.html">The Redis Command Reference</a> contains an up to date list of these commands, organized primarily by data type.  The Redis source also includes a <a href="RedisCLI.html">Redis command line interface</a> which allows you to interact directly with the server, and is the means by which this introduction will provide examples.  Once you walk through the <a href="QuickStart.html">Redis Quick Start Guide</a> to get your instance of Redis running, you can follow along.  <br/><br/>One of the most powerful aspects of Redis is the wide range of commands which are optimized to work with specific data value types and executed as atomic server-side operations.  The <a href="Lists.html">List</a> type is a great example - Redis implements O(1) operations such as <a href="RpushCommand.html">LPUSH</a> or <a href="RpushCommand.html">RPUSH</a>, which have accompanying <a href="LpopCommand.html">LPOP</a> and <a href="LpopCommand.html">RPOP</a> methods:<br/><br/><pre class="codeblock python" name="code">
+redis&gt; lpush programming_languages C
+OK
+redis&gt; lpush programming_languages Ruby
+OK
+redis&gt; rpush programming_languages Python
+OK
+redis&gt; rpop programming_languages
+Python
+redis&gt; lpop programming_languages
+Ruby
+</pre>More complex operations are available for each data type as well.  Continuing with lists, you can get a range of elements with <a href="LrangeCommand.html">LRANGE</a> (O(start+n)) or trim the list with <a href="LtrimCommand.html">LTRIM</a> (O(n)):<br/><br/><pre class="codeblock python python" name="code">
+redis&gt; lpush cities NYC
+OK
+redis&gt; lpush cities SF
+OK
+redis&gt; lpush cities Tokyo
+OK
+redis&gt; lpush cities London
+OK
+redis&gt; lpush cities Paris
+OK
+redis&gt; lrange cities 0 2
+1. Paris
+2. London
+3. Tokyo
+redis&gt; ltrim cities 0 1
+OK
+redis&gt; lpop cities
+Paris
+redis&gt; lpop cities
+London
+redis&gt; lpop cities
+(nil)
+</pre>You can also add and remove elements from a set, and perform intersections, unions, and differences. <br/><br/>Redis can also be looked at as a data structures server. A Redis user is virtually provided with an interface to  <a href="http://en.wikipedia.org/wiki/Abstract_data_type" target="_blank">Abstract Data Types</a>, saving them from the responsibility of implementing concrete data structures and algorithms -- indeed both algorithms and data structures in Redis are properly chosen in order to obtain the best performance.<h1><a name="All data in memory, but saved on disk">All data in memory, but saved on disk</a></h1>Redis loads and mantains the whole dataset into memory, but the dataset is persistent, since at the same time it is saved on disk, so that when the server is restarted data can be loaded back in memory.<br/><br/>There are two kinds of persistence supported: the first one is called snapshotting. In this mode Redis periodically writes to disk asynchronously. The dataset is loaded from the dump every time the server is (re)started.<br/><br/>Redis can be configured to save the dataset when a certain number of changes is reached and after a given number of seconds elapses. For example, you can configure Redis to save after 1000 changes and at most 60 seconds since the last save. You can specify any combination for these numbers.<br/><br/>Because data is written asynchronously, when a system crash occurs, the last few queries can get lost (that is acceptable in many applications but not in all).  In order to make this a non issue Redis supports another, safer persistence mode, called <a href="AppendOnlyFileHowto.html">Append Only File</a>, where every command received altering the dataset (so not a read-only command, but a write command) is written on an append only file ASAP. This commands are <i>replayed</i> when the server is restarted in order to rebuild the dataset in memory.<br/><br/>Redis Append Only File supports a very handy feature: the server is able to safely rebuild the append only file in background in a non-blocking fashion when it gets too long. You can find <a href="AppendOnlyFileHowto.html">more details in the Append Only File HOWTO</a>.<h1><a name="Master-Slave replication made trivial">Master-Slave replication made trivial</a></h1>Whatever will be the persistence mode you'll use Redis supports master-slave replications if you want to stay really safe or if you need to scale to huge amounts of reads.<br/><br/><b>Redis Replication is trivial to setup</b>. So trivial that all you need to do in order to configure a Redis server to be a slave of another one, with automatic synchronization if the link will go down and so forth, is the following config line: <code name="code" class="python">slaveof 192.168.1.100 6379</code>. <a href="ReplicationHowto.html">We provide a Replication Howto</a> if you want to know more about this feature.<h1><a name="It's persistent but supports expires">It's persistent but supports expires</a></h1>Redis can be used as a <b>memcached on steroids</b> because is as fast as memcached but with a number of features more. Like memcached, Redis also supports setting timeouts to keys so that this key will be automatically removed when a given amount of time passes.<h1><a name="Beyond key-value databases">Beyond key-value databases</a></h1>All these features allow to use Redis as the sole DB for your scalable application without the need of any relational database. <a href="TwitterAlikeExample.html">We wrote a simple Twitter clone in PHP + Redis</a> to show a real world example, the link points to an article explaining the design and internals in very simple words.<h1><a name="Multiple databases support">Multiple databases support</a></h1>Redis supports multiple databases with commands to atomically move keys from one database to the other. By default DB 0 is selected for every new connection, but using the SELECT command it is possible to select a different database. The MOVE operation can move an item from one DB to another atomically. This can be used as a base for locking free algorithms together with the 'RANDOMKEY' commands.<h1><a name="Know more about Redis!">Know more about Redis!</a></h1>To really get a feeling about what Redis is and how it works please try reading <a href="IntroductionToRedisDataTypes.html">A fifteen minutes introduction to Redis data types</a>.<br/><br/>To know a bit more about how Redis works <i>internally</i> continue reading.<h1><a name="Redis Tutorial">Redis Tutorial</a></h1>(note, you can skip this section if you are only interested in &quot;formal&quot; doc.)<br/><br/>Later in this document you can find detailed information about Redis commands,
 the protocol specification, and so on. This kind of documentation is useful
 but... if you are new to Redis it is also BORING! The Redis protocol is designed
 so that is both pretty efficient to be parsed by computers, but simple enough
@@ -40,7 +71,7 @@ feeling about it, and how it works.<br/><br/>To start just compile redis with 'm
 The server will start and log stuff on the standard output, if you want
 it to log more edit redis.conf, set the loglevel to debug, and restart it.<br/><br/>You can specify a configuration file as unique parameter:<br/><br/><blockquote>./redis-server /etc/redis.conf</blockquote>
 This is NOT required. The server will start even without a configuration file
-using a default built-in configuration.<br/><br/>Now let's try to set a key to a given value:<br/><br/><pre class="codeblock python" name="code">
+using a default built-in configuration.<br/><br/>Now let's try to set a key to a given value:<br/><br/><pre class="codeblock python python python" name="code">
 $ telnet localhost 6379
 Trying 127.0.0.1...
 Connected to localhost.
@@ -59,17 +90,17 @@ the point of view of both the server and client but allows us to play with
 Redis with the telnet command easily.<br/><br/>The last line of the chat between server and client is &quot;+OK&quot;. This means
 our key was added without problems. Actually SET can never fail but
 the &quot;+OK&quot; sent lets us know that the server received everything and
-the command was actually executed.<br/><br/>Let's try to get the key content now:<br/><br/><pre class="codeblock python python" name="code">
+the command was actually executed.<br/><br/>Let's try to get the key content now:<br/><br/><pre class="codeblock python python python python" name="code">
 GET foo
 $3
 bar
 </pre>Ok that's very similar to 'set', just the other way around. We sent &quot;get foo&quot;,
 the server replied with a first line that is just the $ character follwed by
 the number of bytes the value stored at key contained, followed by the actual
-bytes. Again &quot;\r\n&quot; are appended both to the bytes count and the actual data. In Redis slang this is called a bulk reply.<br/><br/>What about requesting a non existing key?<br/><br/><pre class="codeblock python python python" name="code">
+bytes. Again &quot;\r\n&quot; are appended both to the bytes count and the actual data. In Redis slang this is called a bulk reply.<br/><br/>What about requesting a non existing key?<br/><br/><pre class="codeblock python python python python python" name="code">
 GET blabla
 $-1
-</pre>When the key does not exist instead of the length, just the &quot;$-1&quot; string is sent. Since a -1 length of a bulk reply has no meaning it is used in order to specifiy a 'nil' value and distinguish it from a zero length value. Another way to check if a given key exists or not is indeed the EXISTS command:<br/><br/><pre class="codeblock python python python python" name="code">
+</pre>When the key does not exist instead of the length, just the &quot;$-1&quot; string is sent. Since a -1 length of a bulk reply has no meaning it is used in order to specifiy a 'nil' value and distinguish it from a zero length value. Another way to check if a given key exists or not is indeed the EXISTS command:<br/><br/><pre class="codeblock python python python python python python" name="code">
 EXISTS nokey
 :0
 EXISTS foo

doc/RedisBigData.html

@@ -0,0 +1,61 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>RedisBigData: Contents</b><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#BGSAVE and BGREWRITEAOF blocking fork() call">BGSAVE and BGREWRITEAOF blocking fork() call</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Using multiple cores">Using multiple cores</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Splitting data into multiple instances">Splitting data into multiple instances</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#BGSAVE / AOFSAVE memory usage, and copy on write">BGSAVE / AOFSAVE memory usage, and copy on write</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#BGSAVE / AOFSAVE time for big datasets">BGSAVE / AOFSAVE time for big datasets</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Non blocking hash table">Non blocking hash table</a>
+                </div>
+                
+                <h1 class="wikiname">RedisBigData</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;=Redis Big Data: facts and guidelines=<h2><a name="BGSAVE and BGREWRITEAOF blocking fork() call">BGSAVE and BGREWRITEAOF blocking fork() call</a></h2>
+<pre class="codeblock python" name="code">
+fork.c &amp;&amp; ./a.out
+allocated:     1 MB, fork() took 0.000
+allocated:    10 MB, fork() took 0.001
+allocated:   100 MB, fork() took 0.007
+allocated:  1000 MB, fork() took 0.059
+allocated: 10000 MB, fork() took 0.460
+allocated: 20000 MB, fork() took 0.895
+allocated: 30000 MB, fork() took 1.327
+allocated: 40000 MB, fork() took 1.759
+allocated: 50000 MB, fork() took 2.190
+allocated: 60000 MB, fork() took 2.621
+allocated: 70000 MB, fork() took 3.051
+allocated: 80000 MB, fork() took 3.483
+allocated: 90000 MB, fork() took 3.911
+allocated: 100000 MB, fork() took 4.340
+allocated: 110000 MB, fork() took 4.770
+allocated: 120000 MB, fork() took 5.202
+</pre>
+<h2><a name="Using multiple cores">Using multiple cores</a></h2>
+<h2><a name="Splitting data into multiple instances">Splitting data into multiple instances</a></h2>
+<h2><a name="BGSAVE / AOFSAVE memory usage, and copy on write">BGSAVE / AOFSAVE memory usage, and copy on write</a></h2>
+<h2><a name="BGSAVE / AOFSAVE time for big datasets">BGSAVE / AOFSAVE time for big datasets</a></h2>
+<h2><a name="Non blocking hash table">Non blocking hash table</a></h2>
+
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/RedisEventLibrary.html

@@ -0,0 +1,70 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>RedisEventLibrary: Contents</b><br>&nbsp;&nbsp;<a href="#Redis Event Library">Redis Event Library</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Event Loop Initialization">Event Loop Initialization</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#aeCreateEventLoop">aeCreateEventLoop</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#aeCreateTimeEvent">aeCreateTimeEvent</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#aeCreateFileEvent">aeCreateFileEvent</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Event Loop Processing">Event Loop Processing</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#aeProcessEvents">aeProcessEvents</a><br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="#processTimeEvents">processTimeEvents</a>
+                </div>
+                
+                <h1 class="wikiname">RedisEventLibrary</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;#sidebar <a href="RedisInternals.html">RedisInternals</a><h1><a name="Redis Event Library">Redis Event Library</a></h1>Redis implements its own event library. The event library is implemented in <b>ae.c</b>.<br/><br/>The best way to understand how the Redis event library works is to understand how Redis uses it.<h2><a name="Event Loop Initialization">Event Loop Initialization</a></h2>
+<code name="code" class="python">initServer</code> function defined in <b>redis.c</b> initializes the numerous fields of the <code name="code" class="python">redisServer</code> structure variable. One such field is the Redis event loop <code name="code" class="python">el</code>:<br/><br/><pre class="codeblock python" name="code"> 
+aeEventLoop *el 
+</pre><code name="code" class="python">initServer</code> initializes <code name="code" class="python">server.el</code> field by calling <code name="code" class="python">aeCreateEventLoop</code> defined in <b>ae.c</b>. The definition of <code name="code" class="python">aeEventLoop</code> is below:
+<pre class="codeblock python python" name="code">
+typedef struct aeEventLoop 
+{
+    int maxfd;
+    long long timeEventNextId;
+    aeFileEvent events[AE_SETSIZE]; /* Registered events */
+    aeFiredEvent fired[AE_SETSIZE]; /* Fired events */
+    aeTimeEvent *timeEventHead;
+    int stop;
+    void *apidata; /* This is used for polling API specific data */
+    aeBeforeSleepProc *beforesleep;
+} aeEventLoop;
+</pre><h3><a name="aeCreateEventLoop">aeCreateEventLoop</a></h3><code name="code" class="python">aeCreateEventLoop</code> first mallocs aeEventLoop structure then calls ae_epoll.c:aeApiCreate<code name="code" class="python">.
+
+</code>aeApiCreate<code name="code" class="python"> mallocs </code>aeApiState<code name="code" class="python"> that has two fields - </code>epfd<code name="code" class="python"> that holds the epoll file descriptor returned by a call from [http://man.cx/epoll_create%282%29 epoll_create] and </code>events<code name="code" class="python"> that is of type </code>struct epoll_event<code name="code" class="python"> define by the Linux epoll library. The use of the </code>events<code name="code" class="python"> field will be  described later.
+
+Next is 'ae.c:aeCreateTimeEvent</code>. But before that <code name="code" class="python">initServer</code> call <code name="code" class="python">anet.c:anetTcpServer</code> that creates and returns a <i>listening descriptor</i>. The descriptor is listens to <b>port 6379</b> by default. The returned  <i>listening descriptor</i> is stored in <code name="code" class="python">server.fd</code> field.<h3><a name="aeCreateTimeEvent">aeCreateTimeEvent</a></h3><code name="code" class="python">aeCreateTimeEvent</code> accepts the following as parameters:<br/><br/><ul><li> eventLoop: This is <code name="code" class="python">server.el</code> in <b>redis.c</b></li><li> milliseconds: The number of milliseconds from the curent time after which the timer expires.</li><li> proc: Function pointer. Stores the address of the function that has to be called after the timer expires.</li><li> clientData: Mostly NULL.</li><li> finalizerProc: Pointer to the function that has to be called before the timed event is removed from the list of timed events.</li></ul>
+<code name="code" class="python">initServer</code> calls <code name="code" class="python">aeCreateTimeEvent</code> to add a timed event to <code name="code" class="python">timeEventHead</code> field of <code name="code" class="python">server.el</code>. <code name="code" class="python">timeEventHead</code> is a pointer to a list of such timed events. The call to <code name="code" class="python">aeCreateTimeEvent</code> from <code name="code" class="python">redis.c:initServer</code> function is given below:<br/><br/><pre class="codeblock python python python" name="code">
+aeCreateTimeEvent(server.el /*eventLoop*/, 1 /*milliseconds*/, serverCron /*proc*/, NULL /*clientData*/, NULL /*finalizerProc*/);
+</pre><code name="code" class="python">redis.c:serverCron</code> performs many operations that helps keep Redis running properly.<h3><a name="aeCreateFileEvent">aeCreateFileEvent</a></h3>The essence of <code name="code" class="python">aeCreateFileEvent</code> function is to execute <a href="http://man.cx/epoll_ctl" target="_blank">epoll_ctl</a> system call which adds a watch for <code name="code" class="python">EPOLLIN</code> event on the <i>listening descriptor</i> create by <code name="code" class="python">anetTcpServer</code> and associate it with the epoll descriptor created by a call to <code name="code" class="python">aeCreateEventLoop</code>. <br/><br/>Following is an explanation of what precisely <code name="code" class="python">aeCreateFileEvent</code> does when called from <code name="code" class="python">redis.c:initServer</code>.<br/><br/><code name="code" class="python">initServer</code> passes the following arguments to <code name="code" class="python">aeCreateFileEvent</code>:
+<ul><li> server.el: The event loop created by <code name="code" class="python">aeCreateEventLoop</code>. The epoll descriptor is got from server.el. </li><li> server.fd: The <i>listening descriptor</i> that also serves as an index to access the relevant file event structure from the <code name="code" class="python">eventLoop-&gt;events</code> table and store extra information like the callback function.</li><li> AE_READABLE: Signifies that server.fd has to be watched for EPOLLIN event.</li><li> acceptHandler: The function that has to be executed when the event being watched for is ready. This function pointer is stored in <code name="code" class="python">eventLoop-&gt;events[server.fd]-&gt;rfileProc</code>. </li></ul>
+This completes the initialization of Redis event loop.<h2><a name="Event Loop Processing">Event Loop Processing</a></h2><code name="code" class="python">ae.c:aeMain</code> called from <code name="code" class="python">redis.c:main</code> does the job of processing the event loop that is initialized in the previous phase.<br/><br/><code name="code" class="python">ae.c:aeMain</code> calls <code name="code" class="python">ae.c:aeProcessEvents</code> in a while loop that processes pending time and file events.<h3><a name="aeProcessEvents">aeProcessEvents</a></h3><code name="code" class="python">ae.c:aeProcessEvents</code> looks for the time event that will be pending in the smallest amount of time by calling <code name="code" class="python">ae.c:aeSearchNearestTimer</code> on the event loop. In our case there is only one timer event in the event loop that was created by <code name="code" class="python">ae.c:aeCreateTimeEvent</code>. <br/><br/>Remember, that timer event created by <code name="code" class="python">aeCreateTimeEvent</code> has by now probably elapsed because it had a expiry time of one millisecond. Since, the timer has already expired the seconds and microseconds fields of the <code name="code" class="python">tvp</code> timeval structure variable is initialized to zero. <br/><br/>The <code name="code" class="python">tvp</code> structure variable along with the event loop variable is passed to <code name="code" class="python">ae_epoll.c:aeApiPoll</code>.<br/><br/><code name="code" class="python">aeApiPoll</code> functions does a <a href="http://man.cx/epoll_wait" target="_blank">epoll_wait</a> on the epoll descriptor and populates the <code name="code" class="python">eventLoop-&gt;fired</code> table with the details:
+<ul><li> fd: The descriptor that is now ready to do a read/write operation depending on the mask value. The </li><li> mask: The read/write event that can now be performed on the corresponding descriptor.</li></ul>
+<code name="code" class="python">aeApiPoll</code> returns the number of such file events ready for operation. Now to put things in context, if any client has requested for a connection then aeApiPoll would have noticed it and populated the <code name="code" class="python">eventLoop-&gt;fired</code> table with an entry of the descriptor being the <i>listening descriptor</i> and mask being <code name="code" class="python">AE_READABLE</code>.<br/><br/>Now, <code name="code" class="python">aeProcessEvents</code> calls the <code name="code" class="python">redis.c:acceptHandler</code> registered as the callback. <code name="code" class="python">acceptHandler</code> executes [<a href="http://man.cx/accept(2" target="_blank">http://man.cx/accept(2</a>) accept] on the <i>listening descriptor</i> returning a <i>connected descriptor</i> with the client. <code name="code" class="python">redis.c:createClient</code> adds a file event on the <i>connected descriptor</i> through a call to <code name="code" class="python">ae.c:aeCreateFileEvent</code> like below:<br/><br/><pre class="codeblock python python python python" name="code">
+    if (aeCreateFileEvent(server.el, c-&gt;fd, AE_READABLE,
+        readQueryFromClient, c) == AE_ERR) {
+        freeClient(c);
+        return NULL;
+    }
+</pre><code name="code" class="python">c</code> is the <code name="code" class="python">redisClient</code> structure variable and <code name="code" class="python">c-&gt;fd</code> is the connected descriptor.<br/><br/>Next the <code name="code" class="python">ae.c:aeProcessEvent</code> calls <code name="code" class="python">ae.c:processTimeEvents</code><h3><a name="processTimeEvents">processTimeEvents</a></h3><code name="code" class="python">ae.processTimeEvents</code> iterates over list of time events starting at <code name="code" class="python">eventLoop-&gt;timeEventHead</code>.<br/><br/>For every timed event that has elapsed <code name="code" class="python">processTimeEvents</code> calls the registered callback. In this case it calls the only timed event callback registered, that is, <code name="code" class="python">redis.c:serverCron</code>. The callback returns the time in milliseconds after which the callback must be called again. This change is recorded via a call to <code name="code" class="python">ae.c:aeAddMilliSeconds</code> and will be handled on the next iteration of <code name="code" class="python">ae.c:aeMain</code> while loop.<br/><br/>That's all.
+
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/RedisGuides.html

@@ -0,0 +1,37 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>RedisGuides: Contents</b>
+                </div>
+                
+                <h1 class="wikiname">RedisGuides</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;= Redis Guides and Howtos=
+<ul><li> <a href="QuickStart.html">Redis Quick Start</a></li><li> <a href="VirtualMemoryUserGuide.html">Virutal Memory User Guide</a></li><li> <a href="IntroductionToRedisDataTypes.html">A Fifteen Minutes Introduction to the Redis Data Types</a></li><li> <a href="ReplicationHowto.html">The Redis Replication HOWTO</a></li><li> <a href="AppendOnlyFileHowto.html">The Append Only File HOWTO</a></li></ul>
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/RedisInternals.html

@@ -0,0 +1,38 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>RedisInternals: Contents</b><br>&nbsp;&nbsp;<a href="#Redis Internals">Redis Internals</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Redis STRINGS">Redis STRINGS</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Redis Virtual Memory">Redis Virtual Memory</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Redis Event Library">Redis Event Library</a>
+                </div>
+                
+                <h1 class="wikiname">RedisInternals</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    <h1><a name="Redis Internals">Redis Internals</a></h1>This is a source code level documentation of Redis.<h2><a name="Redis STRINGS">Redis STRINGS</a></h2>String is the basic building block of Redis types. <br/><br/>Redis is a key-value store.
+All Redis keys are strings and its also the simplest value type.<br/><br/><blockquote></blockquote>Lists, sets, sorted sets and hashes are other more complex value types and even
+these are composed of strings.<br/><br/><a href="HackingStrings.html">Hacking Strings</a> documents the Redis String implementation details.<h2><a name="Redis Virtual Memory">Redis Virtual Memory</a></h2>A technical specification full of details about the <a href="VirtualMemorySpecification.html">Redis Virtual Memory subsystem</a><h2><a name="Redis Event Library">Redis Event Library</a></h2>Read <a href="EventLibray.html">event library</a> to understand what an event library does and why its needed.<br/><br/><a href="RedisEventLibrary.html">Redis event library</a> documents the implementation details of the event library used by Redis 
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/Redis_1_2_0_Changelog.html

@@ -16,7 +16,7 @@
             <div id="pagecontent">
                 <div class="index">
 <!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
-<b>Redis_1_2_0_Changelog: Contents</b><br>&nbsp;&nbsp;<a href="#CHANGELOG for Redis 1.1.90">CHANGELOG for Redis 1.1.90</a>
+<b>Redis_1_2_0_Changelog: Contents</b><br>&nbsp;&nbsp;<a href="#What's new in Redis 1.2">What's new in Redis 1.2</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#New persistence mode: Append Only File">New persistence mode: Append Only File</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#New data type: sorted sets">New data type: sorted sets</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Specialized integer objects encoding">Specialized integer objects encoding</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#MSET and MSETNX">MSET and MSETNX</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Better Performances">Better Performances</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Solaris Support">Solaris Support</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Support for the new generation protocol">Support for the new generation protocol</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#A few new commands about already supported data types">A few new commands about already supported data types</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Bug fixing">Bug fixing</a><br>&nbsp;&nbsp;<a href="#CHANGELOG for Redis 1.1.90">CHANGELOG for Redis 1.1.90</a>
                 </div>
                 
                 <h1 class="wikiname">Redis_1_2_0_Changelog</h1>
@@ -26,7 +26,11 @@
                 </div>
 
                 <div class="narrow">
-                    <h1><a name="CHANGELOG for Redis 1.1.90">CHANGELOG for Redis 1.1.90</a></h1><ul><li> 2009-09-10 in-memory specialized object encoding. (antirez)</li><li> 2009-09-17 maxmemory fixed in 64 systems for values &gt; 4GB. (antirez)</li><li> 2009-10-07 multi-bulk protocol implemented. (antriez)</li><li> 2009-10-16 MSET and MSETNX commands implemented (antirez)</li><li> 2009-10-21 SRANDMEMBER added (antirez)</li><li> 2009-10-23 Fixed compilation in mac os x snow leopard when compiling a 32 bit binary.  (antirez)</li><li> 2009-10-23 New data type: Sorted sets and Z-commands (antirez)</li><li> 2009-10-26 Solaris fixed (Alan Harder)</li><li> 2009-10-29 Fixed Issue a number of open issues (antirez)</li><li> 2009-10-30 New persistence mode: append only file (antirez)</li><li> 2009-11-01 SORT STORE option (antirez)</li><li> 2009-11-03 redis-cli now accepts a -r (repeat) switch. (antirez)</li><li> 2009-11-04 masterauth option merged (Anthony Lauzon)</li><li> 2009-11-04 redis-test is now a better Redis citizen, testing everything against DB 9 and 10 and only if this DBs are empty. (antirez)</li><li> 2009-11-10 Implemented a much better lazy expiring algorithm for EXPIRE (antirez)</li><li> 2009-11-11 RPUSHLPOP (antirez from an idea of @ezmobius)</li><li> 2009-11-12 Merge git://github.com/ianxm/redis (Can't remmber what this implements, sorry)</li><li> 2009-11-17 multi-bulk reply support for redis-bench, LRANGE speed tests (antirez)</li><li> 2009-11-17 support for writev implemented. (Stefano Barbato)</li><li> 2009-11-19 debug mode (-D) in redis-bench (antirez)</li><li> 2009-11-21 SORT GET # implemented (antirez)</li><li> 2009-11-23 ae.c made modular, with support for epoll. (antirez)</li><li> 2009-11-26 background append log rebuilding (antirez)</li><li> 2009-11-28 Added support for kqueue. (Harish Mallipeddi)</li><li> 2009-11-29 SORT support for sorted sets (antirez, thanks to @tobi for the idea)</li></ul>
+                    <h1><a name="What's new in Redis 1.2">What's new in Redis 1.2</a></h1><h2><a name="New persistence mode: Append Only File">New persistence mode: Append Only File</a></h2>The Append Only File is an alternative way to save your data in Redis that is fully durable! Unlike the snapshotting (default) persistence mode, where the database is saved asynchronously from time to time, the Append Only File saves every change ASAP in a text-only file that works like a journal. Redis will play back this file again at startup reloading the whole dataset back in memory. Redis Append Only File supports background Log compaction. For more info read the <a href="AppendOnlyFileHowto.html">Append Only File HOWTO</a>.<h2><a name="New data type: sorted sets">New data type: sorted sets</a></h2>Sorted sets are collections of elements (like Sets) with an associated score (in the form of a double precision floating point number). Elements in a sorted set are taken in order, so for instance to take the greatest element is an O(1) operation. Insertion and deletion is O(log(N)). Sorted sets are implemented using a dual ported data structure consisting of an hash table and a skip list. For more information please read the <a href="IntroductionToRedisDataTypes.html">Introduction To Redis Data Types</a>.<h2><a name="Specialized integer objects encoding">Specialized integer objects encoding</a></h2>Redis 1.2 will use less memory than Redis 1.0 for values in Strings, Lists or Sets elements that happen to be representable as 32 or 64 bit signed integers (it depends on your arch bits for the long C type). This is totally transparent form the point of view of the user, but will safe a lot of memory (30% less in datasets where there are many integers).<h2><a name="MSET and MSETNX">MSET and MSETNX</a></h2>That is, setting multiple keys in one command, atomically. For more information see the <a href="MsetCommand.html">MSET command</a> wiki page.<h2><a name="Better Performances">Better Performances</a></h2><ul><li> 100x times faster SAVE and BGSAVE! There was a problem in the LZF lib configuration that is now resolved. The effect is this impressive speedup. Also the saving child will no longer use 100% of CPU.</li><li> Glue output buffer and writev(). Many commands producing large outputs, like LRANGE, will now be even 10 times faster, thanks to the new output buffer gluing algorithm and the (optional) use of writev(2) syscall.</li><li> Support for epool and kqueue / kevent. 10,000 clients scalability.</li><li> Much better EXPIRE support, now it's possible to work with very large sets of keys expiring in very short time without to incur in memory problems (the new algorithm expires keys in an adaptive way, so will get more aggressive if there are a lot of expiring keys)</li></ul>
+<h2><a name="Solaris Support">Solaris Support</a></h2>Redis will now compile and work on Solaris without problems. Warning: the Solaris user base is very little, so Redis running on Solaris may not be as tested and stable as it is on  Linux and Mac OS X.<h2><a name="Support for the new generation protocol">Support for the new generation protocol</a></h2><ul><li> Redis is now able to accept commands in a new fully binary safe way: with the new protocol keys are binary safe, not only values, and there is no distinction between bulk commands and inline commands. This new protocol is currently used only for MSET and MSETNX but at some point it will hopefully replace the old one. See the Multi Bulk Commands section in the <a href="ProtocolSpecification.html">Redis Protocol Specification</a> for more information.</li></ul>
+<h2><a name="A few new commands about already supported data types">A few new commands about already supported data types</a></h2><ul><li> <a href="SrandmemberCommand.html">SRANDMEMBER</a></li><li> The <a href="SortCommand.html">SortCommand</a> is now supprots the <b>STORE</b> and <b>GET #</b> forms, the first can be used to save sorted lists, sets or sorted sets into keys for caching. Check the manual page for more information about the <b>GET #</b> form.</li><li> The new <a href="RpoplpushCommand.html">RPOPLPUSH command</a> can do many interesting magics, and a few of this are documented in the wiki page of the command.</li></ul>
+<h2><a name="Bug fixing">Bug fixing</a></h2>Of course, many bugs are now fixed, and I bet, a few others introduced: this is how software works after all, so make sure to report issues in the Redis mailing list or in the Google Code issues tracker.<br/><br/>Enjoy!
+antirez<h1><a name="CHANGELOG for Redis 1.1.90">CHANGELOG for Redis 1.1.90</a></h1><ul><li> 2009-09-10 in-memory specialized object encoding. (antirez)</li><li> 2009-09-17 maxmemory fixed in 64 systems for values &gt; 4GB. (antirez)</li><li> 2009-10-07 multi-bulk protocol implemented. (antriez)</li><li> 2009-10-16 MSET and MSETNX commands implemented (antirez)</li><li> 2009-10-21 SRANDMEMBER added (antirez)</li><li> 2009-10-23 Fixed compilation in mac os x snow leopard when compiling a 32 bit binary.  (antirez)</li><li> 2009-10-23 New data type: Sorted sets and Z-commands (antirez)</li><li> 2009-10-26 Solaris fixed (Alan Harder)</li><li> 2009-10-29 Fixed Issue a number of open issues (antirez)</li><li> 2009-10-30 New persistence mode: append only file (antirez)</li><li> 2009-11-01 SORT STORE option (antirez)</li><li> 2009-11-03 redis-cli now accepts a -r (repeat) switch. (antirez)</li><li> 2009-11-04 masterauth option merged (Anthony Lauzon)</li><li> 2009-11-04 redis-test is now a better Redis citizen, testing everything against DB 9 and 10 and only if this DBs are empty. (antirez)</li><li> 2009-11-10 Implemented a much better lazy expiring algorithm for EXPIRE (antirez)</li><li> 2009-11-11 RPUSHLPOP (antirez from an idea of @ezmobius)</li><li> 2009-11-12 Merge git://github.com/ianxm/redis (Can't remmber what this implements, sorry)</li><li> 2009-11-17 multi-bulk reply support for redis-bench, LRANGE speed tests (antirez)</li><li> 2009-11-17 support for writev implemented. (Stefano Barbato)</li><li> 2009-11-19 debug mode (-D) in redis-bench (antirez)</li><li> 2009-11-21 SORT GET # implemented (antirez)</li><li> 2009-11-23 ae.c made modular, with support for epoll. (antirez)</li><li> 2009-11-26 background append log rebuilding (antirez)</li><li> 2009-11-28 Added support for kqueue. (Harish Mallipeddi)</li><li> 2009-11-29 SORT support for sorted sets (antirez, thanks to @tobi for the idea)</li></ul>
                 </div>
         
             </div>

doc/ReplicationHowto.html

@@ -26,12 +26,14 @@
                 </div>
 
                 <div class="narrow">
-                    <h1><a name="Redis Replication Howto">Redis Replication Howto</a></h1><h2><a name="General Information">General Information</a></h2>Redis replication is a very simple to use and configure master-slave replication that allows slave Redis servers to be exact copies of master servers. The following are some very important facts about Redis replication:<br/><br/><ul><li> A master can have multiple slaves.</li><li> Slaves are able to accept other slaves connections, so instead to connect a number of slaves against the same master it is also possible to connect some of the slaves to other slaves in a graph-alike structure.</li><li> Redis replication is non-blocking on the master side, this means that the master will continue to serve queries while one or more slaves are performing the first synchronization. Instead replication is blocking on the slave side: while the slave is performing the first synchronization it can't reply to queries.</li><li> Replications can be used both for scalability, in order to have multiple slaves for read-only queries (for example heavy <a href="SortCommand.html">SORT</a> operations can be launched against slaves), or simply for data redundancy.</li><li> It is possible to use replication to avoid the saving process on the master side: just configure your master redis.conf in order to avoid saving at all (just comment al the &quot;save&quot; directives), then connect a slave configured to save from time to time.</li></ul>
-<h2><a name="How Redis replication works">How Redis replication works</a></h2>In order to start the replication, or after the connection closes in order resynchronize with the master, the client connects to the master and issues the SYNC command.<br/><br/>The master starts a background saving, and at the same time starts to collect all the new commands received that had the effect to modify the dataset. When the background saving completed the master starts the transfer of the database file to the slave, that saves it on disk, and then load it in memory. At this point the master starts to send all the accumulated commands, and all the new commands received from clients, that had the effect of a dataset modification.<br/><br/>You can try it yourself via telnet. Connect to the Redis port while the server is doing some work and issue the SYNC command. You'll see a bulk transfer and then every command received by the master will be re-issued in the telnet session.<br/><br/>Slaves are able to automatically reconnect when the master <code name="code" class="python">&lt;-&gt;</code> slave link goes down for some reason. If the master receives multiple concurrent slave synchronization requests it performs a single background saving in order to serve all them.<h2><a name="Configuration">Configuration</a></h2>To configure replication is trivial: just add the following line to the slave configuration file:
+                    &iuml;&raquo;&iquest;#sidebar <a href="RedisGuides.html">RedisGuides</a>
+<h1><a name="Redis Replication Howto">Redis Replication Howto</a></h1><h2><a name="General Information">General Information</a></h2>Redis replication is a very simple to use and configure master-slave replication that allows slave Redis servers to be exact copies of master servers. The following are some very important facts about Redis replication:<br/><br/><ul><li> A master can have multiple slaves.</li><li> Slaves are able to accept other slaves connections, so instead to connect a number of slaves against the same master it is also possible to connect some of the slaves to other slaves in a graph-alike structure.</li><li> Redis replication is non-blocking on the master side, this means that the master will continue to serve queries while one or more slaves are performing the first synchronization. Instead replication is blocking on the slave side: while the slave is performing the first synchronization it can't reply to queries.</li><li> Replications can be used both for scalability, in order to have multiple slaves for read-only queries (for example heavy <a href="SortCommand.html">SORT</a> operations can be launched against slaves), or simply for data redundancy.</li><li> It is possible to use replication to avoid the saving process on the master side: just configure your master redis.conf in order to avoid saving at all (just comment al the &quot;save&quot; directives), then connect a slave configured to save from time to time.</li></ul>
+<h2><a name="How Redis replication works">How Redis replication works</a></h2>In order to start the replication, or after the connection closes in order resynchronize with the master, the slave connects to the master and issues the SYNC command.<br/><br/>The master starts a background saving, and at the same time starts to collect all the new commands received that had the effect to modify the dataset. When the background saving completed the master starts the transfer of the database file to the slave, that saves it on disk, and then load it in memory. At this point the master starts to send all the accumulated commands, and all the new commands received from clients that had the effect of a dataset modification, to the slave, as a stream of commands, in the same format of the Redis protocol itself.<br/><br/>You can try it yourself via telnet. Connect to the Redis port while the server is doing some work and issue the SYNC command. You'll see a bulk transfer and then every command received by the master will be re-issued in the telnet session.<br/><br/>Slaves are able to automatically reconnect when the master <code name="code" class="python">&lt;-&gt;</code> slave link goes down for some reason. If the master receives multiple concurrent slave synchronization requests it performs a single background saving in order to serve all them.<h2><a name="Configuration">Configuration</a></h2>To configure replication is trivial: just add the following line to the slave configuration file:
 <pre class="codeblock python" name="code">
 slaveof 192.168.1.1 6379
 </pre>
 Of course you need to replace 192.168.1.1 6379 with your master ip address (or hostname) and port.
+
                 </div>
         
             </div>

doc/RpushCommand.html

@@ -28,8 +28,8 @@
                 <div class="narrow">
                     &iuml;&raquo;&iquest;#sidebar <a href="ListCommandsSidebar.html">ListCommandsSidebar</a><h3><a name="RPUSH _key_ _string_">RPUSH _key_ _string_</a></h3>
 <h3><a name="LPUSH _key_ _string_">LPUSH _key_ _string_</a></h3>
-<i>Time complexity: O(1)</i><blockquote>Add the <i>string</i> value to the head (RPUSH) or tail (LPUSH) of the liststored at <i>key</i>. If the key does not exist an empty list is created just beforethe append operation. If the key exists but is not a List an erroris returned.</blockquote>
-<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Status code reply</a>
+<i>Time complexity: O(1)</i><blockquote>Add the <i>string</i> value to the head (LPUSH) or tail (RPUSH) of the liststored at <i>key</i>. If the key does not exist an empty list is created just beforethe append operation. If the key exists but is not a List an erroris returned.</blockquote>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Integer reply</a>, specifically, the number of elements inside the list after the push operation.
 
                 </div>
         

doc/SaveCommand.html

@@ -27,9 +27,9 @@
 
                 <div class="narrow">
                     &iuml;&raquo;&iquest;#sidebar <a href="ControlCommandsSidebar.html">ControlCommandsSidebar</a><h3><a name="SAVE">SAVE</a></h3>
-<blockquote>Save the DB on disk. The server hangs while the saving is notcompleted, no connection is served in the meanwhile. An OK codeis returned when the DB was fully stored in disk.</blockquote>
+<blockquote>Save the whole dataset on disk (this means that all the databases are saved, as well as keys with an EXPIRE set (the expire is preserved). The server hangs while the saving is notcompleted, no connection is served in the meanwhile. An OK codeis returned when the DB was fully stored in disk.</blockquote>
+<blockquote>The background variant of this command is <a href="BgsaveCommand.html">BGSAVE</a> that is able to perform the saving in the background while the server continues serving other clients.</blockquote>
 <h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Status code reply</a>
-
                 </div>
         
             </div>

doc/SetexCommand.html

@@ -0,0 +1,42 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>SetexCommand: Contents</b><br>&nbsp;&nbsp;<a href="#SETEX _key_ _time_ _value_">SETEX _key_ _time_ _value_</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+                </div>
+                
+                <h1 class="wikiname">SetexCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;#sidebar <a href="StringCommandsSidebar.html">StringCommandsSidebar</a><h1><a name="SETEX _key_ _time_ _value_">SETEX _key_ _time_ _value_</a></h1>
+<i>Time complexity: O(1)</i><blockquote>The command is exactly equivalent to the following group of commands:</blockquote><pre class="codeblock python" name="code">
+SET _key_ _value_
+EXPIRE _key_ _time_
+</pre>
+<blockquote>The operation is atomic. An atomic <a href="SetCommand.html">SET</a>+<a href="ExpireCommand.html">EXPIRE</a> operation was already providedusing <a href="MultiExecCommand.html">MULTI/EXEC</a>, but SETEX is a faster alternative providedbecause this operation is very common when Redis is used as a Cache.</blockquote>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Status code reply</a>
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/SmembersCommand.html

@@ -27,9 +27,8 @@
 
                 <div class="narrow">
                     &iuml;&raquo;&iquest;#sidebar <a href="SetCommandsSidebar.html">SetCommandsSidebar</a><h1><a name="SMEMBERS _key_">SMEMBERS _key_</a></h1>
-<i>Time complexity O(N)</i><blockquote>Return all the members (elements) of the set value stored at <i>key</i>. Thisis just syntax glue for <a href="SintersectCommand.html">SINTERSECT</a>.</blockquote>
+<i>Time complexity O(N)</i><blockquote>Return all the members (elements) of the set value stored at <i>key</i>. Thisis just syntax glue for <a href="SintersectCommand.html">SINTER</a>.</blockquote>
 <h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Multi bulk reply</a>
-
                 </div>
         
             </div>

doc/SortCommand.html

@@ -16,7 +16,7 @@
             <div id="pagecontent">
                 <div class="index">
 <!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
-<b>SortCommand: Contents</b><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Sorting by external keys">Sorting by external keys</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Retrieving external keys">Retrieving external keys</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Storing the result of a SORT operation">Storing the result of a SORT operation</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+<b>SortCommand: Contents</b><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Sorting by external keys">Sorting by external keys</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Not Sorting at all">Not Sorting at all</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Retrieving external keys">Retrieving external keys</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Storing the result of a SORT operation">Storing the result of a SORT operation</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#SORT and Hashes: BY and GET by hash field">SORT and Hashes: BY and GET by hash field</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
                 </div>
                 
                 <h1 class="wikiname">SortCommand</h1>
@@ -46,19 +46,26 @@ SORT mylist LIMIT 0 10 ALPHA DESC
 SORT mylist BY weight_*
 </pre><blockquote>the <b>BY</b> option takes a pattern (<code name="code" class="python">weight_*</code> in our example) that is usedin order to generate the key names of the weights used for sorting.Weight key names are obtained substituting the first occurrence of <code name="code" class="python">*</code>with the actual value of the elements on the list (1,2,3,4 in our example).</blockquote>
 <blockquote>Our previous example will return just the sorted IDs. Often it isneeded to get the actual objects sorted (object_1, ..., object_4 in theexample). We can do it with the following command:</blockquote>
-<h2><a name="Retrieving external keys">Retrieving external keys</a></h2><pre class="codeblock python python python python python python" name="code">
+<h2><a name="Not Sorting at all">Not Sorting at all</a></h2><pre class="codeblock python python python python python python" name="code">
+SORT mylist BY nosort
+</pre><blockquote>also the <b>BY</b> option can take a &quot;nosort&quot; specifier. This is useful if you want to retrieve a external key (using GET, read below) but you don't want the sorting overhead.</blockquote>
+<h2><a name="Retrieving external keys">Retrieving external keys</a></h2><pre class="codeblock python python python python python python python" name="code">
 SORT mylist BY weight_* GET object_*
 </pre><blockquote>Note that <b>GET</b> can be used multiple times in order to get more keys forevery element of the original List, Set or Sorted Set sorted.</blockquote>
 <blockquote>Since Redis &gt;= 1.1 it's possible to also GET the list elements itselfusing the special # pattern:</blockquote>
-<pre class="codeblock python python python python python python python" name="code">
+<pre class="codeblock python python python python python python python python" name="code">
 SORT mylist BY weight_* GET object_* GET #
 </pre><h2><a name="Storing the result of a SORT operation">Storing the result of a SORT operation</a></h2><blockquote>By default SORT returns the sorted elements as its return value.Using the <b>STORE</b> option instead to return the elements SORT willstore this elements as a <a href="Lists.html">Redis List</a> in the specified key.An example:</blockquote>
-<pre class="codeblock python python python python python python python python" name="code">
+<pre class="codeblock python python python python python python python python python" name="code">
 SORT mylist BY weight_* STORE resultkey
 </pre><blockquote>An interesting pattern using SORT ... STORE consists in associatingan <a href="ExpireCommand.html">EXPIRE</a> timeout to the resulting key so that inapplications where the result of a sort operation can be cached forsome time other clients will use the cached list instead to call SORTfor every request. When the key will timeout an updated version ofthe cache can be created using SORT ... STORE again.</blockquote>
 <blockquote>Note that implementing this pattern it is important to avoid that multipleclients will try to rebuild the cached version of the cacheat the same time, so some form of locking should be implemented(for instance using <a href="SetnxCommand.html">SETNX</a>).</blockquote>
-<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Multi bulk reply</a>, specifically a list of sorted elements.
-
+<h2><a name="SORT and Hashes: BY and GET by hash field">SORT and Hashes: BY and GET by hash field</a></h2>
+<blockquote>It's possible to use BY and GET options against Hash fields using the following syntax:</blockquote><pre class="codeblock python python python python python python python python python python" name="code">
+SORT mylist BY weight_*-&gt;fieldname
+SORT mylist GET object_*-&gt;fieldname
+</pre>
+<blockquote>The two chars string -&gt; is used in order to signal the name of the Hash field. The key is substituted as documented above with sort BY and GET against normal keys, and the Hash stored at the resulting key is accessed in order to retrieve the specified field.</blockquote><h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Multi bulk reply</a>, specifically a list of sorted elements.
                 </div>
         
             </div>

doc/SortedSetCommandsSidebar.html

@@ -26,7 +26,7 @@
                 </div>
 
                 <div class="narrow">
-                    &iuml;&raquo;&iquest;== Sorted Set Commands ==<br/><br/><ul><li> <a href="ZaddCommand.html">ZADD</a></li><li> <a href="ZremCommand.html">ZREM</a></li><li> <a href="ZincrbyCommand.html">ZINCRBY</a></li><li> <a href="ZrangeCommand.html">ZRANGE</a></li><li> <a href="ZrangeCommand.html">ZREVRANGE</a></li><li> <a href="ZrangebyscoreCommand.html">ZRANGEBYSCORE</a></li><li> <a href="ZcardCommand.html">ZCARD</a></li><li> <a href="ZscoreCommand.html">ZSCORE</a></li><li> <a href="SortCommand.html">SORT</a></li></ul>
+                    &iuml;&raquo;&iquest;== Sorted Set Commands ==<br/><br/><ul><li> <a href="ZaddCommand.html">ZADD</a></li><li> <a href="ZremCommand.html">ZREM</a></li><li> <a href="ZincrbyCommand.html">ZINCRBY</a></li><li> <a href="ZrankCommand.html">ZRANK</a></li><li> <a href="ZrankCommand.html">ZREVRANK</a></li><li> <a href="ZrangeCommand.html">ZRANGE</a></li><li> <a href="ZrangeCommand.html">ZREVRANGE</a></li><li> <a href="ZrangebyscoreCommand.html">ZRANGEBYSCORE</a></li><li> <a href="ZrangebyscoreCommand.html">ZCOUNT</a></li><li> <a href="ZremrangebyrankCommand.html">ZREMRANGEBYRANK</a></li><li> <a href="ZremrangebyscoreCommand.html">ZREMRANGEBYSCORE</a> </li><li> <a href="ZcardCommand.html">ZCARD</a></li><li> <a href="ZscoreCommand.html">ZSCORE</a></li><li> <a href="ZunionstoreCommand.html">ZUNIONSTORE</a></li><li> <a href="ZunionstoreCommand.html">ZINTERSTORE</a></li><li> <a href="SortCommand.html">SORT</a></li></ul>
                 </div>
         
             </div>

doc/SortedSets.html

@@ -26,7 +26,7 @@
                 </div>
 
                 <div class="narrow">
-                    &iuml;&raquo;&iquest;#sidebar <a href="SortedSetCommandsSidebar.html">SortedSetCommandsSidebar</a><h1><a name="Redis Sorted Set Type">Redis Sorted Set Type</a></h1>Redis Sorted Sets are, similarly to <a href="Sets.html">Sets</a>, collections of <a href="Strings.html">Redis Strings</a>. The difference is that every member of a Sorted Set hash an <b>associated score</b> that is used in order to take this member in order.<br/><br/>The <a href="ZADD.html">ZaddCommand</a> command is used to add a new member to a Sorted Set, specifying the score of the element. Calling ZADD against a member already present in the sorted set but using a different score will update the score for the element, moving it to the right position in order to preserve ordering.<br/><br/>It's possible to get ranges of elements from Sorted Sets in a very similar way to what happens with <a href="Lists.html">Lists</a> and the <a href="LrangeCommnad.html">LRANGE</a> command  using the Sorted Sets <a href="ZrangeCommand.html">ZRANGE</a> command.<br/><br/>It's also possible to get or remove ranges of elements by score using the <a href="ZrangebyscoreCommand.html">ZRANGEBYSCORE</a> and <a href="ZremrangebyscoreCommand.html">ZREMRANGEBYSCORE</a> commands.<br/><br/>The max number of members in a sorted set is 232-1 (4294967295, more than 4 billion of members per set).<br/><br/>Note that while Sorted Sets are already ordered, it is still possible to use the <a href="SortCommand.html">SORT</a> command against sorted sets to get the elements in a different order.<h1><a name="Implementation details">Implementation details</a></h1>Redis Sets are implemented using a dual-ported data structure containing a skip list and an hash table. When an element is added a map between the element and the score is added to the hash table (so that given the element we get the score in O(1)), and a map between the score and the element is added in the skip list so that elements are taken in order.<br/><br/>Redis uses a special skip list implementation that is doubly linked so that it's possible to traverse the sorted set from tail to head if needed (Check the <a href="ZRevrangeCommand.html">ZREVRANGE</a> command).<br/><br/>When <a href="ZaddCommand.html">ZADD</a> is used in order to update the score of an element, Redis retrieve the score of the element using the hash table, so that it's fast to access the element inside the skip list (that's indexed by score) in order to update the position.<br/><br/>Like it happens for Sets the hash table resizing is a blocking operation performed synchronously so working with huge sorted sets (consisting of many millions of elements) care should be taken when mass-inserting a very big amount of elements in a Set while other clients are querying Redis at high speed.<br/><br/>It is possible that in the near future Redis will switch to skip lists even for the element =&gt; score map, so every Sorted Set will have two skip lists, one indexed by element and one indexed by score.
+                    &iuml;&raquo;&iquest;#sidebar <a href="SortedSetCommandsSidebar.html">SortedSetCommandsSidebar</a><h1><a name="Redis Sorted Set Type">Redis Sorted Set Type</a></h1>Redis Sorted Sets are, similarly to <a href="Sets.html">Sets</a>, collections of <a href="Strings.html">Redis Strings</a>. The difference is that every member of a Sorted Set hash an <b>associated score</b> that is used in order to take this member in order.<br/><br/>The <a href="ZaddCommand.html">ZADD</a> command is used to add a new member to a Sorted Set, specifying the score of the element. Calling ZADD against a member already present in the sorted set but using a different score will update the score for the element, moving it to the right position in order to preserve ordering.<br/><br/>It's possible to get ranges of elements from Sorted Sets in a very similar way to what happens with <a href="Lists.html">Lists</a> and the <a href="LrangeCommnad.html">LRANGE</a> command  using the Sorted Sets <a href="ZrangeCommand.html">ZRANGE</a> command.<br/><br/>It's also possible to get or remove ranges of elements by score using the <a href="ZrangebyscoreCommand.html">ZRANGEBYSCORE</a> and <a href="ZremrangebyscoreCommand.html">ZREMRANGEBYSCORE</a> commands.<br/><br/>The max number of members in a sorted set is 232-1 (4294967295, more than 4 billion of members per set).<br/><br/>Note that while Sorted Sets are already ordered, it is still possible to use the <a href="SortCommand.html">SORT</a> command against sorted sets to get the elements in a different order.<h1><a name="Implementation details">Implementation details</a></h1>Redis Sets are implemented using a dual-ported data structure containing a skip list and an hash table. When an element is added a map between the element and the score is added to the hash table (so that given the element we get the score in O(1)), and a map between the score and the element is added in the skip list so that elements are taken in order.<br/><br/>Redis uses a special skip list implementation that is doubly linked so that it's possible to traverse the sorted set from tail to head if needed (Check the <a href="ZRevrangeCommand.html">ZREVRANGE</a> command).<br/><br/>When <a href="ZaddCommand.html">ZADD</a> is used in order to update the score of an element, Redis retrieve the score of the element using the hash table, so that it's fast to access the element inside the skip list (that's indexed by score) in order to update the position.<br/><br/>Like it happens for Sets the hash table resizing is a blocking operation performed synchronously so working with huge sorted sets (consisting of many millions of elements) care should be taken when mass-inserting a very big amount of elements in a Set while other clients are querying Redis at high speed.<br/><br/>It is possible that in the near future Redis will switch to skip lists even for the element =&gt; score map, so every Sorted Set will have two skip lists, one indexed by element and one indexed by score.
                 </div>
         
             </div>

doc/SponsorshipHistory.html

@@ -26,7 +26,7 @@
                 </div>
 
                 <div class="narrow">
-                    <h1><a name="Redis Sponsorship History">Redis Sponsorship History</a></h1>This is a list of companies that sponsorship Redis developments, with details about the sponsored features. <b>Thanks for helping the project!</b>.<br/><br/>If your company is considering a sponsorship please read the <a href="SponsorshipHowto.html">How to Sponsor</a> page.<br/><br/><ul><li> <a href="http://citrusbyte.com" target="_blank"><img src="http://redis.googlecode.com/files/citrusbyte_logo.png" border="0"></img></a><br></br> 18 Dec 2009, part of Virtual Memory.</li><li> <a href="http://www.hitmeister.de/" target="_blank"><img src="http://redis.googlecode.com/files/logo_hitmeister_2.png" border="0"></img></a><br></br> 15 Dec 2009, part of Redis Cluster.</li><li> <a href="http://engineyard.com" target="_blank"><img src="http://redis.googlecode.com/files/engine_yard_logo.jpg" border="0"></img></a><br></br> 13 Dec 2009, for blocking POP (BLPOP) and part of the Virtual Memory implementation.</li></ul>
+                    <h1><a name="Redis Sponsorship History">Redis Sponsorship History</a></h1><b>Important notice: since 15 March 2010 I Joined VMware that is sponsoring all my work on Redis.</b> Thank you to all the companies and people donating in the past. No further donations are accepted.<br/><br/>This is a list of companies that sponsorship Redis developments, with details about the sponsored features. <b>Thanks for helping the project!</b>.<br/><br/><ul><li> <a href="http://www.linode.com/?r=5cf1759a154c981368394fca9918970f60b6a2b3" target="_blank"><img src="http://www.linode.com/images/linode_logo10.gif" border="0"></img></a><br></br> 15 January 2010, provided Virtual Machines for Redis testing in a virtualized environment.</li><li> <a href="https://manage.slicehost.com/customers/new?referrer=d6272cc9e5f38cd2513e760e4d22bd9d" target="_blank"><img src="http://wiki.slicehost.com/lib/exe/fetch.php?w=&h=&cache=cache&media=slicehost.gif" border="0"></img></a><br></br> 14 January 2010, provided Virtual Machines for Redis testing in a virtualized environment.</li><li> <a href="http://citrusbyte.com" target="_blank"><img src="http://redis.googlecode.com/files/citrusbyte_logo.png" border="0"></img></a><br></br> 18 Dec 2009, part of Virtual Memory.</li><li> <a href="http://www.hitmeister.de/" target="_blank"><img src="http://redis.googlecode.com/files/logo_hitmeister_2.png" border="0"></img></a><br></br> 15 Dec 2009, part of Redis Cluster.</li><li> <a href="http://engineyard.com" target="_blank"><img src="http://redis.googlecode.com/files/engine_yard_logo.jpg" border="0"></img></a><br></br> 13 Dec 2009, for blocking POP (BLPOP) and part of the Virtual Memory implementation.</li></ul>
 <b>Also thaks to the following people or organizations that donated to the Project:</b>
 <ul><li> Emil Vladev</li><li> <a href="http://bradjasper.com/" target="_blank">Brad Jasper</a></li><li> <a href="http://www.mrkris.com/" target="_blank">Mrkris</a></li></ul>
                 </div>

doc/SponsorshipHowto.html

@@ -1,37 +0,0 @@
-
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
-<html>
-    <head>
-        <link type="text/css" rel="stylesheet" href="style.css" />
-    </head>
-    <body>
-        <div id="page">
-        
-            <div id='header'>
-            <a href="index.html">
-            <img style="border:none" alt="Redis Documentation" src="redis.png">
-            </a>
-            </div>
-        
-            <div id="pagecontent">
-                <div class="index">
-<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
-<b>SponsorshipHowto: Contents</b><br>&nbsp;&nbsp;<a href="#Other donations">Other donations</a>
-                </div>
-                
-                <h1 class="wikiname">SponsorshipHowto</h1>
-
-                <div class="summary">
-                    
-                </div>
-
-                <div class="narrow">
-                    &iuml;&raquo;&iquest;= How to sponsor my work on Redis =<br/><br/>I'm accepting sponsorships for Redis development, the idea is that a company using Redis and willing to donate some money can receive something back: visibility in the Redis site, and prioritization of features planned in the TODO list that are somewhat more important for this company compared to other features.<br/><br/>In the last year I spent 50% of my time working on Redis. At the same time Redis is released under the very liberal BSD license, this is very important for users as it prevents <a href="http://monty-says.blogspot.com/2009/10/importance-of-license-model-of-mysql-or.html" target="_blank">that in the future the project will be killed</a>, but at the same time it's not possible to build a business model selling licenses like it happens for MySQL. The alternative is to run a consultancy company, but this means to use more time to work with customers than working to the Redis code base itself, or sponsorship, that I think is the best option currently to ensure fast development of the project.<br/><br/>So, if you are considering a donation, thank you! This is a set of simple rules I developed in order to make sure I'm fair with everybody willing to help the project:<br/><br/><ul><li> 1. Every company can donate any amount of money, even 10$, in order to support Redis development.</li><li> 2. Every company donating an amount equal or greater than 1000$ will be featured in the home page for at least 6 months, and anyway for all the time the sponsored feature takes to reach a <b>stable release</b> of Redis.</li><li> 3. Every company donating at least 100$ will anyway be featured in the &quot;Sponsors&quot; page forever, this page is linked near to the logos of the current sponsors in the front page (the logos about point 2 of this list).</li><li> 4. A sponsoring company can donate for sponsorship of a feature already in the TODO list. If a feature not planned is needed we should first get in touch, discuss if this is a good idea, put it in the TODO list, and then the sponsorship can start, but I've to be genuinely convinced this feature will be good and of general interest ;)</li><li> 5. Not really a sponsorship/donation, but in rare case of a vertical, self-contained feature, I could develop it as a patch for the current stable Redis distribution for a &quot;donation&quot; proportional to the work needed to develop the feature, but in order to have the patch for the next release of Redis there will be to donate again for the porting work and so forth.</li><li> 6. Features for which I receive a good sponsorship (proportionally to the work required to implement the sponsored feature) are prioritized and will get developed faster than other features, possibly changing the development roadmap.</li><li> 7. To sponsor a specific feature is not a must, a company can just donate to the project as a whole.</li></ul>
-If you want to get in touch with me about this issues please drop me an email to my gmail account (username is antirez) or direct-message me @antirez on Twitter. Thanks in advance for the help!<h1><a name="Other donations">Other donations</a></h1>If you just feel like donating a small amount to Redis the simplest way is to use paypal, my paypal address is <b>antirez@invece.org</b>. Please specify in the donation if you don't like to have your name / company name published in the donations history (the amount will not be published anyway).
-                </div>
-        
-            </div>
-        </div>
-    </body>
-</html>
-

doc/StringCommandsSidebar.html

@@ -26,7 +26,7 @@
                 </div>
 
                 <div class="narrow">
-                    &iuml;&raquo;&iquest;== String Commands ==<br/><br/><ul><li> <a href="SetCommand.html">SET</a></li><li> <a href="GetCommand.html">GET</a></li><li> <a href="GetsetCommand.html">GETSET</a></li><li> <a href="MgetCommand.html">MGET</a></li><li> <a href="SetnxCommand.html">SETNX</a></li><li> <a href="MsetCommand.html">MSET</a></li><li> <a href="MsetCommand.html">MSETNX</a></li><li> <a href="IncrCommand.html">INCR</a></li><li> <a href="IncrCommand.html">INCRBY</a></li><li> <a href="IncrCommand.html">DECR</a></li><li> <a href="IncrCommand.html">DECRBY</a></li></ul>
+                    &iuml;&raquo;&iquest;== String Commands ==<br/><br/><ul><li> <a href="SetCommand.html">SET</a></li><li> <a href="GetCommand.html">GET</a></li><li> <a href="GetsetCommand.html">GETSET</a></li><li> <a href="MgetCommand.html">MGET</a></li><li> <a href="SetnxCommand.html">SETNX</a></li><li> <a href="SetexCommand.html">SETEX</a></li><li> <a href="MsetCommand.html">MSET</a></li><li> <a href="MsetCommand.html">MSETNX</a></li><li> <a href="IncrCommand.html">INCR</a></li><li> <a href="IncrCommand.html">INCRBY</a></li><li> <a href="IncrCommand.html">DECR</a></li><li> <a href="IncrCommand.html">DECRBY</a></li><li> <a href="AppendCommand.html">APPEND</a></li><li> <a href="SubstrCommand.html">SUBSTR</a></li></ul>
                 </div>
         
             </div>

doc/SubstrCommand.html

@@ -0,0 +1,51 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>SubstrCommand: Contents</b><br>&nbsp;&nbsp;<a href="#SUBSTR _key_ _start_ _end_">SUBSTR _key_ _start_ _end_</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Examples">Examples</a>
+                </div>
+                
+                <h1 class="wikiname">SubstrCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;#sidebar <a href="StringCommandsSidebar.html">StringCommandsSidebar</a><h1><a name="SUBSTR _key_ _start_ _end_">SUBSTR _key_ _start_ _end_</a></h1>
+<i>Time complexity: O(start+n) (with start being the start index and n the total length of the requested range). Note that the lookup part of this command is O(1) so for small strings this is actually an O(1) command.</i><blockquote>Return a subset of the string from offset <i>start</i> to offset <i>end</i>(both offsets are inclusive).Negative offsets can be used in order to provide an offset starting fromthe end of the string. So -1 means the last char, -2 the penultimate andso forth.</blockquote>
+<blockquote>The function handles out of range requests without raising an error, butjust limiting the resulting range to the actual length of the string.</blockquote>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Bulk reply</a><h2><a name="Examples">Examples</a></h2><pre class="codeblock python" name="code">
+redis&gt; set s &quot;This is a string&quot;
+OK
+redis&gt; substr s 0 3
+&quot;This&quot;
+redis&gt; substr s -3 -1
+&quot;ing&quot;
+redis&gt; substr s 0 -1
+&quot;This is a string&quot;
+redis&gt; substr s 9 100000
+&quot; string&quot;
+</pre>
+
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/TypeCommand.html

@@ -33,6 +33,8 @@
 "string" if the key contains a String value
 "list" if the key contains a List value
 "set" if the key contains a Set value
+"zset" if the key contains a Sorted Set value
+"hash" if the key contains a Hash value
 </pre><h2><a name="See also">See also</a></h2>
 <ul><li> <a href="DataTypes.html">Redis Data Types</a></li></ul>
                 </div>

doc/VirtualMemorySpecification.html

@@ -0,0 +1,156 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>VirtualMemorySpecification: Contents</b><br>&nbsp;&nbsp;<a href="#Virtual Memory technical specification">Virtual Memory technical specification</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Keys vs Values: what is swapped out?">Keys vs Values: what is swapped out?</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#How does a swapped value looks like internally">How does a swapped value looks like internally</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#The Swap File">The Swap File</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Transfering objects from memory to swap">Transfering objects from memory to swap</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Loading objects back in memory">Loading objects back in memory</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#How blocking VM works">How blocking VM works</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Blocking VM swapping">Blocking VM swapping</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#What values to swap when we are out of memory?">What values to swap when we are out of memory?</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Blocking VM loading">Blocking VM loading</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Background saving when VM is active">Background saving when VM is active</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#The problem with the blocking VM">The problem with the blocking VM</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Threaded VM">Threaded VM</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#I/O Threads">I/O Threads</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Non blocking VM as probabilistic enhancement of blocking VM">Non blocking VM as probabilistic enhancement of blocking VM</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Blocking clients on swapped keys">Blocking clients on swapped keys</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Aborting I/O jobs">Aborting I/O jobs</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Questions?">Questions?</a>
+                </div>
+                
+                <h1 class="wikiname">VirtualMemorySpecification</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;#sidebar <a href="RedisInternals.html">RedisInternals</a><h1><a name="Virtual Memory technical specification">Virtual Memory technical specification</a></h1>This document details the internals of the Redis Virtual Memory subsystem. The intended audience is not the final user but programmers willing to understand or modify the Virtual Memory implementation.<h2><a name="Keys vs Values: what is swapped out?">Keys vs Values: what is swapped out?</a></h2>
+The goal of the VM subsystem is to free memory transferring Redis Objects from memory to disk. This is a very generic command, but specifically, Redis transfers only objects associated with <i>values</i>. In order to understand better this concept we'll show, using the DEBUG command, how a key holding a value looks from the point of view of the Redis internals:
+<pre class="codeblock python" name="code">
+redis&gt; set foo bar
+OK
+redis&gt; debug object foo
+Key at:0x100101d00 refcount:1, value at:0x100101ce0 refcount:1 encoding:raw serializedlength:4
+</pre>
+As you can see from the above output, the Redis top level hash table maps Redis Objects (keys) to other Redis Objects (values). The Virtual Memory is only able to swap <i>values</i> on disk, the objects associated to <i>keys</i> are always taken in memory: this trade off guarantees very good lookup performances, as one of the main design goals of the Redis VM is to have performances similar to Redis with VM disabled when the part of the dataset frequently used fits in RAM.<h2><a name="How does a swapped value looks like internally">How does a swapped value looks like internally</a></h2>
+When an object is swapped out, this is what happens in the hash table entry:
+<ul><li> The key continues to hold a Redis Object representing the key.</li><li> The value is set to NULL</li></ul>So you may wonder where we store the information that a given value (associated to a given key) was swapped out. Just in the key object!<br/><br/>This is how the Redis Object structure <i>robj</i> looks like:
+<pre class="codeblock python python" name="code">
+/* The actual Redis Object */
+typedef struct redisObject {
+    void *ptr;
+    unsigned char type;
+    unsigned char encoding;
+    unsigned char storage;  /* If this object is a key, where is the value?
+                             * REDIS_VM_MEMORY, REDIS_VM_SWAPPED, ... */
+    unsigned char vtype; /* If this object is a key, and value is swapped out,
+                          * this is the type of the swapped out object. */
+    int refcount;
+    /* VM fields, this are only allocated if VM is active, otherwise the
+     * object allocation function will just allocate
+     * sizeof(redisObjct) minus sizeof(redisObjectVM), so using
+     * Redis without VM active will not have any overhead. */
+    struct redisObjectVM vm;
+} robj;
+</pre>
+As you can see there are a few fields about VM. The most important one is <i>storage</i>, that can be one of this values:
+<ul><li> REDIS_VM_MEMORY: the associated value is in memory.</li><li> REDIS_VM_SWAPPED: the associated values is swapped, and the value entry of the hash table is just set to NULL.</li><li> REDIS_VM_LOADING: the value is swapped on disk, the entry is NULL, but there is a job to load the object from the swap to the memory (this field is only used when threaded VM is active).</li><li> REDIS_VM_SWAPPING: the value is in memory, the entry is a pointer to the actual Redis Object, but there is an I/O job in order to transfer this value to the swap file.</li></ul>If an object is swapped on disk (REDIS_VM_SWAPPED or REDIS_VM_LOADING), how do we know where it is stored, what type it is, and so forth? That's simple: the <i>vtype</i> field is set to the original type of the Redis object swapped, while the <i>vm</i> field (that is a <i>redisObjectVM</i> structure) holds information about the location of the object. This is the definition of this additional structure:
+<pre class="codeblock python python python" name="code">
+/* The VM object structure */
+struct redisObjectVM {
+    off_t page;         /* the page at which the object is stored on disk */
+    off_t usedpages;    /* number of pages used on disk */
+    time_t atime;       /* Last access time */
+} vm;
+</pre>
+As you can see the structure contains the page at which the object is located in the swap file, the number of pages used, and the last access time of the object (this is very useful for the algorithm that select what object is a good candidate for swapping, as we want to transfer on disk objects that are rarely accessed).<br/><br/>As you can see, while all the other fields are using unused bytes in the old Redis Object structure (we had some free bit due to natural memory alignment concerns), the <i>vm</i> field is new, and indeed uses additional memory. Should we pay such a memory cost even when VM is disabled? No! This is the code to create a new Redis Object:
+<pre class="codeblock python python python python" name="code">
+... some code ...
+        if (server.vm_enabled) {
+            pthread_mutex_unlock(&amp;server.obj_freelist_mutex);
+            o = zmalloc(sizeof(*o));
+        } else {
+            o = zmalloc(sizeof(*o)-sizeof(struct redisObjectVM));
+        }
+... some code ...
+</pre>
+As you can see if the VM system is not enabled we allocate just <code name="code" class="python">sizeof(*o)-sizeof(struct redisObjectVM)</code> of memory. Given that the <i>vm</i> field is the last in the object structure, and that this fields are never accessed if VM is disabled, we are safe and Redis without VM does not pay the memory overhead.<h2><a name="The Swap File">The Swap File</a></h2>
+The next step in order to understand how the VM subsystem works is understanding how objects are stored inside the swap file. The good news is that's not some kind of special format, we just use the same format used to store the objects in .rdb files, that are the usual dump files produced by Redis using the SAVE command.<br/><br/>The swap file is composed of a given number of pages, where every page size is a given number of bytes. This parameters can be changed in redis.conf, since different Redis instances may work better with different values: it depends on the actual data you store inside it. The following are the default values:
+<pre class="codeblock python python python python python" name="code">
+vm-page-size 32
+vm-pages 134217728
+</pre>
+Redis takes a &quot;bitmap&quot; (an contiguous array of bits set to zero or one) in memory, every bit represent a page of the swap file on disk: if a given bit is set to 1, it represents a page that is already used (there is some Redis Object stored there), while if the corresponding bit is zero, the page is free.<br/><br/>Taking this bitmap (that will call the page table) in memory is a huge win in terms of performances, and the memory used is small: we just need 1 bit for every page on disk. For instance in the example below 134217728 pages of 32 bytes each (4GB swap file) is using just 16 MB of RAM for the page table.<h2><a name="Transfering objects from memory to swap">Transfering objects from memory to swap</a></h2>
+In order to transfer an object from memory to disk we need to perform the following steps (assuming non threaded VM, just a simple blocking approach):<br/><br/><ul><li> Find how many pages are needed in order to store this object on the swap file. This is trivially accomplished just calling the function <code name="code" class="python">rdbSavedObjectPages</code> that returns the number of pages used by an object on disk. Note that this function does not duplicate the .rdb saving code just to understand what will be the length <b>after</b> an object will be saved on disk, we use the trick of opening /dev/null and writing the object there, finally calling <code name="code" class="python">ftello</code> in order check the amount of bytes required. What we do basically is to save the object on a virtual very fast file, that is, /dev/null.</li><li> Now that we know how many pages are required in the swap file, we need to find this number of contiguous free pages inside the swap file. This task is accomplished by the <code name="code" class="python">vmFindContiguousPages</code> function. As you can guess this function may fail if the swap is full, or so fragmented that we can't easily find the required number of contiguous free pages. When this happens we just abort the swapping of the object, that will continue to live in memory.</li><li> Finally we can write the object on disk, at the specified position, just calling the function <code name="code" class="python">vmWriteObjectOnSwap</code>.</li></ul>As you can guess once the object was correctly written in the swap file, it is freed from memory, the storage field in the associated key is set to REDIS_VM_SWAPPED, and the used pages are marked as used in the page table.
+<h2><a name="Loading objects back in memory">Loading objects back in memory</a></h2>
+Loading an object from swap to memory is simpler, as we already know where the object is located and how many pages it is using. We also know the type of the object (the loading functions are required to know this information, as there is no header or any other information about the object type on disk), but this is stored in the <i>vtype</i> field of the associated key as already seen above.<br/><br/>Calling the function <code name="code" class="python">vmLoadObject</code> passing the key object associated to the value object we want to load back is enough. The function will also take care of fixing the storage type of the key (that will be REDIS_VM_MEMORY), marking the pages as freed in the page table, and so forth.<br/><br/>The return value of the function is the loaded Redis Object itself, that we'll have to set again as value in the main hash table (instead of the NULL value we put in place of the object pointer when the value was originally swapped out).
+<h2><a name="How blocking VM works">How blocking VM works</a></h2>
+Now we have all the building blocks in order to describe how the blocking VM works. First of all, an important detail about configuration. In order to enable blocking VM in Redis <code name="code" class="python">server.vm_max_threads</code> must be set to zero.
+We'll see later how this max number of threads info is used in the threaded VM, for now all it's needed to now is that Redis reverts to fully blocking VM when this is set to zero.<br/><br/>We also need to introduce another important VM parameter, that is, <code name="code" class="python">server.vm_max_memory</code>. This parameter is very important as it is used in order to trigger swapping: Redis will try to swap objects only if it is using more memory than the max memory setting, otherwise there is no need to swap as we are matching the user requested memory usage.
+<h2><a name="Blocking VM swapping">Blocking VM swapping</a></h2>
+Swapping of object from memory to disk happens in the cron function. This function used to be called every second, while in the recent Redis versions on git it is called every 100 milliseconds (that is, 10 times per second).
+If this function detects we are out of memory, that is, the memory used is greater than the vm-max-memory setting, it starts transferring objects from memory to disk in a loop calling the function <code name="code" class="python">vmSwapOneObect</code>. This function takes just one argument, if 0 it will swap objects in a blocking way, otherwise if it is 1, I/O threads are used. In the blocking scenario we just call it with zero as argument.<br/><br/>vmSwapOneObject acts performing the following steps:
+<ul><li> The key space in inspected in order to find a good candidate for swapping (we'll see later what a good candidate for swapping is).</li><li> The associated value is transfered to disk, in a blocking way.</li><li> The key storage field is set to REDIS_VM_SWAPPED, while the <i>vm</i> fields of the object are set to the right values (the page index where the object was swapped, and the number of pages used to swap it).</li><li> Finally the value object is freed and the value entry of the hash table is set to NULL.</li></ul>The function is called again and again until one of the following happens: there is no way to swap more objects because either the swap file is full or nearly all the objects are already transfered on disk, or simply the memory usage is already under the vm-max-memory parameter.
+<h2><a name="What values to swap when we are out of memory?">What values to swap when we are out of memory?</a></h2>
+Understanding what's a good candidate for swapping is not too hard. A few objects at random are sampled, and for each their <i>swappability</i> is commuted as:
+<pre class="codeblock python python python python python python" name="code">
+swappability = age*log(size_in_memory)
+</pre>
+The age is the number of seconds the key was not requested, while size_in_memory is a fast estimation of the amount of memory (in bytes) used by the object in memory. So we try to swap out objects that are rarely accessed, and we try to swap bigger objects over smaller one, but the latter is a less important factor (because of the logarithmic function used). This is because we don't want bigger objects to be swapped out and in too often as the bigger the object the more I/O and CPU is required in order to transfer it.
+<h2><a name="Blocking VM loading">Blocking VM loading</a></h2>
+What happens if an operation against a key associated with a swapped out object is requested? For instance Redis may just happen to process the following command:
+<pre class="codeblock python python python python python python python" name="code">
+GET foo
+</pre>
+If the value object of the <code name="code" class="python">foo</code> key is swapped we need to load it back in memory before processing the operation. In Redis the key lookup process is centralized in the <code name="code" class="python">lookupKeyRead</code> and <code name="code" class="python">lookupKeyWrite</code> functions, this two functions are used in the implementation of all the Redis commands accessing the keyspace, so we have a single point in the code where to handle the loading of the key from the swap file to memory.<br/><br/>So this is what happens:
+<ul><li> The user calls some command having as argumenet a swapped key</li><li> The command implementation calls the lookup function</li><li> The lookup function search for the key in the top level hash table. If the value associated with the requested key is swapped (we can see that checking the <i>storage</i> field of the key object), we load it back in memory in a blocking way before to return to the user.</li></ul>
+This is pretty straightforward, but things will get more <i>interesting</i> with the threads. From the point of view of the blocking VM the only real problem is the saving of the dataset using another process, that is, handling BGSAVE and BGREWRITEAOF commands.
+<h2><a name="Background saving when VM is active">Background saving when VM is active</a></h2>
+The default Redis way to persist on disk is to create .rdb files using a child process. Redis calls the fork() system call in order to create a child, that has the exact copy of the in memory dataset, since fork duplicates the whole program memory space (actually thanks to a technique called Copy on Write memory pages are shared between the parent and child process, so the fork() call will not require too much memory).<br/><br/>In the child process we have a copy of the dataset in a given point in the time. Other commands issued by clients will just be served by the parent process and will not modify the child data.<br/><br/>The child process will just store the whole dataset into the dump.rdb file and finally will exit. But what happens when the VM is active? Values can be swapped out so we don't have all the data in memory, and we need to access the swap file in order to retrieve the swapped values. While child process is saving the swap file is shared between the parent and child process, since:
+<ul><li> The parent process needs to access the swap file in order to load values back into memory if an operation against swapped out values are performed.</li><li> The child process needs to access the swap file in order to retrieve the full dataset while saving the data set on disk.</li></ul>In order to avoid problems while both the processes are accessing the same swap file we do a simple thing, that is, not allowing values to be swapped out in the parent process while a background saving is in progress. This way both the processes will access the swap file in read only. This approach has the problem that while the child process is saving no new values can be transfered on the swap file even if Redis is using more memory than the max memory parameters dictates. This is usually not a problem as the background saving will terminate in a short amount of time and if still needed a percentage of values will be swapped on disk ASAP.<br/><br/>An alternative to this scenario is to enable the Append Only File that will have this problem only when a log rewrite is performed using the BGREWRITEAOF command.
+<h2><a name="The problem with the blocking VM">The problem with the blocking VM</a></h2>
+The problem of blocking VM is that... it's blocking :)
+This is not a problem when Redis is used in batch processing activities, but for real-time usage one of the good points of Redis is the low latency. The blocking VM will have bad latency behaviors as when a client is accessing a swapped out value, or when Redis needs to swap out values, no other clients will be served in the meantime.<br/><br/>Swapping out keys should happen in background. Similarly when a client is accessing a swapped out value other clients accessing in memory values should be served mostly as fast as when VM is disabled. Only the clients dealing with swapped out keys should be delayed.<br/><br/>All this limitations called for a non-blocking VM implementation.
+<h2><a name="Threaded VM">Threaded VM</a></h2>
+There are basically three main ways to turn the blocking VM into a non blocking one.
+<ul><li> 1: One way is obvious, and in my opionion, not a good idea at all, that is, turning Redis itself into a theaded server: if every request is served by a different thread automatically other clients don't need to wait for blocked ones. Redis is fast, exports atomic operations, has no locks, and is just 10k lines of code, <b>because</b> it is single threaded, so this was not an option for me.</li><li> 2: Using non-blocking I/O against the swap file. After all you can think Redis already event-loop based, why don't just handle disk I/O in a non-blocking fashion? I also discarded this possiblity because of two main reasons. One is that non blocking file operations, unlike sockets, are an incompatibility nightmare. It's not just like calling select, you need to use OS-specific things. The other problem is that the I/O is just one part of the time consumed to handle VM, another big part is the CPU used in order to encode/decode data to/from the swap file. This is I picked option three, that is...</li><li> 3: Using I/O threads, that is, a pool of threads handling the swap I/O operations. This is what the Redis VM is using, so let's detail how this works.</li></ul><h2><a name="I/O Threads">I/O Threads</a></h2>
+The threaded VM design goals where the following, in order of importance:
+<ul><li> Simple implementation, little room for race condtions, simple locking, VM system more or less completeley decoupled from the rest of Redis code.</li><li> Good performances, no locks for clients accessing values in memory.</li><li> Ability to decode/encode objects in the I/O threads.</li></ul>The above goals resulted in an implementation where the Redis main thread (the one serving actual clients) and the I/O threads communicate using a queue of jobs, with a single mutex.
+Basically when main thread requires some work done in the background by some I/O thread, it pushes an I/O job structure in the <code name="code" class="python">server.io_newjobs</code> queue (that is, just a linked list). If there are no active I/O threads, one is started. At this point some I/O thread will process the I/O job, and the result of the processing is pushed in the <code name="code" class="python">server.io_processed</code> queue. The I/O thread will send a byte using an UNIX pipe to the main thread in order to signal that a new job was processed and the result is ready to be processed.<br/><br/>This is how the <code name="code" class="python">iojob</code> structure looks like:
+<pre class="codeblock python python python python python python python python" name="code">
+typedef struct iojob {
+    int type;   /* Request type, REDIS_IOJOB_* */
+    redisDb *db;/* Redis database */
+    robj *key;  /* This I/O request is about swapping this key */
+    robj *val;  /* the value to swap for REDIS_IOREQ_*_SWAP, otherwise this
+                 * field is populated by the I/O thread for REDIS_IOREQ_LOAD. */
+    off_t page; /* Swap page where to read/write the object */
+    off_t pages; /* Swap pages needed to save object. PREPARE_SWAP return val */
+    int canceled; /* True if this command was canceled by blocking side of VM */
+    pthread_t thread; /* ID of the thread processing this entry */
+} iojob;
+</pre>
+There are just three type of jobs that an I/O thread can perform (the type is specified by the <code name="code" class="python">type</code> field of the structure):
+<ul><li> REDIS_IOJOB_LOAD: load the value associated to a given key from swap to memory. The object offset inside the swap file is <code name="code" class="python">page</code>, the object type is <code name="code" class="python">key-&gt;vtype</code>. The result of this operation will populate the <code name="code" class="python">val</code> field of the structure.</li><li> REDIS_IOJOB_PREPARE_SWAP: compute the number of pages needed in order to save the object pointed by <code name="code" class="python">val</code> into the swap. The result of this operation will populate the <code name="code" class="python">pages</code> field.</li><li> REDIS_IOJOB_DO_SWAP: Transfer the object pointed by <code name="code" class="python">val</code> to the swap file, at page offset <code name="code" class="python">page</code>.</li></ul>The main thread delegates just the above three tasks. All the rest is handled by the main thread itself, for instance finding a suitable range of free pages in the swap file page table (that is a fast operation), deciding what object to swap, altering the storage field of a Redis object to reflect the current state of a value.
+<h2><a name="Non blocking VM as probabilistic enhancement of blocking VM">Non blocking VM as probabilistic enhancement of blocking VM</a></h2>
+So now we have a way to request background jobs dealing with slow VM operations. How to add this to the mix of the rest of the work done by the main thread? While blocking VM was aware that an object was swapped out just when the object was looked up, this is too late for us: in C it is not trivial to start a background job in the middle of the command, leave the function, and re-enter in the same point the computation when the I/O thread finished what we requested (that is, no co-routines or continuations or alike).<br/><br/>Fortunately there was a much, much simpler way to do this. And we love simple things: basically consider the VM implementation a blocking one, but add an optimization (using non the no blocking VM operations we are able to perform) to make the blocking <b>very</b> unlikely.<br/><br/>This is what we do:
+<ul><li> Every time a client sends us a command, <b>before</b> the command is executed, we examine the argument vector of the command in search for swapped keys. After all we know for every command what arguments are keys, as the Redis command format is pretty simple.</li><li> If we detect that at least a key in the requested command is swapped on disk, we block the client instead of really issuing the command. For every swapped value associated to a requested key, an I/O job is created, in order to bring the values back in memory. The main thread continues the execution of the event loop, without caring about the blocked client.</li><li> In the meanwhile, I/O threads are loading values in memory. Every time an I/O thread finished loading a value, it sends a byte to the main thread using an UNIX pipe. The pipe file descriptor has a readable event associated in the main thread event loop, that is the function <code name="code" class="python">vmThreadedIOCompletedJob</code>. If this function detects that all the values needed for a blocked client were loaded, the client is restarted and the original command called.</li></ul>
+So you can think at this as a blocked VM that almost always happen to have the right keys in memory, since we pause clients that are going to issue commands about swapped out values until this values are loaded.<br/><br/>If the function checking what argument is a key fails in some way, there is no problem: the lookup function will see that a given key is associated to a swapped out value and will block loading it. So our non blocking VM reverts to a blocking one when it is not possible to anticipate what keys are touched.<br/><br/>For instance in the case of the SORT command used together with the GET or BY options, it is not trivial to know beforehand what keys will be requested, so at least in the first implementation, SORT BY/GET resorts to the blocking VM implementation.
+<h2><a name="Blocking clients on swapped keys">Blocking clients on swapped keys</a></h2>
+How to block clients? To suspend a client in an event-loop based server is pretty trivial. All we do is cancelling its read handler. Sometimes we do something different (for instance for BLPOP) that is just marking the client as blocked, but not processing new data (just accumulating the new data into input buffers).
+<h2><a name="Aborting I/O jobs">Aborting I/O jobs</a></h2>
+There is something hard to solve about the interactions between our blocking and non blocking VM, that is, what happens if a blocking operation starts about a key that is also &quot;interested&quot; by a non blocking operation at the same time?<br/><br/>For instance while SORT BY is executed, a few keys are being loaded in a blocking manner by the sort command. At the same time, another client may request the same keys with a simple <i>GET key</i> command, that will trigger the creation of an I/O job to load the key in background.<br/><br/>The only simple way to deal with this problem is to be able to kill I/O jobs in the main thread, so that if a key that we want to load or swap in a blocking way is in the REDIS_VM_LOADING or REDIS_VM_SWAPPING state (that is, there is an I/O job about this key), we can just kill the I/O job about this key, and go ahead with the blocking operation we want to perform.<br/><br/>This is not as trivial as it is. In a given moment an I/O job can be in one of the following three queues:
+<ul><li> server.io_newjobs: the job was already queued but no thread is handling it.</li><li> server.io_processing: the job is being processed by an I/O thread.</li><li> server.io_processed: the job was already processed.</li></ul>The function able to kill an I/O job is <code name="code" class="python">vmCancelThreadedIOJob</code>, and this is what it does:
+<ul><li> If the job is in the newjobs queue, that's simple, removing the iojob structure from the queue is enough as no thread is still executing any operation.</li><li> If the job is in the processing queue, a thread is messing with our job (and possibly with the associated object!). The only thing we can do is waiting for the item to move to the next queue in a <b>blocking way</b>. Fortunately this condition happens very rarely so it's not a performance problem.</li><li> If the job is in the processed queue, we just mark it as <i>canceled</i> marking setting the <code name="code" class="python">canceled</code> field to 1 in the iojob structure. The function processing completed jobs will just ignored and free the job instead of really processing it.</li></ul>
+<h2><a name="Questions?">Questions?</a></h2>
+This document is in no way complete, the only way to get the whole picture is reading the source code, but it should be a good introduction in order to make the code review / understanding a lot simpler.<br/><br/>Something is not clear about this page? Please leave a comment and I'll try to address the issue possibly integrating the answer in this document.
+
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/VirtualMemoryUserGuide.html

@@ -0,0 +1,67 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>VirtualMemoryUserGuide: Contents</b><br>&nbsp;&nbsp;<a href="#Virtual Memory User Guide">Virtual Memory User Guide</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Virtual Memory explained in simple words">Virtual Memory explained in simple words</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#When using Virtual Memory is a good idea">When using Virtual Memory is a good idea</a><br>&nbsp;&nbsp;<a href="#VM Configuration">VM Configuration</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#The vm-max-memory setting">The vm-max-memory setting</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Configuring the swap file">Configuring the swap file</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Threaded VM vs Blocking VM">Threaded VM vs Blocking VM</a><br>&nbsp;&nbsp;<a href="#Random things to know">Random things to know</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#A good place for the swap file">A good place for the swap file</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Monitoring the VM">Monitoring the VM</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Redis with VM enabled: better .rdb files or Append Only File?">Redis with VM enabled: better .rdb files or Append Only File?</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Using as little memory as possible">Using as little memory as possible</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#VM Stability">VM Stability</a>
+                </div>
+                
+                <h1 class="wikiname">VirtualMemoryUserGuide</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    &iuml;&raquo;&iquest;#sidebar <a href="RedisGuides.html">RedisGuides</a><h1><a name="Virtual Memory User Guide">Virtual Memory User Guide</a></h1>Redis Virtual Memory is a feature that will appear for the first time in a stable Redis distribution in Redis 2.0. However Virtual Memory (called VM starting from now) is already available and stable enough to be tests in the unstable branch of Redis available <a href="http://github.com/antirez/redis" target="_blank">on Git</a>.<h2><a name="Virtual Memory explained in simple words">Virtual Memory explained in simple words</a></h2>Redis follows a Key-Value model. You have keys associated with some values.
+Usually Redis takes both Keys and associated Values in memory. Sometimes this is not the best option, and while Keys <b>must</b> be taken in memory for the way Redis is designed (and in order to ensure fast lookups), Values can be swapped out to disk when they are rarely used.<br/><br/>In practical terms this means that if you have a dataset of 100,000 keys in memory, but only 10% of this keys are often used, Redis with Virtual Memory enabled will try to transfer the values associated to the rarely used keys on disk.<br/><br/>When this values are requested, as a result of a command issued by a client, the values are loaded back from the swap file to the main memory.<h2><a name="When using Virtual Memory is a good idea">When using Virtual Memory is a good idea</a></h2>Before using VM you should ask yourself if you really need it. Redis is a disk backed, in memory database. The right way to use Redis is almost always to have enough RAM to fit all the data in memory. Still there are a scenarios where to hold all the data in memory is not possible:
+<ul><li> Data access is very biased. Only a small percentage of keys (for instance related to active users in your web site) gets the vast majority of accesses. At the same time there is too much data per key to take everything in memory.</li><li> There is simply not enough memory available to hold all the data in memory, regardless of the data access pattern, and values are large. In this configuration Redis can be used as an on-disk DB where keys are in memory, so the key lookup is fast, but the access to the actual values require accessing the (slower) disk.</li></ul>
+An important concept to take in mind is that Redis <b>is not able to swap the keys</b>, so if your memory problems are related to the fact you have too much keys with very small values, VM is not the solution.<br/><br/>Instead if a good amount of memory is used because values are pretty large (for example large strings, lists, sets or hashes with many elements), then VM can be a good idea.<br/><br/>Sometimes you can turn your &quot;many keys with small values&quot; problem into a &quot;less keys but with very large values&quot; one just using Hashes in order to group related data into fields of a single key. For instance instead of having a key for every attribute of your object you have a single key per object where Hash fields represent the different attributes.<h1><a name="VM Configuration">VM Configuration</a></h1>Configuring the VM is not hard but requires some care to set the best parameters accordingly to the requirements.<br/><br/>The VM is enabled and configured editing redis.conf, the first step is switching it on with:<br/><br/><pre class="codeblock python" name="code">
+vm-enabled yes
+</pre>Many other configuration options are able to change the behavior of VM. The rule is that you don't want to run with the default configuration, as every problem and dataset requires some tuning in order to get the maximum advantages.<h2><a name="The vm-max-memory setting">The vm-max-memory setting</a></h2>The <b>vm-max-memory</b> setting specifies how much memory Redis is free to use before starting swapping values on disk.<br/><br/>Basically if this memory limit is still not reached, no object will be swapped, Redis will work all in memory as usually. Once this limit is hit, enough objects are swapped out in order to return just under the limit.<br/><br/>The swapped objects are the one with the highest &quot;age&quot; (that is, the number of seconds since they are not used in any way) mainly, but the &quot;swappability&quot; of an object is also proportional to the logarithm of it's size in memory. So basically older objects are preferred, but when they are about the same size, bigger objects are preferred.<br/><br/><b>WARNING:</b> Because keys can't be swapped out, Redis will not be able to honour the <b>vm-max-memory</b> setting if the keys alone are using more space than the limit.<br/><br/>The best value for this setting is enough RAM in order to hold the &quot;working set&quot; of data. In practical terms, just give Redis as much memory as you can, and swapping will work better.<h2><a name="Configuring the swap file">Configuring the swap file</a></h2>In order to transfer data from memory to disk, Redis uses a swap file. The swap file has nothing to do with durability of data, and can be removed when a Redis instance is terminated. Still the swap file should not be moved, deleted, or altered in any other way while Redis is running.<br/><br/>Because the Redis swap file is used mostly in a random access fashion, to put the swap file into a Solid State Disk will lead to better performances.<br/><br/>The swap file is divided into &quot;pages&quot;. A value can be swapped into one or multiple pages, but a single page can't hold more than a value.<br/><br/>There is no direct way to tell Redis how much bytes of swap file it should be using. Instead two different values are configured, that multiplied together will produce the total number of bytes used. This two values are the number of pages inside the swap file, and the page size. It is possible to configure this two parameters in redis.conf.<br/><br/><ul><li> The <b>vm-pages</b> configuration directive is used to set the total number of pages in the swap file.</li><li> the <b>vm-page-size</b> configuration directive is used in order to set the page size in bytes.</li></ul>
+So for instance if the page size is set to the value of 32 bytes, and the total number of pages is set to 10000000 (10 millions), the swap file can hold a total of 320 MB of data.<br/><br/>Because a single page can't be used to hold more than a value (but a value can be stored into multiple pages), care must be taken in setting this parameters.
+Usually the best idea is setting the page size so that the majority of the values can be swapped using a few pages.<h2><a name="Threaded VM vs Blocking VM">Threaded VM vs Blocking VM</a></h2>Another very important configuration parameter is <b>vm-max-threads</b>:<br/><br/><pre class="codeblock python python" name="code">
+# The default vm-max-threads configuration
+vm-max-threads 4
+</pre>This is the maximum number of threads used in order to perform I/O from/to the swap file. A good value is just to match the number of cores in your system.<br/><br/>However the special value of &quot;0&quot; will enable blocking VM. When VM is configured to be blocking it performs the I/O in a synchronous blocking way. This is what you can expect from blocking VM:
+<ul><li> Clients accessing swapped out keys will block other clients while reading from disk, so the latency experimented by clients can be larger, especially if the disk is slow or busy and/or if there are big values swapped on disk.</li><li> The blocking VM performances are <b>overall</b> better, as there is no time lost in synchronization, spawning of threads, resuming blocked clients waiting for values.</li></ul>So if you are willing to accept an higher latency from time to time, blocking VM can be a good pick, especially if swapping happens rarely as most of your often accessed data happens to fit in your memory.<br/><br/>If instead you have a lot of swap in and swap out operations and you have many cores that you want to exploit, and in general when you don't want that clients dealing with swapped values will block other clients for a few milliseconds (or more if the swapped value is very big), then it's better to use threaded VM.<br/><br/>To experiment with your dataset and different configurations is warmly encouraged...<h1><a name="Random things to know">Random things to know</a></h1>
+<h2><a name="A good place for the swap file">A good place for the swap file</a></h2>In many configurations the swap file can be fairly large, even 40GB or more.
+Not all the kind of file systems are able to deal with large files in a good way, especially Mac OS X file system tends to be really lame about it.<br/><br/>The suggestion is to use Linux ext3 file system, or any other file system with good support for <b>sparse files</b>. What are sparse files?<br/><br/>Sparse files are files where a lot of the content happen to be empty. Advanced file systems like ext2, ext3, ext4, RaiserFS, Raiser4, and many others, are able to encode this files in a more efficient way and will allocate more space for the file when needed, that is, when more actual blocks of the file will be used.<br/><br/>The swap file is obviously pretty sparse, especially if the server is running since little time or it is much bigger compared to the amount of data swapped out. A file system not supporting sparse files can at some point block the Redis process while creating a very big file at once.<br/><br/>For a list of file systems supporting spare files <a href="http://en.wikipedia.org/wiki/Comparison_of_file_systems" target="_blank">check this Wikipedia page comparing different files systems</a>.<h2><a name="Monitoring the VM">Monitoring the VM</a></h2>Once you have a Redis system with VM enabled up and running, you may be very interested in knowing how it's working: how many objects are swapped in total, the number of objects swapped and loaded every second, and so forth.<br/><br/>There is an utility that is very handy in checking how the VM is working, that is part of <a href="http://code.google.com/p/redis-tools" target="_blank">Redis Tools</a>. This tool is called redis-stat, and using it is pretty straightforward:<br/><br/><pre class="codeblock python python python" name="code">
+$ ./redis-stat vmstat
+ --------------- objects --------------- ------ pages ------ ----- memory -----
+ load-in  swap-out  swapped   delta      used     delta      used     delta    
+ 138837   1078936   800402    +800402    807620   +807620    209.50M  +209.50M  
+ 4277     38011     829802    +29400     837441   +29821     206.47M  -3.03M   
+ 3347     39508     862619    +32817     870340   +32899     202.96M  -3.51M   
+ 4445     36943     890646    +28027     897925   +27585     199.92M  -3.04M   
+ 10391    16902     886783    -3863     894104   -3821     200.22M  +309.56K  
+ 8888     19507     888371    +1588      895678   +1574      200.05M  -171.81K 
+ 8377     20082     891664    +3293      899850   +4172      200.10M  +53.55K   
+ 9671     20210     892586    +922       899917   +67        199.82M  -285.30K 
+ 10861    16723     887638    -4948     895003   -4914     200.13M  +312.35K  
+ 9541     21945     890618    +2980      898004   +3001      199.94M  -197.11K 
+ 9689     17257     888345    -2273     896405   -1599     200.27M  +337.77K  
+ 10087    18784     886771    -1574     894577   -1828     200.36M  +91.60K   
+ 9330     19350     887411    +640       894817   +240       200.17M  -189.72K 
+</pre>The above output is about a redis-server with VM enable, around 1 million of keys inside, and a lot of simulated load using the redis-load utility.<br/><br/>As you can see from the output a number of load-in and swap-out operations are happening every second. Note that the first line reports the actual values since the server was started, while the next lines are differences compared to the previous reading.<br/><br/>If you assigned enough memory to hold your working set of data, probably you should see a lot less dramatic swapping happening, so redis-stat can be a really valuable tool in order to understand if you need to shop for RAM ;)<h2><a name="Redis with VM enabled: better .rdb files or Append Only File?">Redis with VM enabled: better .rdb files or Append Only File?</a></h2>When VM is enabled, saving and loading the database are <b>much slower</b> operations. A DB that usually loads in 2 seconds takes 13 seconds with VM enabled if the server is configured to use the smallest memory possible (that is, vm-max-memory set to 0).<br/><br/>So you probably want to switch to a configuration using the Append Only File for persistence, so that you can perform the BGREWRITEAOF from time to time.<br/><br/>It is important to note that while a BGSAVE or BGREWRITEAOF is in progress Redis does <b>not</b> swap new values on disk. The VM will be read-only while there is another child accessing it. So if you have a lot of writes while there is a child working, the memory usage may grow.<h2><a name="Using as little memory as possible">Using as little memory as possible</a></h2>An interesting setup to turn Redis into an on-disk DB with just keys in memory is setting vm-max-memory to 0. If you don't mind some latency more and poorer performances but want to use very little memory for very big values, this is a good setup.<br/><br/>In this setup you should first try setting the VM as blocking (vm-max-threads 0) as with this configuration and high traffic the number of swap in and swap out operations will be huge, and threading will consume a lot of resources compared to a simple blocking implementation.<h2><a name="VM Stability">VM Stability</a></h2>VM is still experimental code, but in the latest weeks it was tested in many ways in development environments, and even in some production environment. No bugs were noticed during this testing period. Still the more obscure bugs may happen in non controlled environments where there are setups that we are not able to reproduce for some reason.<br/><br/>In this stage you are encouraged to try VM in your development environment, and even in production if your DB is not mission critical, but for instance just a big persistent cache of data that may go away without too much problems.<br/><br/>Please report any problem you will notice to the Redis Google Group or by IRC joining the #redis IRC channel on freenode.
+
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/ZincrbyCommand.html

@@ -30,8 +30,8 @@
 <i>Time complexity O(log(N)) with N being the number of elements in the sorted set</i><blockquote>If <i>member</i> already exists in the sorted set adds the <i>increment</i> to its scoreand updates the position of the element in the sorted set accordingly.If <i>member</i> does not already exist in the sorted set it is added with_increment_ as score (that is, like if the previous score was virtually zero).If <i>key</i> does not exist a new sorted set with the specified_member_ as sole member is crated. If the key exists but does not hold asorted set value an error is returned.</blockquote>
 <blockquote>The score value can be the string representation of a double precision floatingpoint number. It's possible to provide a negative value to perform a decrement.</blockquote>
 <blockquote>For an introduction to sorted sets check the <a href="IntroductionToRedisDataTypes.html">Introduction to Redis data types</a> page.</blockquote>
-<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Integer reply</a>, specifically:<br/><br/><pre class="codeblock python" name="code">
-The score of the member after the increment is performed.
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Bulk reply</a><pre class="codeblock python" name="code">
+The new score (a double precision floating point number) represented as string.
 </pre>
                 </div>
         

doc/ZrangebyscoreCommand.html

@@ -16,7 +16,7 @@
             <div id="pagecontent">
                 <div class="index">
 <!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
-<b>ZrangebyscoreCommand: Contents</b><br>&nbsp;&nbsp;<a href="#ZRANGEBYSCORE _key_ _min_ _max_ `[`LIMIT _offset_ _count_`]` (Redis &gt;">ZRANGEBYSCORE _key_ _min_ _max_ `[`LIMIT _offset_ _count_`]` (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+<b>ZrangebyscoreCommand: Contents</b><br>&nbsp;&nbsp;<a href="#ZRANGEBYSCORE _key_ _min_ _max_ `[`LIMIT _offset_ _count_`]` (Redis &gt;">ZRANGEBYSCORE _key_ _min_ _max_ `[`LIMIT _offset_ _count_`]` (Redis &gt;</a><br>&nbsp;&nbsp;<a href="#ZRANGEBYSCORE _key_ _min_ _max_ `[`LIMIT _offset_ _count_`]` `[`WITHSCORES`]` (Redis &gt;">ZRANGEBYSCORE _key_ _min_ _max_ `[`LIMIT _offset_ _count_`]` `[`WITHSCORES`]` (Redis &gt;</a><br>&nbsp;&nbsp;<a href="#ZCOUNT _key_ _min_ _max_">ZCOUNT _key_ _min_ _max_</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Exclusive intervals and infinity">Exclusive intervals and infinity</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Examples">Examples</a>
                 </div>
                 
                 <h1 class="wikiname">ZrangebyscoreCommand</h1>
@@ -27,11 +27,47 @@
 
                 <div class="narrow">
                     &iuml;&raquo;&iquest;#sidebar <a href="SortedSetCommandsSidebar.html">SortedSetCommandsSidebar</a><h1><a name="ZRANGEBYSCORE _key_ _min_ _max_ `[`LIMIT _offset_ _count_`]` (Redis &gt;">ZRANGEBYSCORE _key_ _min_ _max_ `[`LIMIT _offset_ _count_`]` (Redis &gt;</a></h1> 1.1) =
+<h1><a name="ZRANGEBYSCORE _key_ _min_ _max_ `[`LIMIT _offset_ _count_`]` `[`WITHSCORES`]` (Redis &gt;">ZRANGEBYSCORE _key_ _min_ _max_ `[`LIMIT _offset_ _count_`]` `[`WITHSCORES`]` (Redis &gt;</a></h1> 1.3.4) =
+<h1><a name="ZCOUNT _key_ _min_ _max_">ZCOUNT _key_ _min_ _max_</a></h1>
 <i>Time complexity: O(log(N))+O(M) with N being the number of elements in the sorted set and M the number of elements returned by the command, so if M is constant (for instance you always ask for the first ten elements with LIMIT) you can consider it O(log(N))</i><blockquote>Return the all the elements in the sorted set at key with a score between_min_ and <i>max</i> (including elements with score equal to min or max).</blockquote>
 <blockquote>The elements having the same score are returned sorted lexicographically asASCII strings (this follows from a property of Redis sorted sets and does notinvolve further computation).</blockquote>
 <blockquote>Using the optional LIMIT it's possible to get only a range of the matchingelements in an SQL-alike way. Note that if <i>offset</i> is large the commandsneeds to traverse the list for <i>offset</i> elements and this adds up to theO(M) figure.</blockquote>
-<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Multi bulk reply</a>, specifically a list of elements in the specified score range.
-
+<blockquote>The <b>ZCOUNT</b> command is similar to <b>ZRANGEBYSCORE</b> but instead of returningthe actual elements in the specified interval, it just returns the numberof matching elements.</blockquote><h2><a name="Exclusive intervals and infinity">Exclusive intervals and infinity</a></h2>
+<i>min</i> and <i>max</i> can be -inf and +inf, so that you are not required to know what's the greatest or smallest element in order to take, for instance, elements &quot;up to a given value&quot;.<br/><br/>Also while the interval is for default closed (inclusive) it's possible to specify open intervals prefixing the score with a &quot;(&quot; character, so for instance:
+<pre class="codeblock python" name="code">
+ZRANGEBYSCORE zset (1.3 5
+</pre>
+Will return all the values with score <b>&gt; 1.3 and &lt;= 5</b>, while for instance:
+<pre class="codeblock python python" name="code">
+ZRANGEBYSCORE zset (5 (10
+</pre>
+Will return all the values with score <b>&gt; 5 and &lt; 10</b> (5 and 10 excluded).
+<h2><a name="Return value">Return value</a></h2>ZRANGEBYSCORE returns a <a href="ReplyTypes.html">Multi bulk reply</a> specifically a list of elements in the specified score range.<br/><br/>ZCOUNT returns a <a href="ReplyTypes.html">Integer reply</a> specifically the number of elements matching the specified score range.
+<h2><a name="Examples">Examples</a></h2>
+<pre class="codeblock python python python" name="code">
+redis&gt; zadd zset 1 foo
+(integer) 1
+redis&gt; zadd zset 2 bar
+(integer) 1
+redis&gt; zadd zset 3 biz
+(integer) 1
+redis&gt; zadd zset 4 foz
+(integer) 1
+redis&gt; zrangebyscore zset -inf +inf
+1. &quot;foo&quot;
+2. &quot;bar&quot;
+3. &quot;biz&quot;
+4. &quot;foz&quot;
+redis&gt; zcount zset 1 2
+(integer) 2
+redis&gt; zrangebyscore zset 1 2
+1. &quot;foo&quot;
+2. &quot;bar&quot;
+redis&gt; zrangebyscore zset (1 2
+1. &quot;bar&quot;
+redis&gt; zrangebyscore zset (1 (2
+(empty list or set)
+</pre>
                 </div>
         
             </div>

doc/ZrankCommand.html

@@ -0,0 +1,42 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>ZrankCommand: Contents</b><br>&nbsp;&nbsp;<a href="#ZRANK _key_ _member_ (Redis &gt;">ZRANK _key_ _member_ (Redis &gt;</a><br>&nbsp;&nbsp;<a href="#ZREVRANK _key_ _member_ (Redis &gt;">ZREVRANK _key_ _member_ (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+                </div>
+                
+                <h1 class="wikiname">ZrankCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    
+<h1><a name="ZRANK _key_ _member_ (Redis &gt;">ZRANK _key_ _member_ (Redis &gt;</a></h1> 1.3.4) =
+<h1><a name="ZREVRANK _key_ _member_ (Redis &gt;">ZREVRANK _key_ _member_ (Redis &gt;</a></h1> 1.3.4) =
+<i>Time complexity: O(log(N))</i><blockquote>ZRANK returns the rank of the member in the sorted set, with scores ordered from low to high. ZREVRANK returns the rank with scores ordered from high to low. When the given member does not exist in the sorted set, the special value 'nil' is returned. The returned rank (or index) of the member is 0-based for both commands.</blockquote>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Bulk reply</a>, specifically:<br/><br/><pre class="codeblock python" name="code">
+the rank (an integer number) represented as an string.
+</pre>
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/ZremrangebyrankCommand.html

@@ -0,0 +1,39 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>ZremrangebyrankCommand: Contents</b><br>&nbsp;&nbsp;<a href="#ZREMRANGEBYRANK _key_ _start_ _end_ (Redis &gt;">ZREMRANGEBYRANK _key_ _start_ _end_ (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+                </div>
+                
+                <h1 class="wikiname">ZremrangebyrankCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    
+<h1><a name="ZREMRANGEBYRANK _key_ _start_ _end_ (Redis &gt;">ZREMRANGEBYRANK _key_ _start_ _end_ (Redis &gt;</a></h1> 1.3.4) =
+<i>Time complexity: O(log(N))+O(M) with N being the number of elements in the sorted set and M the number of elements removed by the operation</i><blockquote>Remove all elements in the sorted set at <i>key</i> with rank between <i>start</i> and <i>end</i>. Start and end are 0-based with rank 0 being the element with the lowest score. Both start and end can be negative numbers, where they indicate offsets starting at the element with the highest rank. For example: -1 is the element with the highest score, -2 the element with the second highest score and so forth.</blockquote>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Integer reply</a>, specifically the number of elements removed.
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/ZunionCommand.html

@@ -0,0 +1,42 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>ZunionCommand: Contents</b><br>&nbsp;&nbsp;<a href="#ZUNIONSTORE _dstkey_ _N_ _k1_ ... _kN_ `[`WEIGHTS _w1_ ... _wN_`]` `[`AGGREGATE SUM|MIN|MAX`]`  (Redis &gt;">ZUNIONSTORE _dstkey_ _N_ _k1_ ... _kN_ `[`WEIGHTS _w1_ ... _wN_`]` `[`AGGREGATE SUM|MIN|MAX`]`  (Redis &gt;</a><br>&nbsp;&nbsp;<a href="#ZINTERSTORE _dstkey_ _N_ _k1_ ... _kN_ `[`WEIGHTS _w1_ ... _wN_`]` `[`AGGREGATE SUM|MIN|MAX`]`  (Redis &gt;">ZINTERSTORE _dstkey_ _N_ _k1_ ... _kN_ `[`WEIGHTS _w1_ ... _wN_`]` `[`AGGREGATE SUM|MIN|MAX`]`  (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+                </div>
+                
+                <h1 class="wikiname">ZunionCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    
+<h1><a name="ZUNIONSTORE _dstkey_ _N_ _k1_ ... _kN_ `[`WEIGHTS _w1_ ... _wN_`]` `[`AGGREGATE SUM|MIN|MAX`]`  (Redis &gt;">ZUNIONSTORE _dstkey_ _N_ _k1_ ... _kN_ `[`WEIGHTS _w1_ ... _wN_`]` `[`AGGREGATE SUM|MIN|MAX`]`  (Redis &gt;</a></h1> 1.3.12) =
+<h1><a name="ZINTERSTORE _dstkey_ _N_ _k1_ ... _kN_ `[`WEIGHTS _w1_ ... _wN_`]` `[`AGGREGATE SUM|MIN|MAX`]`  (Redis &gt;">ZINTERSTORE _dstkey_ _N_ _k1_ ... _kN_ `[`WEIGHTS _w1_ ... _wN_`]` `[`AGGREGATE SUM|MIN|MAX`]`  (Redis &gt;</a></h1> 1.3.12) =<br/><br/><i>Time complexity: O(N) + O(M log(M)) with N being the sum of the sizes of the input sorted sets, and M being the number of elements in the resulting sorted set</i><blockquote>Creates a union or intersection of <i>N</i> sorted sets given by keys <i>k1</i> through <i>kN</i>, and stores it at <i>dstkey</i>. It is mandatory to provide the number of input keys <i>N</i>, before passing the input keys and the other (optional) arguments.</blockquote>
+<blockquote>As the terms imply, the ZINTERSTORE command requires an element to be present in each of the given inputs to be inserted in the result. The ZUNIONSTORE command inserts all elements across all inputs.</blockquote>
+<blockquote>Using the WEIGHTS option, it is possible to add weight to each input sorted set. This means that the score of each element in the sorted set is first multiplied by this weight before being passed to the aggregation. When this option is not given, all weights default to 1.</blockquote>
+<blockquote>With the AGGREGATE option, it's possible to specify how the results of the union or intersection are aggregated. This option defaults to SUM, where the score of an element is summed across the inputs where it exists. When this option is set to be either MIN or MAX, the resulting set will contain the minimum or maximum score of an element across the inputs where it exists.</blockquote>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Integer reply</a>, specifically the number of elements in the sorted set at <i>dstkey</i>.
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/ZunionstoreCommand.html

@@ -0,0 +1,43 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+    <head>
+        <link type="text/css" rel="stylesheet" href="style.css" />
+    </head>
+    <body>
+        <div id="page">
+        
+            <div id='header'>
+            <a href="index.html">
+            <img style="border:none" alt="Redis Documentation" src="redis.png">
+            </a>
+            </div>
+        
+            <div id="pagecontent">
+                <div class="index">
+<!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
+<b>ZunionstoreCommand: Contents</b><br>&nbsp;&nbsp;<a href="#ZUNIONSTORE _dstkey_ _N_ _k1_ ... _kN_ `[`WEIGHTS _w1_ ... _wN_`]` `[`AGGREGATE SUM|MIN|MAX`]`  (Redis &gt;">ZUNIONSTORE _dstkey_ _N_ _k1_ ... _kN_ `[`WEIGHTS _w1_ ... _wN_`]` `[`AGGREGATE SUM|MIN|MAX`]`  (Redis &gt;</a><br>&nbsp;&nbsp;<a href="#ZINTERSTORE _dstkey_ _N_ _k1_ ... _kN_ `[`WEIGHTS _w1_ ... _wN_`]` `[`AGGREGATE SUM|MIN|MAX`]`  (Redis &gt;">ZINTERSTORE _dstkey_ _N_ _k1_ ... _kN_ `[`WEIGHTS _w1_ ... _wN_`]` `[`AGGREGATE SUM|MIN|MAX`]`  (Redis &gt;</a><br>&nbsp;&nbsp;&nbsp;&nbsp;<a href="#Return value">Return value</a>
+                </div>
+                
+                <h1 class="wikiname">ZunionstoreCommand</h1>
+
+                <div class="summary">
+                    
+                </div>
+
+                <div class="narrow">
+                    
+<h1><a name="ZUNIONSTORE _dstkey_ _N_ _k1_ ... _kN_ `[`WEIGHTS _w1_ ... _wN_`]` `[`AGGREGATE SUM|MIN|MAX`]`  (Redis &gt;">ZUNIONSTORE _dstkey_ _N_ _k1_ ... _kN_ `[`WEIGHTS _w1_ ... _wN_`]` `[`AGGREGATE SUM|MIN|MAX`]`  (Redis &gt;</a></h1> 1.3.12) =
+<h1><a name="ZINTERSTORE _dstkey_ _N_ _k1_ ... _kN_ `[`WEIGHTS _w1_ ... _wN_`]` `[`AGGREGATE SUM|MIN|MAX`]`  (Redis &gt;">ZINTERSTORE _dstkey_ _N_ _k1_ ... _kN_ `[`WEIGHTS _w1_ ... _wN_`]` `[`AGGREGATE SUM|MIN|MAX`]`  (Redis &gt;</a></h1> 1.3.12) =
+<i>Time complexity: O(N) + O(M log(M)) with N being the sum of the sizes of the input sorted sets, and M being the number of elements in the resulting sorted set</i><blockquote>Creates a union or intersection of <i>N</i> sorted sets given by keys <i>k1</i> through <i>kN</i>, and stores it at <i>dstkey</i>. It is mandatory to provide the number of input keys <i>N</i>, before passing the input keys and the other (optional) arguments.</blockquote>
+<blockquote>As the terms imply, the ZINTERSTORE command requires an element to be present in each of the given inputs to be inserted in the result. The ZUNIONSTORE command inserts all elements across all inputs.</blockquote>
+<blockquote>Using the WEIGHTS option, it is possible to add weight to each input sorted set. This means that the score of each element in the sorted set is first multiplied by this weight before being passed to the aggregation. When this option is not given, all weights default to 1.</blockquote>
+<blockquote>With the AGGREGATE option, it's possible to specify how the results of the union or intersection are aggregated. This option defaults to SUM, where the score of an element is summed across the inputs where it exists. When this option is set to be either MIN or MAX, the resulting set will contain the minimum or maximum score of an element across the inputs where it exists.</blockquote>
+<h2><a name="Return value">Return value</a></h2><a href="ReplyTypes.html">Integer reply</a>, specifically the number of elements in the sorted set at <i>dstkey</i>.
+                </div>
+        
+            </div>
+        </div>
+    </body>
+</html>
+

doc/index.html

@@ -16,7 +16,7 @@
             <div id="pagecontent">
                 <div class="index">
 <!-- This is a (PRE) block.  Make sure it's left aligned or your toc title will be off. -->
-<b>index: Contents</b><br>&nbsp;&nbsp;<a href="#Redis Documentation">Redis Documentation</a><br>&nbsp;&nbsp;<a href="#HOWTOs about selected features">HOWTOs about selected features</a><br>&nbsp;&nbsp;<a href="#Hacking">Hacking</a><br>&nbsp;&nbsp;<a href="#Videos">Videos</a>
+<b>index: Contents</b><br>&nbsp;&nbsp;<a href="#HOWTOs about selected features">HOWTOs about selected features</a><br>&nbsp;&nbsp;<a href="#Hacking">Hacking</a><br>&nbsp;&nbsp;<a href="#Videos">Videos</a>
                 </div>
                 
                 <h1 class="wikiname">index</h1>
@@ -26,11 +26,12 @@
                 </div>
 
                 <div class="narrow">
-                    <h1><a name="Redis Documentation">Redis Documentation</a></h1>Hello! The followings are pointers to different parts of the Redis Documentation.<br/><br/><ul><li> <a href="README.html">The README</a> is the best starting point to know more about the project.</li><li> <a href="QuickStart.html">This short Quick Start</a> provides a five minutes step-by-step istructions on how to download, compile, run and test the basic workings of a Redis server.</li><li> <a href="CommandReference.html">The command reference</a> is a description of all the Redis commands with links to command specific pages.</li><li> <a href="TwitterAlikeExample.html">This is a tuturial about creating a Twitter clone using *only* Redis as database, no relational DB at all is used</a>, it is a good start to understand the key-value database paradigm.</li><li> <a href="IntroductionToRedisDataTypes.html">A Fifteen Minutes Introduction to the Redis Data Types</a> explains how Redis data types work and the basic patterns of working with Redis.</li><li> <a href="Features.html">The features page</a> (currently in draft) is a good start to understand the strength and limitations of Redis.</li><li> <a href="Benchmarks.html">The benchmark page</a> is about the speed performances of Redis.</li><li> <a href="FAQ.html">Our FAQ</a> contains of course some answers to common questions about Redis.</li><li> <b><a href="SponsorshipHowto.html">How to donate</a></b> to the project sponsoring features.</li></ul>
-<h1><a name="HOWTOs about selected features">HOWTOs about selected features</a></h1><ul><li> <a href="ReplicationHowto.html">The Redis Replication HOWTO</a> is what you need to read in order to understand how Redis master <code name="code" class="python">&lt;-&gt;</code> slave replication works.</li><li> <a href="AppendOnlyFileHowto.html">The Append Only File HOWTO</a> explains how the alternative Redis durability mode works. AOF is an alternative to snapshotting on disk from time to time (the default).</li></ul>
+                    &iuml;&raquo;&iquest;= Redis Documentation =<br/><br/><a href="http://pyha.ru/wiki/index.php?title=Redis:index" target="_blank">Russian Translation</a>Hello! The followings are pointers to different parts of the Redis Documentation.<br/><br/><ul><li> New! You can now <a href="http://try.redis-db.com" target="_blank">Try Redis directly in your browser!</a>.</li><li> <a href="README.html">The README</a> is the best starting point to know more about the project.</li><li> <a href="QuickStart.html">This short Quick Start</a> provides a five minutes step-by-step istructions on how to download, compile, run and test the basic workings of a Redis server.</li><li> <a href="CommandReference.html">The command reference</a> is a description of all the Redis commands with links to command specific pages. You can also download the <a href="http://go2.wordpress.com/?id=725X1342&site=masonoise.wordpress.com&url=http%3A%2F%2Fmasonoise.files.wordpress.com%2F2010%2F03%2Fredis-cheatsheet-v1.pdf" target="_blank">Redis Commands Cheat-Sheet</a> provided by Mason Jones (btw some command may be missing, the primary source is the wiki).</li><li> <a href="TwitterAlikeExample.html">This is a tutorial about creating a Twitter clone using *only* Redis as database, no relational DB at all is used</a>, it is a good start to understand the key-value database paradigm.</li><li> <a href="IntroductionToRedisDataTypes.html">A Fifteen Minutes Introduction to the Redis Data Types</a> explains how Redis data types work and the basic patterns of working with Redis.</li><li> <a href="http://simonwillison.net/static/2010/redis-tutorial/" target="_blank">the Simon Willison Redis Tutorial</a> is a <b>must read</b>, very good documentation where you will find a lot of real world ideas and use cases.</li><li> <a href="Features.html">The features page</a> (currently in draft) is a good start to understand the strength and limitations of Redis.</li><li> <a href="Benchmarks.html">The benchmark page</a> is about the speed performances of Redis.</li><li> <a href="FAQ.html">Our FAQ</a> contains of course some answers to common questions about Redis.</li><li> <a href="http://www.rediscookbook.org/" target="_blank">The Redis Cookbook</a> is a collaborative effort to provide some good recipe ;)</li></ul>
+<h1><a name="HOWTOs about selected features">HOWTOs about selected features</a></h1><ul><li> <a href="ReplicationHowto.html">The Redis Replication HOWTO</a> is what you need to read in order to understand how Redis master <code name="code" class="python">&lt;-&gt;</code> slave replication works.</li><li> <a href="AppendOnlyFileHowto.html">The Append Only File HOWTO</a> explains how the alternative Redis durability mode works. AOF is an alternative to snapshotting on disk from time to time (the default).</li><li> <a href="VirtualMemoryUserGuide.html">Virtual Memory User Guide</a>. A simple to understand guide about using and configuring the Redis Virtual Memory.</li></ul>
 <h1><a name="Hacking">Hacking</a></h1>
 <ul><li> <a href="ProtocolSpecification.html">The Protocol Specification</a> is all you need in order to implement a Redis client library for a missing language. PHP, Python, Ruby and Erlang are already supported.</li></ul>
-<h1><a name="Videos">Videos</a></h1><ul><li> <a href="http://mwrc2009.confreaks.com/13-mar-2009-19-24-redis-key-value-nirvana-ezra-zygmuntowicz.html" target="_blank">watch the Ezra Zygmuntowicz talk about Redis</a> to know the most important Redis ideas in few minutes.</li></ul>
+<ul><li> Look at <a href="RedisInternals.html">Redis Internals</a> if you are interested in the implementation details of the Redis server.</li></ul>
+<h1><a name="Videos">Videos</a></h1><ul><li> <a href="http://mwrc2009.confreaks.com/13-mar-2009-19-24-redis-key-value-nirvana-ezra-zygmuntowicz.html" target="_blank">watch the Ezra Zygmuntowicz talk about Redis</a> to know the most important Redis ideas in few minutes.</li><li> <a href="http://www.ustream.tv/recorded/7855635" target="_blank">Salvatore Sanfilippo and Pieter Noordhuis at the SF Redis Meetup</a></li></ul>
                 </div>
         
             </div>

redis-cli.c

@@ -1,531 +0,0 @@
-/* Redis CLI (command line interface)
- *
- * Copyright (c) 2009-2010, Salvatore Sanfilippo <antirez at gmail dot com>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *   * Redistributions of source code must retain the above copyright notice,
- *     this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in the
- *     documentation and/or other materials provided with the distribution.
- *   * Neither the name of Redis nor the names of its contributors may be used
- *     to endorse or promote products derived from this software without
- *     specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "fmacros.h"
-
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <unistd.h>
-
-#include "anet.h"
-#include "sds.h"
-#include "adlist.h"
-#include "zmalloc.h"
-
-#define REDIS_CMD_INLINE 1
-#define REDIS_CMD_BULK 2
-#define REDIS_CMD_MULTIBULK 4
-
-#define REDIS_NOTUSED(V) ((void) V)
-
-static struct config {
-    char *hostip;
-    int hostport;
-    long repeat;
-    int dbnum;
-    int interactive;
-    char *auth;
-} config;
-
-struct redisCommand {
-    char *name;
-    int arity;
-    int flags;
-};
-
-static struct redisCommand cmdTable[] = {
-    {"auth",2,REDIS_CMD_INLINE},
-    {"get",2,REDIS_CMD_INLINE},
-    {"set",3,REDIS_CMD_BULK},
-    {"setnx",3,REDIS_CMD_BULK},
-    {"append",3,REDIS_CMD_BULK},
-    {"substr",4,REDIS_CMD_INLINE},
-    {"del",-2,REDIS_CMD_INLINE},
-    {"exists",2,REDIS_CMD_INLINE},
-    {"incr",2,REDIS_CMD_INLINE},
-    {"decr",2,REDIS_CMD_INLINE},
-    {"rpush",3,REDIS_CMD_BULK},
-    {"lpush",3,REDIS_CMD_BULK},
-    {"rpop",2,REDIS_CMD_INLINE},
-    {"lpop",2,REDIS_CMD_INLINE},
-    {"brpop",-3,REDIS_CMD_INLINE},
-    {"blpop",-3,REDIS_CMD_INLINE},
-    {"llen",2,REDIS_CMD_INLINE},
-    {"lindex",3,REDIS_CMD_INLINE},
-    {"lset",4,REDIS_CMD_BULK},
-    {"lrange",4,REDIS_CMD_INLINE},
-    {"ltrim",4,REDIS_CMD_INLINE},
-    {"lrem",4,REDIS_CMD_BULK},
-    {"rpoplpush",3,REDIS_CMD_BULK},
-    {"sadd",3,REDIS_CMD_BULK},
-    {"srem",3,REDIS_CMD_BULK},
-    {"smove",4,REDIS_CMD_BULK},
-    {"sismember",3,REDIS_CMD_BULK},
-    {"scard",2,REDIS_CMD_INLINE},
-    {"spop",2,REDIS_CMD_INLINE},
-    {"srandmember",2,REDIS_CMD_INLINE},
-    {"sinter",-2,REDIS_CMD_INLINE},
-    {"sinterstore",-3,REDIS_CMD_INLINE},
-    {"sunion",-2,REDIS_CMD_INLINE},
-    {"sunionstore",-3,REDIS_CMD_INLINE},
-    {"sdiff",-2,REDIS_CMD_INLINE},
-    {"sdiffstore",-3,REDIS_CMD_INLINE},
-    {"smembers",2,REDIS_CMD_INLINE},
-    {"zadd",4,REDIS_CMD_BULK},
-    {"zincrby",4,REDIS_CMD_BULK},
-    {"zrem",3,REDIS_CMD_BULK},
-    {"zremrangebyscore",4,REDIS_CMD_INLINE},
-    {"zmerge",-3,REDIS_CMD_INLINE},
-    {"zmergeweighed",-4,REDIS_CMD_INLINE},
-    {"zrange",-4,REDIS_CMD_INLINE},
-    {"zrank",3,REDIS_CMD_BULK},
-    {"zrevrank",3,REDIS_CMD_BULK},
-    {"zrangebyscore",-4,REDIS_CMD_INLINE},
-    {"zcount",4,REDIS_CMD_INLINE},
-    {"zrevrange",-4,REDIS_CMD_INLINE},
-    {"zcard",2,REDIS_CMD_INLINE},
-    {"zscore",3,REDIS_CMD_BULK},
-    {"incrby",3,REDIS_CMD_INLINE},
-    {"decrby",3,REDIS_CMD_INLINE},
-    {"getset",3,REDIS_CMD_BULK},
-    {"randomkey",1,REDIS_CMD_INLINE},
-    {"select",2,REDIS_CMD_INLINE},
-    {"move",3,REDIS_CMD_INLINE},
-    {"rename",3,REDIS_CMD_INLINE},
-    {"renamenx",3,REDIS_CMD_INLINE},
-    {"keys",2,REDIS_CMD_INLINE},
-    {"dbsize",1,REDIS_CMD_INLINE},
-    {"ping",1,REDIS_CMD_INLINE},
-    {"echo",2,REDIS_CMD_BULK},
-    {"save",1,REDIS_CMD_INLINE},
-    {"bgsave",1,REDIS_CMD_INLINE},
-    {"rewriteaof",1,REDIS_CMD_INLINE},
-    {"bgrewriteaof",1,REDIS_CMD_INLINE},
-    {"shutdown",1,REDIS_CMD_INLINE},
-    {"lastsave",1,REDIS_CMD_INLINE},
-    {"type",2,REDIS_CMD_INLINE},
-    {"flushdb",1,REDIS_CMD_INLINE},
-    {"flushall",1,REDIS_CMD_INLINE},
-    {"sort",-2,REDIS_CMD_INLINE},
-    {"info",1,REDIS_CMD_INLINE},
-    {"mget",-2,REDIS_CMD_INLINE},
-    {"expire",3,REDIS_CMD_INLINE},
-    {"expireat",3,REDIS_CMD_INLINE},
-    {"ttl",2,REDIS_CMD_INLINE},
-    {"slaveof",3,REDIS_CMD_INLINE},
-    {"debug",-2,REDIS_CMD_INLINE},
-    {"mset",-3,REDIS_CMD_MULTIBULK},
-    {"msetnx",-3,REDIS_CMD_MULTIBULK},
-    {"monitor",1,REDIS_CMD_INLINE},
-    {"multi",1,REDIS_CMD_INLINE},
-    {"exec",1,REDIS_CMD_INLINE},
-    {"discard",1,REDIS_CMD_INLINE},
-    {"hset",4,REDIS_CMD_MULTIBULK},
-    {"hget",3,REDIS_CMD_BULK},
-    {"hdel",3,REDIS_CMD_BULK},
-    {"hlen",2,REDIS_CMD_INLINE},
-    {"hkeys",2,REDIS_CMD_INLINE},
-    {"hvals",2,REDIS_CMD_INLINE},
-    {"hgetall",2,REDIS_CMD_INLINE},
-    {"hexists",3,REDIS_CMD_BULK},
-    {NULL,0,0}
-};
-
-static int cliReadReply(int fd);
-static void usage();
-
-static struct redisCommand *lookupCommand(char *name) {
-    int j = 0;
-    while(cmdTable[j].name != NULL) {
-        if (!strcasecmp(name,cmdTable[j].name)) return &cmdTable[j];
-        j++;
-    }
-    return NULL;
-}
-
-static int cliConnect(void) {
-    char err[ANET_ERR_LEN];
-    static int fd = ANET_ERR;
-
-    if (fd == ANET_ERR) {
-        fd = anetTcpConnect(err,config.hostip,config.hostport);
-        if (fd == ANET_ERR) {
-            fprintf(stderr, "Could not connect to Redis at %s:%d: %s", config.hostip, config.hostport, err);
-            return -1;
-        }
-        anetTcpNoDelay(NULL,fd);
-    }
-    return fd;
-}
-
-static sds cliReadLine(int fd) {
-    sds line = sdsempty();
-
-    while(1) {
-        char c;
-        ssize_t ret;
-
-        ret = read(fd,&c,1);
-        if (ret == -1) {
-            sdsfree(line);
-            return NULL;
-        } else if ((ret == 0) || (c == '\n')) {
-            break;
-        } else {
-            line = sdscatlen(line,&c,1);
-        }
-    }
-    return sdstrim(line,"\r\n");
-}
-
-static int cliReadSingleLineReply(int fd, int quiet) {
-    sds reply = cliReadLine(fd);
-
-    if (reply == NULL) return 1;
-    if (!quiet)
-        printf("%s\n", reply);
-    sdsfree(reply);
-    return 0;
-}
-
-static int cliReadBulkReply(int fd) {
-    sds replylen = cliReadLine(fd);
-    char *reply, crlf[2];
-    int bulklen;
-
-    if (replylen == NULL) return 1;
-    bulklen = atoi(replylen);
-    if (bulklen == -1) {
-        sdsfree(replylen);
-        printf("(nil)\n");
-        return 0;
-    }
-    reply = zmalloc(bulklen);
-    anetRead(fd,reply,bulklen);
-    anetRead(fd,crlf,2);
-    if (bulklen && fwrite(reply,bulklen,1,stdout) == 0) {
-        zfree(reply);
-        return 1;
-    }
-    if (isatty(fileno(stdout)) && reply[bulklen-1] != '\n')
-        printf("\n");
-    zfree(reply);
-    return 0;
-}
-
-static int cliReadMultiBulkReply(int fd) {
-    sds replylen = cliReadLine(fd);
-    int elements, c = 1;
-
-    if (replylen == NULL) return 1;
-    elements = atoi(replylen);
-    if (elements == -1) {
-        sdsfree(replylen);
-        printf("(nil)\n");
-        return 0;
-    }
-    if (elements == 0) {
-        printf("(empty list or set)\n");
-    }
-    while(elements--) {
-        printf("%d. ", c);
-        if (cliReadReply(fd)) return 1;
-        c++;
-    }
-    return 0;
-}
-
-static int cliReadReply(int fd) {
-    char type;
-
-    if (anetRead(fd,&type,1) <= 0) exit(1);
-    switch(type) {
-    case '-':
-        printf("(error) ");
-        cliReadSingleLineReply(fd,0);
-        return 1;
-    case '+':
-        return cliReadSingleLineReply(fd,0);
-    case ':':
-        printf("(integer) ");
-        return cliReadSingleLineReply(fd,0);
-    case '$':
-        return cliReadBulkReply(fd);
-    case '*':
-        return cliReadMultiBulkReply(fd);
-    default:
-        printf("protocol error, got '%c' as reply type byte\n", type);
-        return 1;
-    }
-}
-
-static int selectDb(int fd) {
-    int retval;
-    sds cmd;
-    char type;
-
-    if (config.dbnum == 0)
-        return 0;
-
-    cmd = sdsempty();
-    cmd = sdscatprintf(cmd,"SELECT %d\r\n",config.dbnum);
-    anetWrite(fd,cmd,sdslen(cmd));
-    anetRead(fd,&type,1);
-    if (type <= 0 || type != '+') return 1;
-    retval = cliReadSingleLineReply(fd,1);
-    if (retval) {
-        return retval;
-    }
-    return 0;
-}
-
-static int cliSendCommand(int argc, char **argv) {
-    struct redisCommand *rc = lookupCommand(argv[0]);
-    int fd, j, retval = 0;
-    int read_forever = 0;
-    sds cmd;
-
-    if (!rc) {
-        fprintf(stderr,"Unknown command '%s'\n",argv[0]);
-        return 1;
-    }
-
-    if ((rc->arity > 0 && argc != rc->arity) ||
-        (rc->arity < 0 && argc < -rc->arity)) {
-            fprintf(stderr,"Wrong number of arguments for '%s'\n",rc->name);
-            return 1;
-    }
-    if (!strcasecmp(rc->name,"monitor")) read_forever = 1;
-    if ((fd = cliConnect()) == -1) return 1;
-
-    /* Select db number */
-    retval = selectDb(fd);
-    if (retval) {
-        fprintf(stderr,"Error setting DB num\n");
-        return 1;
-    }
-
-    while(config.repeat--) {
-        /* Build the command to send */
-        cmd = sdsempty();
-        if (rc->flags & REDIS_CMD_MULTIBULK) {
-            cmd = sdscatprintf(cmd,"*%d\r\n",argc);
-            for (j = 0; j < argc; j++) {
-                cmd = sdscatprintf(cmd,"$%lu\r\n",
-                    (unsigned long)sdslen(argv[j]));
-                cmd = sdscatlen(cmd,argv[j],sdslen(argv[j]));
-                cmd = sdscatlen(cmd,"\r\n",2);
-            }
-        } else {
-            for (j = 0; j < argc; j++) {
-                if (j != 0) cmd = sdscat(cmd," ");
-                if (j == argc-1 && rc->flags & REDIS_CMD_BULK) {
-                    cmd = sdscatprintf(cmd,"%lu",
-                        (unsigned long)sdslen(argv[j]));
-                } else {
-                    cmd = sdscatlen(cmd,argv[j],sdslen(argv[j]));
-                }
-            }
-            cmd = sdscat(cmd,"\r\n");
-            if (rc->flags & REDIS_CMD_BULK) {
-                cmd = sdscatlen(cmd,argv[argc-1],sdslen(argv[argc-1]));
-                cmd = sdscatlen(cmd,"\r\n",2);
-            }
-        }
-        anetWrite(fd,cmd,sdslen(cmd));
-        sdsfree(cmd);
-
-        while (read_forever) {
-            cliReadSingleLineReply(fd,0);
-        }
-
-        retval = cliReadReply(fd);
-        if (retval) {
-            return retval;
-        }
-    }
-    return 0;
-}
-
-static int parseOptions(int argc, char **argv) {
-    int i;
-
-    for (i = 1; i < argc; i++) {
-        int lastarg = i==argc-1;
-
-        if (!strcmp(argv[i],"-h") && !lastarg) {
-            char *ip = zmalloc(32);
-            if (anetResolve(NULL,argv[i+1],ip) == ANET_ERR) {
-                printf("Can't resolve %s\n", argv[i]);
-                exit(1);
-            }
-            config.hostip = ip;
-            i++;
-        } else if (!strcmp(argv[i],"-h") && lastarg) {
-            usage();
-        } else if (!strcmp(argv[i],"-p") && !lastarg) {
-            config.hostport = atoi(argv[i+1]);
-            i++;
-        } else if (!strcmp(argv[i],"-r") && !lastarg) {
-            config.repeat = strtoll(argv[i+1],NULL,10);
-            i++;
-        } else if (!strcmp(argv[i],"-n") && !lastarg) {
-            config.dbnum = atoi(argv[i+1]);
-            i++;
-        } else if (!strcmp(argv[i],"-a") && !lastarg) {
-            config.auth = argv[i+1];
-            i++;
-        } else if (!strcmp(argv[i],"-i")) {
-            config.interactive = 1;
-        } else {
-            break;
-        }
-    }
-    return i;
-}
-
-static sds readArgFromStdin(void) {
-    char buf[1024];
-    sds arg = sdsempty();
-
-    while(1) {
-        int nread = read(fileno(stdin),buf,1024);
-
-        if (nread == 0) break;
-        else if (nread == -1) {
-            perror("Reading from standard input");
-            exit(1);
-        }
-        arg = sdscatlen(arg,buf,nread);
-    }
-    return arg;
-}
-
-static void usage() {
-    fprintf(stderr, "usage: redis-cli [-h host] [-p port] [-a authpw] [-r repeat_times] [-n db_num] [-i] cmd arg1 arg2 arg3 ... argN\n");
-    fprintf(stderr, "usage: echo \"argN\" | redis-cli [-h host] [-a authpw] [-p port] [-r repeat_times] [-n db_num] cmd arg1 arg2 ... arg(N-1)\n");
-    fprintf(stderr, "\nIf a pipe from standard input is detected this data is used as last argument.\n\n");
-    fprintf(stderr, "example: cat /etc/passwd | redis-cli set my_passwd\n");
-    fprintf(stderr, "example: redis-cli get my_passwd\n");
-    fprintf(stderr, "example: redis-cli -r 100 lpush mylist x\n");
-    fprintf(stderr, "\nRun in interactive mode: redis-cli -i or just don't pass any command\n");
-    exit(1);
-}
-
-/* Turn the plain C strings into Sds strings */
-static char **convertToSds(int count, char** args) {
-  int j;
-  char **sds = zmalloc(sizeof(char*)*count+1);
-
-  for(j = 0; j < count; j++)
-    sds[j] = sdsnew(args[j]);
-
-  return sds;
-}
-
-static char *prompt(char *line, int size) {
-    char *retval;
-
-    do {
-        printf(">> ");
-        retval = fgets(line, size, stdin);
-    } while (retval && *line == '\n');
-    line[strlen(line) - 1] = '\0';
-
-    return retval;
-}
-
-static void repl() {
-    int size = 4096, max = size >> 1, argc;
-    char buffer[size];
-    char *line = buffer;
-    char **ap, *args[max];
-
-    if (config.auth != NULL) {
-        char *authargv[2];
-
-        authargv[0] = "AUTH";
-        authargv[1] = config.auth;
-        cliSendCommand(2, convertToSds(2, authargv));
-    }
-
-    while (prompt(line, size)) {
-        argc = 0;
-
-        for (ap = args; (*ap = strsep(&line, " \t")) != NULL;) {
-            if (**ap != '\0') {
-                if (argc >= max) break;
-                if (strcasecmp(*ap,"quit") == 0 || strcasecmp(*ap,"exit") == 0)
-                    exit(0);
-                ap++;
-                argc++;
-            }
-        }
-
-        config.repeat = 1;
-        cliSendCommand(argc, convertToSds(argc, args));
-        line = buffer;
-    }
-
-    exit(0);
-}
-
-int main(int argc, char **argv) {
-    int firstarg;
-    char **argvcopy;
-    struct redisCommand *rc;
-
-    config.hostip = "127.0.0.1";
-    config.hostport = 6379;
-    config.repeat = 1;
-    config.dbnum = 0;
-    config.interactive = 0;
-    config.auth = NULL;
-
-    firstarg = parseOptions(argc,argv);
-    argc -= firstarg;
-    argv += firstarg;
-
-    if (argc == 0 || config.interactive == 1) repl();
-
-    argvcopy = convertToSds(argc, argv);
-
-    /* Read the last argument from stdandard input if needed */
-    if ((rc = lookupCommand(argv[0])) != NULL) {
-      if (rc->arity > 0 && argc == rc->arity-1) {
-        sds lastarg = readArgFromStdin();
-        argvcopy[argc] = lastarg;
-        argc++;
-      }
-    }
-
-    return cliSendCommand(argc, argvcopy);
-}

redis.conf

@@ -1,18 +1,30 @@
 # Redis configuration file example
 
+# Note on units: when memory size is needed, it is possible to specifiy
+# it in the usual form of 1k 5GB 4M and so forth:
+#
+# 1k => 1000 bytes
+# 1kb => 1024 bytes
+# 1m => 1000000 bytes
+# 1mb => 1024*1024 bytes
+# 1g => 1000000000 bytes
+# 1gb => 1024*1024*1024 bytes
+#
+# units are case insensitive so 1GB 1Gb 1gB are all the same.
+
 # By default Redis does not run as a daemon. Use 'yes' if you need it.
 # Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
 daemonize no
 
-# When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
-# You can specify a custom pid file location here.
+# When running daemonized, Redis writes a pid file in /var/run/redis.pid by
+# default. You can specify a custom pid file location here.
 pidfile /var/run/redis.pid
 
 # Accept connections on the specified port, default is 6379
 port 6379
 
 # If you want you can bind a single interface, if the bind option is not
-# specified all the interfaces will listen for connections.
+# specified all the interfaces will listen for incoming connections.
 #
 # bind 127.0.0.1
 
@@ -28,7 +40,7 @@ timeout 300
 loglevel verbose
 
 # Specify the log file name. Also 'stdout' can be used to force
-# the demon to log on the standard output. Note that if you use standard
+# Redis to log on the standard output. Note that if you use standard
 # output for logging but daemonize, logs will be sent to /dev/null
 logfile stdout
 
@@ -66,8 +78,14 @@ rdbcompression yes
 # The filename where to dump the DB
 dbfilename dump.rdb
 
-# For default save/load DB in/from the working directory
-# Note that you must specify a directory not a file name.
+# The working directory.
+#
+# The DB will be written inside this directory, with the filename specified
+# above using the 'dbfilename' configuration directive.
+# 
+# Also the Append Only File will be created inside this directory.
+# 
+# Note that you must specify a directory here, not a file name.
 dir ./
 
 ################################# REPLICATION #################################
@@ -94,6 +112,10 @@ dir ./
 #
 # This should stay commented out for backward compatibility and because most
 # people do not need auth (e.g. they run their own servers).
+# 
+# Warning: since Redis is pretty fast an outside user can try up to
+# 150k passwords per second against a good box. This means that you should
+# use a very strong password otherwise it will be very easy to break.
 #
 # requirepass foobared
 
@@ -133,7 +155,7 @@ dir ./
 # happens this is the preferred way to run Redis. If instead you care a lot
 # about your data and don't want to that a single record can get lost you should
 # enable the append only mode: when this mode is enabled Redis will append
-# every write operation received in the file appendonly.log. This file will
+# every write operation received in the file appendonly.aof. This file will
 # be read on startup in order to rebuild the full dataset in memory.
 #
 # Note that you can have both the async dumps and the append only file if you
@@ -141,13 +163,14 @@ dir ./
 # Still if append only mode is enabled Redis will load the data from the
 # log file at startup ignoring the dump.rdb file.
 #
-# The name of the append only file is "appendonly.log"
-#
 # IMPORTANT: Check the BGREWRITEAOF to check how to rewrite the append
 # log file in background when it gets too big.
 
 appendonly no
 
+# The name of the append only file (default: "appendonly.aof")
+# appendfilename appendonly.aof
+
 # The fsync() call tells the Operating System to actually write data on disk
 # instead to wait for more data in the output buffer. Some OS will really flush 
 # data on disk, some other OS will just try to do it ASAP.
@@ -172,6 +195,26 @@ appendonly no
 appendfsync everysec
 # appendfsync no
 
+# When the AOF fsync policy is set to always or everysec, and a background
+# saving process (a background save or AOF log background rewriting) is
+# performing a lot of I/O against the disk, in some Linux configurations
+# Redis may block too long on the fsync() call. Note that there is no fix for
+# this currently, as even performing fsync in a different thread will block
+# our synchronous write(2) call.
+#
+# In order to mitigate this problem it's possible to use the following option
+# that will prevent fsync() from being called in the main process while a
+# BGSAVE or BGREWRITEAOF is in progress.
+#
+# This means that while another child is saving the durability of Redis is
+# the same as "appendfsync none", that in pratical terms means that it is
+# possible to lost up to 30 seconds of log in the worst scenario (with the
+# default Linux settings).
+# 
+# If you have latency problems turn this to "yes". Otherwise leave it as
+# "no" that is the safest pick from the point of view of durability.
+no-appendfsync-on-rewrite no
+
 ################################ VIRTUAL MEMORY ###############################
 
 # Virtual Memory allows Redis to work with datasets bigger than the actual
@@ -188,20 +231,16 @@ vm-enabled no
 
 # This is the path of the Redis swap file. As you can guess, swap files
 # can't be shared by different Redis instances, so make sure to use a swap
-# file for every redis process you are running.
+# file for every redis process you are running. Redis will complain if the
+# swap file is already in use.
 #
-# The swap file name may contain "%p" that is substituted with the PID of
-# the Redis process, so the default name /tmp/redis-%p.vm will work even
-# with multiple instances as Redis will use, for example, redis-811.vm
-# for one instance and redis-593.vm for another one.
-#
-# Useless to say, the best kind of disk for a Redis swap file (that's accessed
-# at random) is a Solid State Disk (SSD).
+# The best kind of storage for the Redis swap file (that's accessed at random) 
+# is a Solid State Disk (SSD).
 #
 # *** WARNING *** if you are using a shared hosting the default of putting
 # the swap file under /tmp is not secure. Create a dir with access granted
 # only to Redis user and configure Redis to create the swap file there.
-vm-swap-file /tmp/redis-%p.vm
+vm-swap-file /tmp/redis.swap
 
 # vm-max-memory configures the VM to use at max the specified amount of
 # RAM. Everything that deos not fit will be swapped on disk *if* possible, that
@@ -255,23 +294,6 @@ vm-max-threads 4
 # in terms of number of queries per second. Use 'yes' if unsure.
 glueoutputbuf yes
 
-# Use object sharing. Can save a lot of memory if you have many common
-# string in your dataset, but performs lookups against the shared objects
-# pool so it uses more CPU and can be a bit slower. Usually it's a good
-# idea.
-#
-# When object sharing is enabled (shareobjects yes) you can use
-# shareobjectspoolsize to control the size of the pool used in order to try
-# object sharing. A bigger pool size will lead to better sharing capabilities.
-# In general you want this value to be at least the double of the number of
-# very common strings you have in your dataset.
-#
-# WARNING: object sharing is experimental, don't enable this feature
-# in production before of Redis 1.0-stable. Still please try this feature in
-# your development environment so that we can test it better.
-shareobjects no
-shareobjectspoolsize 1024
-
 # Hashes are encoded in a special way (much more memory efficient) when they
 # have at max a given numer of elements, and the biggest element does not
 # exceed a given threshold. You can configure this limits with the following
@@ -279,6 +301,26 @@ shareobjectspoolsize 1024
 hash-max-zipmap-entries 64
 hash-max-zipmap-value 512
 
+# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in
+# order to help rehashing the main Redis hash table (the one mapping top-level
+# keys to values). The hash table implementation redis uses (see dict.c)
+# performs a lazy rehashing: the more operation you run into an hash table
+# that is rhashing, the more rehashing "steps" are performed, so if the
+# server is idle the rehashing is never complete and some more memory is used
+# by the hash table.
+# 
+# The default is to use this millisecond 10 times every second in order to
+# active rehashing the main dictionaries, freeing memory when possible.
+#
+# If unsure:
+# use "activerehashing no" if you have hard latency requirements and it is
+# not a good thing in your environment that Redis can reply form time to time
+# to queries with 2 milliseconds delay.
+#
+# use "activerehashing yes" if you don't have such hard requirements but
+# want to free memory asap when possible.
+activerehashing yes
+
 ################################## INCLUDES ###################################
 
 # Include one or more other config files here.  This is useful if you

redis.tcl

@@ -1,131 +0,0 @@
-# Tcl clinet library - used by test-redis.tcl script for now
-# Copyright (C) 2009 Salvatore Sanfilippo
-# Released under the BSD license like Redis itself
-#
-# Example usage:
-#
-# set r [redis 127.0.0.1 6379]
-# $r lpush mylist foo
-# $r lpush mylist bar
-# $r lrange mylist 0 -1
-# $r close
-
-package provide redis 0.1
-
-namespace eval redis {}
-set ::redis::id 0
-array set ::redis::fd {}
-array set ::redis::bulkarg {}
-array set ::redis::multibulkarg {}
-
-# Flag commands requiring last argument as a bulk write operation
-foreach redis_bulk_cmd {
-    set setnx rpush lpush lset lrem sadd srem sismember echo getset smove zadd zrem zscore zincrby append zrank zrevrank hget hdel hexists
-} {
-    set ::redis::bulkarg($redis_bulk_cmd) {}
-}
-
-# Flag commands requiring last argument as a bulk write operation
-foreach redis_multibulk_cmd {
-    mset msetnx hset
-} {
-    set ::redis::multibulkarg($redis_multibulk_cmd) {}
-}
-
-unset redis_bulk_cmd
-unset redis_multibulk_cmd
-
-proc redis {{server 127.0.0.1} {port 6379}} {
-    set fd [socket $server $port]
-    fconfigure $fd -translation binary
-    set id [incr ::redis::id]
-    set ::redis::fd($id) $fd
-    interp alias {} ::redis::redisHandle$id {} ::redis::__dispatch__ $id
-}
-
-proc ::redis::__dispatch__ {id method args} {
-    set fd $::redis::fd($id)
-    if {[info command ::redis::__method__$method] eq {}} {
-        if {[info exists ::redis::bulkarg($method)]} {
-            set cmd "$method "
-            append cmd [join [lrange $args 0 end-1]]
-            append cmd " [string length [lindex $args end]]\r\n"
-            append cmd [lindex $args end]
-            ::redis::redis_writenl $fd $cmd
-        } elseif {[info exists ::redis::multibulkarg($method)]} {
-            set cmd "*[expr {[llength $args]+1}]\r\n"
-            append cmd "$[string length $method]\r\n$method\r\n"
-            foreach a $args {
-                append cmd "$[string length $a]\r\n$a\r\n"
-            }
-            ::redis::redis_write $fd $cmd
-            flush $fd
-        } else {
-            set cmd "$method "
-            append cmd [join $args]
-            ::redis::redis_writenl $fd $cmd
-        }
-        ::redis::redis_read_reply $fd
-    } else {
-        uplevel 1 [list ::redis::__method__$method $id $fd] $args
-    }
-}
-
-proc ::redis::__method__close {id fd} {
-    catch {close $fd}
-    catch {unset ::redis::fd($id)}
-    catch {interp alias {} ::redis::redisHandle$id {}}
-}
-
-proc ::redis::__method__channel {id fd} {
-    return $fd
-}
-
-proc ::redis::redis_write {fd buf} {
-    puts -nonewline $fd $buf
-}
-
-proc ::redis::redis_writenl {fd buf} {
-    redis_write $fd $buf
-    redis_write $fd "\r\n"
-    flush $fd
-}
-
-proc ::redis::redis_readnl {fd len} {
-    set buf [read $fd $len]
-    read $fd 2 ; # discard CR LF
-    return $buf
-}
-
-proc ::redis::redis_bulk_read {fd} {
-    set count [redis_read_line $fd]
-    if {$count == -1} return {}
-    set buf [redis_readnl $fd $count]
-    return $buf
-}
-
-proc ::redis::redis_multi_bulk_read fd {
-    set count [redis_read_line $fd]
-    if {$count == -1} return {}
-    set l {}
-    for {set i 0} {$i < $count} {incr i} {
-        lappend l [redis_read_reply $fd]
-    }
-    return $l
-}
-
-proc ::redis::redis_read_line fd {
-    string trim [gets $fd]
-}
-
-proc ::redis::redis_read_reply fd {
-    set type [read $fd 1]
-    switch -exact -- $type {
-        : -
-        + {redis_read_line $fd}
-        - {return -code error [redis_read_line $fd]}
-        $ {redis_bulk_read $fd}
-        * {redis_multi_bulk_read $fd}
-        default {return -code error "Bad protocol, $type as reply type byte"}
-    }
-}

src/Makefile

@@ -0,0 +1,160 @@
+# Redis Makefile
+# Copyright (C) 2009 Salvatore Sanfilippo <antirez at gmail dot com>
+# This file is released under the BSD license, see the COPYING file
+
+release_hdr := $(shell sh -c './mkreleasehdr.sh')
+uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
+OPTIMIZATION?=-O2
+ifeq ($(uname_S),SunOS)
+  CFLAGS?= -std=c99 -pedantic $(OPTIMIZATION) -Wall -W -D__EXTENSIONS__ -D_XPG6
+  CCLINK?= -ldl -lnsl -lsocket -lm -lpthread
+else
+  CFLAGS?= -std=c99 -pedantic $(OPTIMIZATION) -Wall -W $(ARCH) $(PROF)
+  CCLINK?= -lm -pthread
+endif
+CCOPT= $(CFLAGS) $(CCLINK) $(ARCH) $(PROF)
+DEBUG?= -g -rdynamic -ggdb 
+
+INSTALL_TOP= /usr/local
+INSTALL_BIN= $(INSTALL_TOP)/bin
+INSTALL= cp -p
+
+OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o vm.o pubsub.o multi.o debug.o sort.o intset.o
+BENCHOBJ = ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o
+CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o linenoise.o
+CHECKDUMPOBJ = redis-check-dump.o lzf_c.o lzf_d.o
+CHECKAOFOBJ = redis-check-aof.o
+
+PRGNAME = redis-server
+BENCHPRGNAME = redis-benchmark
+CLIPRGNAME = redis-cli
+CHECKDUMPPRGNAME = redis-check-dump
+CHECKAOFPRGNAME = redis-check-aof
+
+all: redis-server redis-benchmark redis-cli redis-check-dump redis-check-aof
+
+
+# Deps (use make dep to generate this)
+adlist.o: adlist.c adlist.h zmalloc.h
+ae.o: ae.c ae.h zmalloc.h config.h ae_kqueue.c
+ae_epoll.o: ae_epoll.c
+ae_kqueue.o: ae_kqueue.c
+ae_select.o: ae_select.c
+anet.o: anet.c fmacros.h anet.h
+aof.o: aof.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
+  zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
+config.o: config.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
+  zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
+db.o: db.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
+  zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
+debug.o: debug.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
+  zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h sha1.h
+dict.o: dict.c fmacros.h dict.h zmalloc.h
+intset.o: intset.c intset.h zmalloc.h
+linenoise.o: linenoise.c fmacros.h
+lzf_c.o: lzf_c.c lzfP.h
+lzf_d.o: lzf_d.c lzfP.h
+multi.o: multi.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
+  zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
+networking.o: networking.c redis.h fmacros.h config.h ae.h sds.h dict.h \
+  adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
+object.o: object.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
+  zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
+pqsort.o: pqsort.c
+pubsub.o: pubsub.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
+  zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
+rdb.o: rdb.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
+  zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h lzf.h
+redis-benchmark.o: redis-benchmark.c fmacros.h ae.h anet.h sds.h adlist.h \
+  zmalloc.h
+redis-check-aof.o: redis-check-aof.c fmacros.h config.h
+redis-check-dump.o: redis-check-dump.c lzf.h
+redis-cli.o: redis-cli.c fmacros.h version.h anet.h sds.h adlist.h \
+  zmalloc.h linenoise.h
+redis.o: redis.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
+  zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
+release.o: release.c release.h
+replication.o: replication.c redis.h fmacros.h config.h ae.h sds.h dict.h \
+  adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
+sds.o: sds.c sds.h zmalloc.h
+sha1.o: sha1.c sha1.h
+sort.o: sort.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
+  zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h pqsort.h
+t_hash.o: t_hash.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
+  zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
+t_list.o: t_list.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
+  zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
+t_set.o: t_set.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
+  zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
+t_string.o: t_string.c redis.h fmacros.h config.h ae.h sds.h dict.h \
+  adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
+t_zset.o: t_zset.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
+  zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
+util.o: util.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
+  zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
+vm.o: vm.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
+  zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
+ziplist.o: ziplist.c zmalloc.h ziplist.h
+zipmap.o: zipmap.c zmalloc.h
+zmalloc.o: zmalloc.c config.h
+
+redis-server: $(OBJ)
+	$(CC) -o $(PRGNAME) $(CCOPT) $(DEBUG) $(OBJ)
+	@echo ""
+	@echo "Hint: To run 'make test' is a good idea ;)"
+	@echo ""
+
+redis-benchmark: $(BENCHOBJ)
+	$(CC) -o $(BENCHPRGNAME) $(CCOPT) $(DEBUG) $(BENCHOBJ)
+
+redis-cli: $(CLIOBJ)
+	$(CC) -o $(CLIPRGNAME) $(CCOPT) $(DEBUG) $(CLIOBJ)
+
+redis-check-dump: $(CHECKDUMPOBJ)
+	$(CC) -o $(CHECKDUMPPRGNAME) $(CCOPT) $(DEBUG) $(CHECKDUMPOBJ)
+
+redis-check-aof: $(CHECKAOFOBJ)
+	$(CC) -o $(CHECKAOFPRGNAME) $(CCOPT) $(DEBUG) $(CHECKAOFOBJ)
+
+.c.o:
+	$(CC) -c $(CFLAGS) $(DEBUG) $(COMPILE_TIME) $<
+
+clean:
+	rm -rf $(PRGNAME) $(BENCHPRGNAME) $(CLIPRGNAME) $(CHECKDUMPPRGNAME) $(CHECKAOFPRGNAME) *.o *.gcda *.gcno *.gcov
+
+dep:
+	$(CC) -MM *.c
+
+test:
+	(cd ..; tclsh8.5 tests/test_helper.tcl --tags "${TAGS}")
+
+bench:
+	./redis-benchmark
+
+log:
+	git log '--pretty=format:%ad %s (%cn)' --date=short > ../Changelog
+
+32bit:
+	@echo ""
+	@echo "WARNING: if it fails under Linux you probably need to install libc6-dev-i386"
+	@echo ""
+	make ARCH="-m32"
+
+gprof:
+	make PROF="-pg"
+
+gcov:
+	make PROF="-fprofile-arcs -ftest-coverage"
+
+noopt:
+	make OPTIMIZATION=""
+
+32bitgprof:
+	make PROF="-pg" ARCH="-arch i386"
+
+install: all
+	$(INSTALL) $(PRGNAME) $(INSTALL_BIN)
+	$(INSTALL) $(BENCHPRGNAME) $(INSTALL_BIN)
+	$(INSTALL) $(CLIPRGNAME) $(INSTALL_BIN)
+	$(INSTALL) $(CHECKDUMPPRGNAME) $(INSTALL_BIN)
+	$(INSTALL) $(CHECKAOFPRGNAME) $(INSTALL_BIN)

src/adlist.c

@@ -123,6 +123,35 @@ list *listAddNodeTail(list *list, void *value)
     return list;
 }
 
+list *listInsertNode(list *list, listNode *old_node, void *value, int after) {
+    listNode *node;
+
+    if ((node = zmalloc(sizeof(*node))) == NULL)
+        return NULL;
+    node->value = value;
+    if (after) {
+        node->prev = old_node;
+        node->next = old_node->next;
+        if (list->tail == old_node) {
+            list->tail = node;
+        }
+    } else {
+        node->next = old_node;
+        node->prev = old_node->prev;
+        if (list->head == old_node) {
+            list->head = node;
+        }
+    }
+    if (node->prev != NULL) {
+        node->prev->next = node;
+    }
+    if (node->next != NULL) {
+        node->next->prev = node;
+    }
+    list->len++;
+    return list;
+}
+
 /* Remove the specified node from the specified list.
  * It's up to the caller to free the private value of the node.
  *
@@ -183,9 +212,9 @@ void listRewindTail(list *list, listIter *li) {
  * or NULL if there are no more elements, so the classical usage patter
  * is:
  *
- * iter = listGetItarotr(list,<direction>);
- * while ((node = listNextIterator(iter)) != NULL) {
- *     DoSomethingWith(listNodeValue(node));
+ * iter = listGetIterator(list,<direction>);
+ * while ((node = listNext(iter)) != NULL) {
+ *     doSomethingWith(listNodeValue(node));
  * }
  *
  * */

src/adlist.h

@@ -74,6 +74,7 @@ list *listCreate(void);
 void listRelease(list *list);
 list *listAddNodeHead(list *list, void *value);
 list *listAddNodeTail(list *list, void *value);
+list *listInsertNode(list *list, listNode *old_node, void *value, int after);
 void listDelNode(list *list, listNode *node);
 listIter *listGetIterator(list *list, int direction);
 listNode *listNext(listIter *iter);

src/aof.c

@@ -0,0 +1,710 @@
+#include "redis.h"
+
+#include <signal.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <sys/wait.h>
+
+/* Called when the user switches from "appendonly yes" to "appendonly no"
+ * at runtime using the CONFIG command. */
+void stopAppendOnly(void) {
+    flushAppendOnlyFile();
+    aof_fsync(server.appendfd);
+    close(server.appendfd);
+
+    server.appendfd = -1;
+    server.appendseldb = -1;
+    server.appendonly = 0;
+    /* rewrite operation in progress? kill it, wait child exit */
+    if (server.bgsavechildpid != -1) {
+        int statloc;
+
+        if (kill(server.bgsavechildpid,SIGKILL) != -1)
+            wait3(&statloc,0,NULL);
+        /* reset the buffer accumulating changes while the child saves */
+        sdsfree(server.bgrewritebuf);
+        server.bgrewritebuf = sdsempty();
+        server.bgsavechildpid = -1;
+    }
+}
+
+/* Called when the user switches from "appendonly no" to "appendonly yes"
+ * at runtime using the CONFIG command. */
+int startAppendOnly(void) {
+    server.appendonly = 1;
+    server.lastfsync = time(NULL);
+    server.appendfd = open(server.appendfilename,O_WRONLY|O_APPEND|O_CREAT,0644);
+    if (server.appendfd == -1) {
+        redisLog(REDIS_WARNING,"Used tried to switch on AOF via CONFIG, but I can't open the AOF file: %s",strerror(errno));
+        return REDIS_ERR;
+    }
+    if (rewriteAppendOnlyFileBackground() == REDIS_ERR) {
+        server.appendonly = 0;
+        close(server.appendfd);
+        redisLog(REDIS_WARNING,"Used tried to switch on AOF via CONFIG, I can't trigger a background AOF rewrite operation. Check the above logs for more info about the error.",strerror(errno));
+        return REDIS_ERR;
+    }
+    return REDIS_OK;
+}
+
+/* Write the append only file buffer on disk.
+ *
+ * Since we are required to write the AOF before replying to the client,
+ * and the only way the client socket can get a write is entering when the
+ * the event loop, we accumulate all the AOF writes in a memory
+ * buffer and write it on disk using this function just before entering
+ * the event loop again. */
+void flushAppendOnlyFile(void) {
+    time_t now;
+    ssize_t nwritten;
+
+    if (sdslen(server.aofbuf) == 0) return;
+
+    /* We want to perform a single write. This should be guaranteed atomic
+     * at least if the filesystem we are writing is a real physical one.
+     * While this will save us against the server being killed I don't think
+     * there is much to do about the whole server stopping for power problems
+     * or alike */
+     nwritten = write(server.appendfd,server.aofbuf,sdslen(server.aofbuf));
+     if (nwritten != (signed)sdslen(server.aofbuf)) {
+        /* Ooops, we are in troubles. The best thing to do for now is
+         * aborting instead of giving the illusion that everything is
+         * working as expected. */
+         if (nwritten == -1) {
+            redisLog(REDIS_WARNING,"Exiting on error writing to the append-only file: %s",strerror(errno));
+         } else {
+            redisLog(REDIS_WARNING,"Exiting on short write while writing to the append-only file: %s",strerror(errno));
+         }
+         exit(1);
+    }
+    sdsfree(server.aofbuf);
+    server.aofbuf = sdsempty();
+
+    /* Don't Fsync if no-appendfsync-on-rewrite is set to yes and we have
+     * childs performing heavy I/O on disk. */
+    if (server.no_appendfsync_on_rewrite &&
+        (server.bgrewritechildpid != -1 || server.bgsavechildpid != -1))
+            return;
+    /* Fsync if needed */
+    now = time(NULL);
+    if (server.appendfsync == APPENDFSYNC_ALWAYS ||
+        (server.appendfsync == APPENDFSYNC_EVERYSEC &&
+         now-server.lastfsync > 1))
+    {
+        /* aof_fsync is defined as fdatasync() for Linux in order to avoid
+         * flushing metadata. */
+        aof_fsync(server.appendfd); /* Let's try to get this data on the disk */
+        server.lastfsync = now;
+    }
+}
+
+sds catAppendOnlyGenericCommand(sds buf, int argc, robj **argv) {
+    int j;
+    buf = sdscatprintf(buf,"*%d\r\n",argc);
+    for (j = 0; j < argc; j++) {
+        robj *o = getDecodedObject(argv[j]);
+        buf = sdscatprintf(buf,"$%lu\r\n",(unsigned long)sdslen(o->ptr));
+        buf = sdscatlen(buf,o->ptr,sdslen(o->ptr));
+        buf = sdscatlen(buf,"\r\n",2);
+        decrRefCount(o);
+    }
+    return buf;
+}
+
+sds catAppendOnlyExpireAtCommand(sds buf, robj *key, robj *seconds) {
+    int argc = 3;
+    long when;
+    robj *argv[3];
+
+    /* Make sure we can use strtol */
+    seconds = getDecodedObject(seconds);
+    when = time(NULL)+strtol(seconds->ptr,NULL,10);
+    decrRefCount(seconds);
+
+    argv[0] = createStringObject("EXPIREAT",8);
+    argv[1] = key;
+    argv[2] = createObject(REDIS_STRING,
+        sdscatprintf(sdsempty(),"%ld",when));
+    buf = catAppendOnlyGenericCommand(buf, argc, argv);
+    decrRefCount(argv[0]);
+    decrRefCount(argv[2]);
+    return buf;
+}
+
+void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int argc) {
+    sds buf = sdsempty();
+    robj *tmpargv[3];
+
+    /* The DB this command was targetting is not the same as the last command
+     * we appendend. To issue a SELECT command is needed. */
+    if (dictid != server.appendseldb) {
+        char seldb[64];
+
+        snprintf(seldb,sizeof(seldb),"%d",dictid);
+        buf = sdscatprintf(buf,"*2\r\n$6\r\nSELECT\r\n$%lu\r\n%s\r\n",
+            (unsigned long)strlen(seldb),seldb);
+        server.appendseldb = dictid;
+    }
+
+    if (cmd->proc == expireCommand) {
+        /* Translate EXPIRE into EXPIREAT */
+        buf = catAppendOnlyExpireAtCommand(buf,argv[1],argv[2]);
+    } else if (cmd->proc == setexCommand) {
+        /* Translate SETEX to SET and EXPIREAT */
+        tmpargv[0] = createStringObject("SET",3);
+        tmpargv[1] = argv[1];
+        tmpargv[2] = argv[3];
+        buf = catAppendOnlyGenericCommand(buf,3,tmpargv);
+        decrRefCount(tmpargv[0]);
+        buf = catAppendOnlyExpireAtCommand(buf,argv[1],argv[2]);
+    } else {
+        buf = catAppendOnlyGenericCommand(buf,argc,argv);
+    }
+
+    /* Append to the AOF buffer. This will be flushed on disk just before
+     * of re-entering the event loop, so before the client will get a
+     * positive reply about the operation performed. */
+    server.aofbuf = sdscatlen(server.aofbuf,buf,sdslen(buf));
+
+    /* If a background append only file rewriting is in progress we want to
+     * accumulate the differences between the child DB and the current one
+     * in a buffer, so that when the child process will do its work we
+     * can append the differences to the new append only file. */
+    if (server.bgrewritechildpid != -1)
+        server.bgrewritebuf = sdscatlen(server.bgrewritebuf,buf,sdslen(buf));
+
+    sdsfree(buf);
+}
+
+/* In Redis commands are always executed in the context of a client, so in
+ * order to load the append only file we need to create a fake client. */
+struct redisClient *createFakeClient(void) {
+    struct redisClient *c = zmalloc(sizeof(*c));
+
+    selectDb(c,0);
+    c->fd = -1;
+    c->querybuf = sdsempty();
+    c->argc = 0;
+    c->argv = NULL;
+    c->flags = 0;
+    /* We set the fake client as a slave waiting for the synchronization
+     * so that Redis will not try to send replies to this client. */
+    c->replstate = REDIS_REPL_WAIT_BGSAVE_START;
+    c->reply = listCreate();
+    c->watched_keys = listCreate();
+    listSetFreeMethod(c->reply,decrRefCount);
+    listSetDupMethod(c->reply,dupClientReplyValue);
+    initClientMultiState(c);
+    return c;
+}
+
+void freeFakeClient(struct redisClient *c) {
+    sdsfree(c->querybuf);
+    listRelease(c->reply);
+    listRelease(c->watched_keys);
+    freeClientMultiState(c);
+    zfree(c);
+}
+
+/* Replay the append log file. On error REDIS_OK is returned. On non fatal
+ * error (the append only file is zero-length) REDIS_ERR is returned. On
+ * fatal error an error message is logged and the program exists. */
+int loadAppendOnlyFile(char *filename) {
+    struct redisClient *fakeClient;
+    FILE *fp = fopen(filename,"r");
+    struct redis_stat sb;
+    int appendonly = server.appendonly;
+
+    if (redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0)
+        return REDIS_ERR;
+
+    if (fp == NULL) {
+        redisLog(REDIS_WARNING,"Fatal error: can't open the append log file for reading: %s",strerror(errno));
+        exit(1);
+    }
+
+    /* Temporarily disable AOF, to prevent EXEC from feeding a MULTI
+     * to the same file we're about to read. */
+    server.appendonly = 0;
+
+    fakeClient = createFakeClient();
+    while(1) {
+        int argc, j;
+        unsigned long len;
+        robj **argv;
+        char buf[128];
+        sds argsds;
+        struct redisCommand *cmd;
+        int force_swapout;
+
+        if (fgets(buf,sizeof(buf),fp) == NULL) {
+            if (feof(fp))
+                break;
+            else
+                goto readerr;
+        }
+        if (buf[0] != '*') goto fmterr;
+        argc = atoi(buf+1);
+        argv = zmalloc(sizeof(robj*)*argc);
+        for (j = 0; j < argc; j++) {
+            if (fgets(buf,sizeof(buf),fp) == NULL) goto readerr;
+            if (buf[0] != '$') goto fmterr;
+            len = strtol(buf+1,NULL,10);
+            argsds = sdsnewlen(NULL,len);
+            if (len && fread(argsds,len,1,fp) == 0) goto fmterr;
+            argv[j] = createObject(REDIS_STRING,argsds);
+            if (fread(buf,2,1,fp) == 0) goto fmterr; /* discard CRLF */
+        }
+
+        /* Command lookup */
+        cmd = lookupCommand(argv[0]->ptr);
+        if (!cmd) {
+            redisLog(REDIS_WARNING,"Unknown command '%s' reading the append only file", argv[0]->ptr);
+            exit(1);
+        }
+        /* Try object encoding */
+        if (cmd->flags & REDIS_CMD_BULK)
+            argv[argc-1] = tryObjectEncoding(argv[argc-1]);
+        /* Run the command in the context of a fake client */
+        fakeClient->argc = argc;
+        fakeClient->argv = argv;
+        cmd->proc(fakeClient);
+        /* Discard the reply objects list from the fake client */
+        while(listLength(fakeClient->reply))
+            listDelNode(fakeClient->reply,listFirst(fakeClient->reply));
+        /* Clean up, ready for the next command */
+        for (j = 0; j < argc; j++) decrRefCount(argv[j]);
+        zfree(argv);
+        /* Handle swapping while loading big datasets when VM is on */
+        force_swapout = 0;
+        if ((zmalloc_used_memory() - server.vm_max_memory) > 1024*1024*32)
+            force_swapout = 1;
+
+        if (server.vm_enabled && force_swapout) {
+            while (zmalloc_used_memory() > server.vm_max_memory) {
+                if (vmSwapOneObjectBlocking() == REDIS_ERR) break;
+            }
+        }
+    }
+
+    /* This point can only be reached when EOF is reached without errors.
+     * If the client is in the middle of a MULTI/EXEC, log error and quit. */
+    if (fakeClient->flags & REDIS_MULTI) goto readerr;
+
+    fclose(fp);
+    freeFakeClient(fakeClient);
+    server.appendonly = appendonly;
+    return REDIS_OK;
+
+readerr:
+    if (feof(fp)) {
+        redisLog(REDIS_WARNING,"Unexpected end of file reading the append only file");
+    } else {
+        redisLog(REDIS_WARNING,"Unrecoverable error reading the append only file: %s", strerror(errno));
+    }
+    exit(1);
+fmterr:
+    redisLog(REDIS_WARNING,"Bad file format reading the append only file");
+    exit(1);
+}
+
+/* Write binary-safe string into a file in the bulkformat
+ * $<count>\r\n<payload>\r\n */
+int fwriteBulkString(FILE *fp, char *s, unsigned long len) {
+    char cbuf[128];
+    int clen;
+    cbuf[0] = '$';
+    clen = 1+ll2string(cbuf+1,sizeof(cbuf)-1,len);
+    cbuf[clen++] = '\r';
+    cbuf[clen++] = '\n';
+    if (fwrite(cbuf,clen,1,fp) == 0) return 0;
+    if (len > 0 && fwrite(s,len,1,fp) == 0) return 0;
+    if (fwrite("\r\n",2,1,fp) == 0) return 0;
+    return 1;
+}
+
+/* Write a double value in bulk format $<count>\r\n<payload>\r\n */
+int fwriteBulkDouble(FILE *fp, double d) {
+    char buf[128], dbuf[128];
+
+    snprintf(dbuf,sizeof(dbuf),"%.17g\r\n",d);
+    snprintf(buf,sizeof(buf),"$%lu\r\n",(unsigned long)strlen(dbuf)-2);
+    if (fwrite(buf,strlen(buf),1,fp) == 0) return 0;
+    if (fwrite(dbuf,strlen(dbuf),1,fp) == 0) return 0;
+    return 1;
+}
+
+/* Write a long value in bulk format $<count>\r\n<payload>\r\n */
+int fwriteBulkLongLong(FILE *fp, long long l) {
+    char bbuf[128], lbuf[128];
+    unsigned int blen, llen;
+    llen = ll2string(lbuf,32,l);
+    blen = snprintf(bbuf,sizeof(bbuf),"$%u\r\n%s\r\n",llen,lbuf);
+    if (fwrite(bbuf,blen,1,fp) == 0) return 0;
+    return 1;
+}
+
+/* Delegate writing an object to writing a bulk string or bulk long long. */
+int fwriteBulkObject(FILE *fp, robj *obj) {
+    /* Avoid using getDecodedObject to help copy-on-write (we are often
+     * in a child process when this function is called). */
+    if (obj->encoding == REDIS_ENCODING_INT) {
+        return fwriteBulkLongLong(fp,(long)obj->ptr);
+    } else if (obj->encoding == REDIS_ENCODING_RAW) {
+        return fwriteBulkString(fp,obj->ptr,sdslen(obj->ptr));
+    } else {
+        redisPanic("Unknown string encoding");
+    }
+}
+
+/* Write a sequence of commands able to fully rebuild the dataset into
+ * "filename". Used both by REWRITEAOF and BGREWRITEAOF. */
+int rewriteAppendOnlyFile(char *filename) {
+    dictIterator *di = NULL;
+    dictEntry *de;
+    FILE *fp;
+    char tmpfile[256];
+    int j;
+    time_t now = time(NULL);
+
+    /* Note that we have to use a different temp name here compared to the
+     * one used by rewriteAppendOnlyFileBackground() function. */
+    snprintf(tmpfile,256,"temp-rewriteaof-%d.aof", (int) getpid());
+    fp = fopen(tmpfile,"w");
+    if (!fp) {
+        redisLog(REDIS_WARNING, "Failed rewriting the append only file: %s", strerror(errno));
+        return REDIS_ERR;
+    }
+    for (j = 0; j < server.dbnum; j++) {
+        char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n";
+        redisDb *db = server.db+j;
+        dict *d = db->dict;
+        if (dictSize(d) == 0) continue;
+        di = dictGetIterator(d);
+        if (!di) {
+            fclose(fp);
+            return REDIS_ERR;
+        }
+
+        /* SELECT the new DB */
+        if (fwrite(selectcmd,sizeof(selectcmd)-1,1,fp) == 0) goto werr;
+        if (fwriteBulkLongLong(fp,j) == 0) goto werr;
+
+        /* Iterate this DB writing every entry */
+        while((de = dictNext(di)) != NULL) {
+            sds keystr = dictGetEntryKey(de);
+            robj key, *o;
+            time_t expiretime;
+            int swapped;
+
+            keystr = dictGetEntryKey(de);
+            o = dictGetEntryVal(de);
+            initStaticStringObject(key,keystr);
+            /* If the value for this key is swapped, load a preview in memory.
+             * We use a "swapped" flag to remember if we need to free the
+             * value object instead to just increment the ref count anyway
+             * in order to avoid copy-on-write of pages if we are forked() */
+            if (!server.vm_enabled || o->storage == REDIS_VM_MEMORY ||
+                o->storage == REDIS_VM_SWAPPING) {
+                swapped = 0;
+            } else {
+                o = vmPreviewObject(o);
+                swapped = 1;
+            }
+            expiretime = getExpire(db,&key);
+
+            /* Save the key and associated value */
+            if (o->type == REDIS_STRING) {
+                /* Emit a SET command */
+                char cmd[]="*3\r\n$3\r\nSET\r\n";
+                if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                /* Key and value */
+                if (fwriteBulkObject(fp,&key) == 0) goto werr;
+                if (fwriteBulkObject(fp,o) == 0) goto werr;
+            } else if (o->type == REDIS_LIST) {
+                /* Emit the RPUSHes needed to rebuild the list */
+                char cmd[]="*3\r\n$5\r\nRPUSH\r\n";
+                if (o->encoding == REDIS_ENCODING_ZIPLIST) {
+                    unsigned char *zl = o->ptr;
+                    unsigned char *p = ziplistIndex(zl,0);
+                    unsigned char *vstr;
+                    unsigned int vlen;
+                    long long vlong;
+
+                    while(ziplistGet(p,&vstr,&vlen,&vlong)) {
+                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
+                        if (vstr) {
+                            if (fwriteBulkString(fp,(char*)vstr,vlen) == 0)
+                                goto werr;
+                        } else {
+                            if (fwriteBulkLongLong(fp,vlong) == 0)
+                                goto werr;
+                        }
+                        p = ziplistNext(zl,p);
+                    }
+                } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
+                    list *list = o->ptr;
+                    listNode *ln;
+                    listIter li;
+
+                    listRewind(list,&li);
+                    while((ln = listNext(&li))) {
+                        robj *eleobj = listNodeValue(ln);
+
+                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
+                        if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
+                    }
+                } else {
+                    redisPanic("Unknown list encoding");
+                }
+            } else if (o->type == REDIS_SET) {
+                char cmd[]="*3\r\n$4\r\nSADD\r\n";
+
+                /* Emit the SADDs needed to rebuild the set */
+                if (o->encoding == REDIS_ENCODING_INTSET) {
+                    int ii = 0;
+                    int64_t llval;
+                    while(intsetGet(o->ptr,ii++,&llval)) {
+                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
+                        if (fwriteBulkLongLong(fp,llval) == 0) goto werr;
+                    }
+                } else if (o->encoding == REDIS_ENCODING_HT) {
+                    dictIterator *di = dictGetIterator(o->ptr);
+                    dictEntry *de;
+                    while((de = dictNext(di)) != NULL) {
+                        robj *eleobj = dictGetEntryKey(de);
+                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
+                        if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
+                    }
+                    dictReleaseIterator(di);
+                } else {
+                    redisPanic("Unknown set encoding");
+                }
+            } else if (o->type == REDIS_ZSET) {
+                /* Emit the ZADDs needed to rebuild the sorted set */
+                zset *zs = o->ptr;
+                dictIterator *di = dictGetIterator(zs->dict);
+                dictEntry *de;
+
+                while((de = dictNext(di)) != NULL) {
+                    char cmd[]="*4\r\n$4\r\nZADD\r\n";
+                    robj *eleobj = dictGetEntryKey(de);
+                    double *score = dictGetEntryVal(de);
+
+                    if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                    if (fwriteBulkObject(fp,&key) == 0) goto werr;
+                    if (fwriteBulkDouble(fp,*score) == 0) goto werr;
+                    if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
+                }
+                dictReleaseIterator(di);
+            } else if (o->type == REDIS_HASH) {
+                char cmd[]="*4\r\n$4\r\nHSET\r\n";
+
+                /* Emit the HSETs needed to rebuild the hash */
+                if (o->encoding == REDIS_ENCODING_ZIPMAP) {
+                    unsigned char *p = zipmapRewind(o->ptr);
+                    unsigned char *field, *val;
+                    unsigned int flen, vlen;
+
+                    while((p = zipmapNext(p,&field,&flen,&val,&vlen)) != NULL) {
+                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
+                        if (fwriteBulkString(fp,(char*)field,flen) == 0)
+                            goto werr;
+                        if (fwriteBulkString(fp,(char*)val,vlen) == 0)
+                            goto werr;
+                    }
+                } else {
+                    dictIterator *di = dictGetIterator(o->ptr);
+                    dictEntry *de;
+
+                    while((de = dictNext(di)) != NULL) {
+                        robj *field = dictGetEntryKey(de);
+                        robj *val = dictGetEntryVal(de);
+
+                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
+                        if (fwriteBulkObject(fp,field) == 0) goto werr;
+                        if (fwriteBulkObject(fp,val) == 0) goto werr;
+                    }
+                    dictReleaseIterator(di);
+                }
+            } else {
+                redisPanic("Unknown object type");
+            }
+            /* Save the expire time */
+            if (expiretime != -1) {
+                char cmd[]="*3\r\n$8\r\nEXPIREAT\r\n";
+                /* If this key is already expired skip it */
+                if (expiretime < now) continue;
+                if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                if (fwriteBulkObject(fp,&key) == 0) goto werr;
+                if (fwriteBulkLongLong(fp,expiretime) == 0) goto werr;
+            }
+            if (swapped) decrRefCount(o);
+        }
+        dictReleaseIterator(di);
+    }
+
+    /* Make sure data will not remain on the OS's output buffers */
+    fflush(fp);
+    aof_fsync(fileno(fp));
+    fclose(fp);
+
+    /* Use RENAME to make sure the DB file is changed atomically only
+     * if the generate DB file is ok. */
+    if (rename(tmpfile,filename) == -1) {
+        redisLog(REDIS_WARNING,"Error moving temp append only file on the final destination: %s", strerror(errno));
+        unlink(tmpfile);
+        return REDIS_ERR;
+    }
+    redisLog(REDIS_NOTICE,"SYNC append only file rewrite performed");
+    return REDIS_OK;
+
+werr:
+    fclose(fp);
+    unlink(tmpfile);
+    redisLog(REDIS_WARNING,"Write error writing append only file on disk: %s", strerror(errno));
+    if (di) dictReleaseIterator(di);
+    return REDIS_ERR;
+}
+
+/* This is how rewriting of the append only file in background works:
+ *
+ * 1) The user calls BGREWRITEAOF
+ * 2) Redis calls this function, that forks():
+ *    2a) the child rewrite the append only file in a temp file.
+ *    2b) the parent accumulates differences in server.bgrewritebuf.
+ * 3) When the child finished '2a' exists.
+ * 4) The parent will trap the exit code, if it's OK, will append the
+ *    data accumulated into server.bgrewritebuf into the temp file, and
+ *    finally will rename(2) the temp file in the actual file name.
+ *    The the new file is reopened as the new append only file. Profit!
+ */
+int rewriteAppendOnlyFileBackground(void) {
+    pid_t childpid;
+
+    if (server.bgrewritechildpid != -1) return REDIS_ERR;
+    if (server.vm_enabled) waitEmptyIOJobsQueue();
+    if ((childpid = fork()) == 0) {
+        /* Child */
+        char tmpfile[256];
+
+        if (server.vm_enabled) vmReopenSwapFile();
+        close(server.fd);
+        snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
+        if (rewriteAppendOnlyFile(tmpfile) == REDIS_OK) {
+            _exit(0);
+        } else {
+            _exit(1);
+        }
+    } else {
+        /* Parent */
+        if (childpid == -1) {
+            redisLog(REDIS_WARNING,
+                "Can't rewrite append only file in background: fork: %s",
+                strerror(errno));
+            return REDIS_ERR;
+        }
+        redisLog(REDIS_NOTICE,
+            "Background append only file rewriting started by pid %d",childpid);
+        server.bgrewritechildpid = childpid;
+        updateDictResizePolicy();
+        /* We set appendseldb to -1 in order to force the next call to the
+         * feedAppendOnlyFile() to issue a SELECT command, so the differences
+         * accumulated by the parent into server.bgrewritebuf will start
+         * with a SELECT statement and it will be safe to merge. */
+        server.appendseldb = -1;
+        return REDIS_OK;
+    }
+    return REDIS_OK; /* unreached */
+}
+
+void bgrewriteaofCommand(redisClient *c) {
+    if (server.bgrewritechildpid != -1) {
+        addReplySds(c,sdsnew("-ERR background append only file rewriting already in progress\r\n"));
+        return;
+    }
+    if (rewriteAppendOnlyFileBackground() == REDIS_OK) {
+        char *status = "+Background append only file rewriting started\r\n";
+        addReplySds(c,sdsnew(status));
+    } else {
+        addReply(c,shared.err);
+    }
+}
+
+void aofRemoveTempFile(pid_t childpid) {
+    char tmpfile[256];
+
+    snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) childpid);
+    unlink(tmpfile);
+}
+
+/* A background append only file rewriting (BGREWRITEAOF) terminated its work.
+ * Handle this. */
+void backgroundRewriteDoneHandler(int statloc) {
+    int exitcode = WEXITSTATUS(statloc);
+    int bysignal = WIFSIGNALED(statloc);
+
+    if (!bysignal && exitcode == 0) {
+        int fd;
+        char tmpfile[256];
+
+        redisLog(REDIS_NOTICE,
+            "Background append only file rewriting terminated with success");
+        /* Now it's time to flush the differences accumulated by the parent */
+        snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) server.bgrewritechildpid);
+        fd = open(tmpfile,O_WRONLY|O_APPEND);
+        if (fd == -1) {
+            redisLog(REDIS_WARNING, "Not able to open the temp append only file produced by the child: %s", strerror(errno));
+            goto cleanup;
+        }
+        /* Flush our data... */
+        if (write(fd,server.bgrewritebuf,sdslen(server.bgrewritebuf)) !=
+                (signed) sdslen(server.bgrewritebuf)) {
+            redisLog(REDIS_WARNING, "Error or short write trying to flush the parent diff of the append log file in the child temp file: %s", strerror(errno));
+            close(fd);
+            goto cleanup;
+        }
+        redisLog(REDIS_NOTICE,"Parent diff flushed into the new append log file with success (%lu bytes)",sdslen(server.bgrewritebuf));
+        /* Now our work is to rename the temp file into the stable file. And
+         * switch the file descriptor used by the server for append only. */
+        if (rename(tmpfile,server.appendfilename) == -1) {
+            redisLog(REDIS_WARNING,"Can't rename the temp append only file into the stable one: %s", strerror(errno));
+            close(fd);
+            goto cleanup;
+        }
+        /* Mission completed... almost */
+        redisLog(REDIS_NOTICE,"Append only file successfully rewritten.");
+        if (server.appendfd != -1) {
+            /* If append only is actually enabled... */
+            close(server.appendfd);
+            server.appendfd = fd;
+            if (server.appendfsync != APPENDFSYNC_NO) aof_fsync(fd);
+            server.appendseldb = -1; /* Make sure it will issue SELECT */
+            redisLog(REDIS_NOTICE,"The new append only file was selected for future appends.");
+        } else {
+            /* If append only is disabled we just generate a dump in this
+             * format. Why not? */
+            close(fd);
+        }
+    } else if (!bysignal && exitcode != 0) {
+        redisLog(REDIS_WARNING, "Background append only file rewriting error");
+    } else {
+        redisLog(REDIS_WARNING,
+            "Background append only file rewriting terminated by signal %d",
+            WTERMSIG(statloc));
+    }
+cleanup:
+    sdsfree(server.bgrewritebuf);
+    server.bgrewritebuf = sdsempty();
+    aofRemoveTempFile(server.bgrewritechildpid);
+    server.bgrewritechildpid = -1;
+}

src/config.c

@@ -0,0 +1,440 @@
+#include "redis.h"
+
+/*-----------------------------------------------------------------------------
+ * Config file parsing
+ *----------------------------------------------------------------------------*/
+
+int yesnotoi(char *s) {
+    if (!strcasecmp(s,"yes")) return 1;
+    else if (!strcasecmp(s,"no")) return 0;
+    else return -1;
+}
+
+void appendServerSaveParams(time_t seconds, int changes) {
+    server.saveparams = zrealloc(server.saveparams,sizeof(struct saveparam)*(server.saveparamslen+1));
+    server.saveparams[server.saveparamslen].seconds = seconds;
+    server.saveparams[server.saveparamslen].changes = changes;
+    server.saveparamslen++;
+}
+
+void resetServerSaveParams() {
+    zfree(server.saveparams);
+    server.saveparams = NULL;
+    server.saveparamslen = 0;
+}
+
+/* I agree, this is a very rudimental way to load a configuration...
+   will improve later if the config gets more complex */
+void loadServerConfig(char *filename) {
+    FILE *fp;
+    char buf[REDIS_CONFIGLINE_MAX+1], *err = NULL;
+    int linenum = 0;
+    sds line = NULL;
+
+    if (filename[0] == '-' && filename[1] == '\0')
+        fp = stdin;
+    else {
+        if ((fp = fopen(filename,"r")) == NULL) {
+            redisLog(REDIS_WARNING, "Fatal error, can't open config file '%s'", filename);
+            exit(1);
+        }
+    }
+
+    while(fgets(buf,REDIS_CONFIGLINE_MAX+1,fp) != NULL) {
+        sds *argv;
+        int argc, j;
+
+        linenum++;
+        line = sdsnew(buf);
+        line = sdstrim(line," \t\r\n");
+
+        /* Skip comments and blank lines*/
+        if (line[0] == '#' || line[0] == '\0') {
+            sdsfree(line);
+            continue;
+        }
+
+        /* Split into arguments */
+        argv = sdssplitlen(line,sdslen(line)," ",1,&argc);
+        sdstolower(argv[0]);
+
+        /* Execute config directives */
+        if (!strcasecmp(argv[0],"timeout") && argc == 2) {
+            server.maxidletime = atoi(argv[1]);
+            if (server.maxidletime < 0) {
+                err = "Invalid timeout value"; goto loaderr;
+            }
+        } else if (!strcasecmp(argv[0],"port") && argc == 2) {
+            server.port = atoi(argv[1]);
+            if (server.port < 1 || server.port > 65535) {
+                err = "Invalid port"; goto loaderr;
+            }
+        } else if (!strcasecmp(argv[0],"bind") && argc == 2) {
+            server.bindaddr = zstrdup(argv[1]);
+        } else if (!strcasecmp(argv[0],"save") && argc == 3) {
+            int seconds = atoi(argv[1]);
+            int changes = atoi(argv[2]);
+            if (seconds < 1 || changes < 0) {
+                err = "Invalid save parameters"; goto loaderr;
+            }
+            appendServerSaveParams(seconds,changes);
+        } else if (!strcasecmp(argv[0],"dir") && argc == 2) {
+            if (chdir(argv[1]) == -1) {
+                redisLog(REDIS_WARNING,"Can't chdir to '%s': %s",
+                    argv[1], strerror(errno));
+                exit(1);
+            }
+        } else if (!strcasecmp(argv[0],"loglevel") && argc == 2) {
+            if (!strcasecmp(argv[1],"debug")) server.verbosity = REDIS_DEBUG;
+            else if (!strcasecmp(argv[1],"verbose")) server.verbosity = REDIS_VERBOSE;
+            else if (!strcasecmp(argv[1],"notice")) server.verbosity = REDIS_NOTICE;
+            else if (!strcasecmp(argv[1],"warning")) server.verbosity = REDIS_WARNING;
+            else {
+                err = "Invalid log level. Must be one of debug, notice, warning";
+                goto loaderr;
+            }
+        } else if (!strcasecmp(argv[0],"logfile") && argc == 2) {
+            FILE *logfp;
+
+            server.logfile = zstrdup(argv[1]);
+            if (!strcasecmp(server.logfile,"stdout")) {
+                zfree(server.logfile);
+                server.logfile = NULL;
+            }
+            if (server.logfile) {
+                /* Test if we are able to open the file. The server will not
+                 * be able to abort just for this problem later... */
+                logfp = fopen(server.logfile,"a");
+                if (logfp == NULL) {
+                    err = sdscatprintf(sdsempty(),
+                        "Can't open the log file: %s", strerror(errno));
+                    goto loaderr;
+                }
+                fclose(logfp);
+            }
+        } else if (!strcasecmp(argv[0],"databases") && argc == 2) {
+            server.dbnum = atoi(argv[1]);
+            if (server.dbnum < 1) {
+                err = "Invalid number of databases"; goto loaderr;
+            }
+        } else if (!strcasecmp(argv[0],"include") && argc == 2) {
+            loadServerConfig(argv[1]);
+        } else if (!strcasecmp(argv[0],"maxclients") && argc == 2) {
+            server.maxclients = atoi(argv[1]);
+        } else if (!strcasecmp(argv[0],"maxmemory") && argc == 2) {
+            server.maxmemory = memtoll(argv[1],NULL);
+        } else if (!strcasecmp(argv[0],"slaveof") && argc == 3) {
+            server.masterhost = sdsnew(argv[1]);
+            server.masterport = atoi(argv[2]);
+            server.replstate = REDIS_REPL_CONNECT;
+        } else if (!strcasecmp(argv[0],"masterauth") && argc == 2) {
+        	server.masterauth = zstrdup(argv[1]);
+        } else if (!strcasecmp(argv[0],"glueoutputbuf") && argc == 2) {
+            if ((server.glueoutputbuf = yesnotoi(argv[1])) == -1) {
+                err = "argument must be 'yes' or 'no'"; goto loaderr;
+            }
+        } else if (!strcasecmp(argv[0],"rdbcompression") && argc == 2) {
+            if ((server.rdbcompression = yesnotoi(argv[1])) == -1) {
+                err = "argument must be 'yes' or 'no'"; goto loaderr;
+            }
+        } else if (!strcasecmp(argv[0],"activerehashing") && argc == 2) {
+            if ((server.activerehashing = yesnotoi(argv[1])) == -1) {
+                err = "argument must be 'yes' or 'no'"; goto loaderr;
+            }
+        } else if (!strcasecmp(argv[0],"daemonize") && argc == 2) {
+            if ((server.daemonize = yesnotoi(argv[1])) == -1) {
+                err = "argument must be 'yes' or 'no'"; goto loaderr;
+            }
+        } else if (!strcasecmp(argv[0],"appendonly") && argc == 2) {
+            if ((server.appendonly = yesnotoi(argv[1])) == -1) {
+                err = "argument must be 'yes' or 'no'"; goto loaderr;
+            }
+        } else if (!strcasecmp(argv[0],"appendfilename") && argc == 2) {
+            zfree(server.appendfilename);
+            server.appendfilename = zstrdup(argv[1]);
+        } else if (!strcasecmp(argv[0],"no-appendfsync-on-rewrite")
+                   && argc == 2) {
+            if ((server.no_appendfsync_on_rewrite= yesnotoi(argv[1])) == -1) {
+                err = "argument must be 'yes' or 'no'"; goto loaderr;
+            }
+        } else if (!strcasecmp(argv[0],"appendfsync") && argc == 2) {
+            if (!strcasecmp(argv[1],"no")) {
+                server.appendfsync = APPENDFSYNC_NO;
+            } else if (!strcasecmp(argv[1],"always")) {
+                server.appendfsync = APPENDFSYNC_ALWAYS;
+            } else if (!strcasecmp(argv[1],"everysec")) {
+                server.appendfsync = APPENDFSYNC_EVERYSEC;
+            } else {
+                err = "argument must be 'no', 'always' or 'everysec'";
+                goto loaderr;
+            }
+        } else if (!strcasecmp(argv[0],"requirepass") && argc == 2) {
+            server.requirepass = zstrdup(argv[1]);
+        } else if (!strcasecmp(argv[0],"pidfile") && argc == 2) {
+            zfree(server.pidfile);
+            server.pidfile = zstrdup(argv[1]);
+        } else if (!strcasecmp(argv[0],"dbfilename") && argc == 2) {
+            zfree(server.dbfilename);
+            server.dbfilename = zstrdup(argv[1]);
+        } else if (!strcasecmp(argv[0],"vm-enabled") && argc == 2) {
+            if ((server.vm_enabled = yesnotoi(argv[1])) == -1) {
+                err = "argument must be 'yes' or 'no'"; goto loaderr;
+            }
+        } else if (!strcasecmp(argv[0],"vm-swap-file") && argc == 2) {
+            zfree(server.vm_swap_file);
+            server.vm_swap_file = zstrdup(argv[1]);
+        } else if (!strcasecmp(argv[0],"vm-max-memory") && argc == 2) {
+            server.vm_max_memory = memtoll(argv[1],NULL);
+        } else if (!strcasecmp(argv[0],"vm-page-size") && argc == 2) {
+            server.vm_page_size = memtoll(argv[1], NULL);
+        } else if (!strcasecmp(argv[0],"vm-pages") && argc == 2) {
+            server.vm_pages = memtoll(argv[1], NULL);
+        } else if (!strcasecmp(argv[0],"vm-max-threads") && argc == 2) {
+            server.vm_max_threads = strtoll(argv[1], NULL, 10);
+        } else if (!strcasecmp(argv[0],"hash-max-zipmap-entries") && argc == 2){
+            server.hash_max_zipmap_entries = memtoll(argv[1], NULL);
+        } else if (!strcasecmp(argv[0],"hash-max-zipmap-value") && argc == 2){
+            server.hash_max_zipmap_value = memtoll(argv[1], NULL);
+        } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){
+            server.list_max_ziplist_entries = memtoll(argv[1], NULL);
+        } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2){
+            server.list_max_ziplist_value = memtoll(argv[1], NULL);
+        } else if (!strcasecmp(argv[0],"set-max-intset-entries") && argc == 2){
+            server.set_max_intset_entries = memtoll(argv[1], NULL);
+        } else {
+            err = "Bad directive or wrong number of arguments"; goto loaderr;
+        }
+        for (j = 0; j < argc; j++)
+            sdsfree(argv[j]);
+        zfree(argv);
+        sdsfree(line);
+    }
+    if (fp != stdin) fclose(fp);
+    return;
+
+loaderr:
+    fprintf(stderr, "\n*** FATAL CONFIG FILE ERROR ***\n");
+    fprintf(stderr, "Reading the configuration file, at line %d\n", linenum);
+    fprintf(stderr, ">>> '%s'\n", line);
+    fprintf(stderr, "%s\n", err);
+    exit(1);
+}
+
+/*-----------------------------------------------------------------------------
+ * CONFIG command for remote configuration
+ *----------------------------------------------------------------------------*/
+
+void configSetCommand(redisClient *c) {
+    robj *o = getDecodedObject(c->argv[3]);
+    long long ll;
+
+    if (!strcasecmp(c->argv[2]->ptr,"dbfilename")) {
+        zfree(server.dbfilename);
+        server.dbfilename = zstrdup(o->ptr);
+    } else if (!strcasecmp(c->argv[2]->ptr,"requirepass")) {
+        zfree(server.requirepass);
+        server.requirepass = zstrdup(o->ptr);
+    } else if (!strcasecmp(c->argv[2]->ptr,"masterauth")) {
+        zfree(server.masterauth);
+        server.masterauth = zstrdup(o->ptr);
+    } else if (!strcasecmp(c->argv[2]->ptr,"maxmemory")) {
+        if (getLongLongFromObject(o,&ll) == REDIS_ERR ||
+            ll < 0) goto badfmt;
+        server.maxmemory = ll;
+    } else if (!strcasecmp(c->argv[2]->ptr,"timeout")) {
+        if (getLongLongFromObject(o,&ll) == REDIS_ERR ||
+            ll < 0 || ll > LONG_MAX) goto badfmt;
+        server.maxidletime = ll;
+    } else if (!strcasecmp(c->argv[2]->ptr,"appendfsync")) {
+        if (!strcasecmp(o->ptr,"no")) {
+            server.appendfsync = APPENDFSYNC_NO;
+        } else if (!strcasecmp(o->ptr,"everysec")) {
+            server.appendfsync = APPENDFSYNC_EVERYSEC;
+        } else if (!strcasecmp(o->ptr,"always")) {
+            server.appendfsync = APPENDFSYNC_ALWAYS;
+        } else {
+            goto badfmt;
+        }
+    } else if (!strcasecmp(c->argv[2]->ptr,"no-appendfsync-on-rewrite")) {
+        int yn = yesnotoi(o->ptr);
+
+        if (yn == -1) goto badfmt;
+        server.no_appendfsync_on_rewrite = yn;
+    } else if (!strcasecmp(c->argv[2]->ptr,"appendonly")) {
+        int old = server.appendonly;
+        int new = yesnotoi(o->ptr);
+
+        if (new == -1) goto badfmt;
+        if (old != new) {
+            if (new == 0) {
+                stopAppendOnly();
+            } else {
+                if (startAppendOnly() == REDIS_ERR) {
+                    addReplySds(c,sdscatprintf(sdsempty(),
+                        "-ERR Unable to turn on AOF. Check server logs.\r\n"));
+                    decrRefCount(o);
+                    return;
+                }
+            }
+        }
+    } else if (!strcasecmp(c->argv[2]->ptr,"save")) {
+        int vlen, j;
+        sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen);
+
+        /* Perform sanity check before setting the new config:
+         * - Even number of args
+         * - Seconds >= 1, changes >= 0 */
+        if (vlen & 1) {
+            sdsfreesplitres(v,vlen);
+            goto badfmt;
+        }
+        for (j = 0; j < vlen; j++) {
+            char *eptr;
+            long val;
+
+            val = strtoll(v[j], &eptr, 10);
+            if (eptr[0] != '\0' ||
+                ((j & 1) == 0 && val < 1) ||
+                ((j & 1) == 1 && val < 0)) {
+                sdsfreesplitres(v,vlen);
+                goto badfmt;
+            }
+        }
+        /* Finally set the new config */
+        resetServerSaveParams();
+        for (j = 0; j < vlen; j += 2) {
+            time_t seconds;
+            int changes;
+
+            seconds = strtoll(v[j],NULL,10);
+            changes = strtoll(v[j+1],NULL,10);
+            appendServerSaveParams(seconds, changes);
+        }
+        sdsfreesplitres(v,vlen);
+    } else {
+        addReplySds(c,sdscatprintf(sdsempty(),
+            "-ERR not supported CONFIG parameter %s\r\n",
+            (char*)c->argv[2]->ptr));
+        decrRefCount(o);
+        return;
+    }
+    decrRefCount(o);
+    addReply(c,shared.ok);
+    return;
+
+badfmt: /* Bad format errors */
+    addReplySds(c,sdscatprintf(sdsempty(),
+        "-ERR invalid argument '%s' for CONFIG SET '%s'\r\n",
+            (char*)o->ptr,
+            (char*)c->argv[2]->ptr));
+    decrRefCount(o);
+}
+
+void configGetCommand(redisClient *c) {
+    robj *o = getDecodedObject(c->argv[2]);
+    robj *lenobj = createObject(REDIS_STRING,NULL);
+    char *pattern = o->ptr;
+    int matches = 0;
+
+    addReply(c,lenobj);
+    decrRefCount(lenobj);
+
+    if (stringmatch(pattern,"dbfilename",0)) {
+        addReplyBulkCString(c,"dbfilename");
+        addReplyBulkCString(c,server.dbfilename);
+        matches++;
+    }
+    if (stringmatch(pattern,"requirepass",0)) {
+        addReplyBulkCString(c,"requirepass");
+        addReplyBulkCString(c,server.requirepass);
+        matches++;
+    }
+    if (stringmatch(pattern,"masterauth",0)) {
+        addReplyBulkCString(c,"masterauth");
+        addReplyBulkCString(c,server.masterauth);
+        matches++;
+    }
+    if (stringmatch(pattern,"maxmemory",0)) {
+        char buf[128];
+
+        ll2string(buf,128,server.maxmemory);
+        addReplyBulkCString(c,"maxmemory");
+        addReplyBulkCString(c,buf);
+        matches++;
+    }
+    if (stringmatch(pattern,"timeout",0)) {
+        char buf[128];
+
+        ll2string(buf,128,server.maxidletime);
+        addReplyBulkCString(c,"timeout");
+        addReplyBulkCString(c,buf);
+        matches++;
+    }
+    if (stringmatch(pattern,"appendonly",0)) {
+        addReplyBulkCString(c,"appendonly");
+        addReplyBulkCString(c,server.appendonly ? "yes" : "no");
+        matches++;
+    }
+    if (stringmatch(pattern,"no-appendfsync-on-rewrite",0)) {
+        addReplyBulkCString(c,"no-appendfsync-on-rewrite");
+        addReplyBulkCString(c,server.no_appendfsync_on_rewrite ? "yes" : "no");
+        matches++;
+    }
+    if (stringmatch(pattern,"appendfsync",0)) {
+        char *policy;
+
+        switch(server.appendfsync) {
+        case APPENDFSYNC_NO: policy = "no"; break;
+        case APPENDFSYNC_EVERYSEC: policy = "everysec"; break;
+        case APPENDFSYNC_ALWAYS: policy = "always"; break;
+        default: policy = "unknown"; break; /* too harmless to panic */
+        }
+        addReplyBulkCString(c,"appendfsync");
+        addReplyBulkCString(c,policy);
+        matches++;
+    }
+    if (stringmatch(pattern,"save",0)) {
+        sds buf = sdsempty();
+        int j;
+
+        for (j = 0; j < server.saveparamslen; j++) {
+            buf = sdscatprintf(buf,"%ld %d",
+                    server.saveparams[j].seconds,
+                    server.saveparams[j].changes);
+            if (j != server.saveparamslen-1)
+                buf = sdscatlen(buf," ",1);
+        }
+        addReplyBulkCString(c,"save");
+        addReplyBulkCString(c,buf);
+        sdsfree(buf);
+        matches++;
+    }
+    decrRefCount(o);
+    lenobj->ptr = sdscatprintf(sdsempty(),"*%d\r\n",matches*2);
+}
+
+void configCommand(redisClient *c) {
+    if (!strcasecmp(c->argv[1]->ptr,"set")) {
+        if (c->argc != 4) goto badarity;
+        configSetCommand(c);
+    } else if (!strcasecmp(c->argv[1]->ptr,"get")) {
+        if (c->argc != 3) goto badarity;
+        configGetCommand(c);
+    } else if (!strcasecmp(c->argv[1]->ptr,"resetstat")) {
+        if (c->argc != 2) goto badarity;
+        server.stat_numcommands = 0;
+        server.stat_numconnections = 0;
+        server.stat_expiredkeys = 0;
+        server.stat_starttime = time(NULL);
+        addReply(c,shared.ok);
+    } else {
+        addReplySds(c,sdscatprintf(sdsempty(),
+            "-ERR CONFIG subcommand must be one of GET, SET, RESETSTAT\r\n"));
+    }
+    return;
+
+badarity:
+    addReplySds(c,sdscatprintf(sdsempty(),
+        "-ERR Wrong number of arguments for CONFIG %s\r\n",
+        (char*) c->argv[1]->ptr));
+}

src/config.h

@@ -21,6 +21,16 @@
 #define redis_stat stat
 #endif
 
+/* test for proc filesystem */
+#ifdef __linux__
+#define HAVE_PROCFS 1
+#endif
+
+/* test for task_info() */
+#if defined(__APPLE__)
+#define HAVE_TASKINFO 1
+#endif
+
 /* test for backtrace() */
 #if defined(__APPLE__) || defined(__linux__)
 #define HAVE_BACKTRACE 1
@@ -35,4 +45,11 @@
 #define HAVE_KQUEUE 1
 #endif
 
+/* define aof_fsync to fdatasync() in Linux and fsync() for all the rest */
+#ifdef __linux__
+#define aof_fsync fdatasync
+#else
+#define aof_fsync fsync
+#endif
+
 #endif

src/db.c

@@ -0,0 +1,546 @@
+#include "redis.h"
+
+#include <signal.h>
+
+/*-----------------------------------------------------------------------------
+ * C-level DB API
+ *----------------------------------------------------------------------------*/
+
+robj *lookupKey(redisDb *db, robj *key) {
+    dictEntry *de = dictFind(db->dict,key->ptr);
+    if (de) {
+        robj *val = dictGetEntryVal(de);
+
+        if (server.vm_enabled) {
+            if (val->storage == REDIS_VM_MEMORY ||
+                val->storage == REDIS_VM_SWAPPING)
+            {
+                /* If we were swapping the object out, cancel the operation */
+                if (val->storage == REDIS_VM_SWAPPING)
+                    vmCancelThreadedIOJob(val);
+                /* Update the access time for the aging algorithm. */
+                val->lru = server.lruclock;
+            } else {
+                int notify = (val->storage == REDIS_VM_LOADING);
+
+                /* Our value was swapped on disk. Bring it at home. */
+                redisAssert(val->type == REDIS_VMPOINTER);
+                val = vmLoadObject(val);
+                dictGetEntryVal(de) = val;
+
+                /* Clients blocked by the VM subsystem may be waiting for
+                 * this key... */
+                if (notify) handleClientsBlockedOnSwappedKey(db,key);
+            }
+        }
+        return val;
+    } else {
+        return NULL;
+    }
+}
+
+robj *lookupKeyRead(redisDb *db, robj *key) {
+    expireIfNeeded(db,key);
+    return lookupKey(db,key);
+}
+
+robj *lookupKeyWrite(redisDb *db, robj *key) {
+    expireIfNeeded(db,key);
+    return lookupKey(db,key);
+}
+
+robj *lookupKeyReadOrReply(redisClient *c, robj *key, robj *reply) {
+    robj *o = lookupKeyRead(c->db, key);
+    if (!o) addReply(c,reply);
+    return o;
+}
+
+robj *lookupKeyWriteOrReply(redisClient *c, robj *key, robj *reply) {
+    robj *o = lookupKeyWrite(c->db, key);
+    if (!o) addReply(c,reply);
+    return o;
+}
+
+/* Add the key to the DB. If the key already exists REDIS_ERR is returned,
+ * otherwise REDIS_OK is returned, and the caller should increment the
+ * refcount of 'val'. */
+int dbAdd(redisDb *db, robj *key, robj *val) {
+    /* Perform a lookup before adding the key, as we need to copy the
+     * key value. */
+    if (dictFind(db->dict, key->ptr) != NULL) {
+        return REDIS_ERR;
+    } else {
+        sds copy = sdsdup(key->ptr);
+        dictAdd(db->dict, copy, val);
+        return REDIS_OK;
+    }
+}
+
+/* If the key does not exist, this is just like dbAdd(). Otherwise
+ * the value associated to the key is replaced with the new one.
+ *
+ * On update (key already existed) 0 is returned. Otherwise 1. */
+int dbReplace(redisDb *db, robj *key, robj *val) {
+    if (dictFind(db->dict,key->ptr) == NULL) {
+        sds copy = sdsdup(key->ptr);
+        dictAdd(db->dict, copy, val);
+        return 1;
+    } else {
+        dictReplace(db->dict, key->ptr, val);
+        return 0;
+    }
+}
+
+int dbExists(redisDb *db, robj *key) {
+    return dictFind(db->dict,key->ptr) != NULL;
+}
+
+/* Return a random key, in form of a Redis object.
+ * If there are no keys, NULL is returned.
+ *
+ * The function makes sure to return keys not already expired. */
+robj *dbRandomKey(redisDb *db) {
+    struct dictEntry *de;
+
+    while(1) {
+        sds key;
+        robj *keyobj;
+
+        de = dictGetRandomKey(db->dict);
+        if (de == NULL) return NULL;
+
+        key = dictGetEntryKey(de);
+        keyobj = createStringObject(key,sdslen(key));
+        if (dictFind(db->expires,key)) {
+            if (expireIfNeeded(db,keyobj)) {
+                decrRefCount(keyobj);
+                continue; /* search for another key. This expired. */
+            }
+        }
+        return keyobj;
+    }
+}
+
+/* Delete a key, value, and associated expiration entry if any, from the DB */
+int dbDelete(redisDb *db, robj *key) {
+    /* If VM is enabled make sure to awake waiting clients for this key:
+     * deleting the key will kill the I/O thread bringing the key from swap
+     * to memory, so the client will never be notified and unblocked if we
+     * don't do it now. */
+    if (server.vm_enabled) handleClientsBlockedOnSwappedKey(db,key);
+    /* Deleting an entry from the expires dict will not free the sds of
+     * the key, because it is shared with the main dictionary. */
+    if (dictSize(db->expires) > 0) dictDelete(db->expires,key->ptr);
+    return dictDelete(db->dict,key->ptr) == DICT_OK;
+}
+
+/* Empty the whole database */
+long long emptyDb() {
+    int j;
+    long long removed = 0;
+
+    for (j = 0; j < server.dbnum; j++) {
+        removed += dictSize(server.db[j].dict);
+        dictEmpty(server.db[j].dict);
+        dictEmpty(server.db[j].expires);
+    }
+    return removed;
+}
+
+int selectDb(redisClient *c, int id) {
+    if (id < 0 || id >= server.dbnum)
+        return REDIS_ERR;
+    c->db = &server.db[id];
+    return REDIS_OK;
+}
+
+/*-----------------------------------------------------------------------------
+ * Type agnostic commands operating on the key space
+ *----------------------------------------------------------------------------*/
+
+void flushdbCommand(redisClient *c) {
+    server.dirty += dictSize(c->db->dict);
+    touchWatchedKeysOnFlush(c->db->id);
+    dictEmpty(c->db->dict);
+    dictEmpty(c->db->expires);
+    addReply(c,shared.ok);
+}
+
+void flushallCommand(redisClient *c) {
+    touchWatchedKeysOnFlush(-1);
+    server.dirty += emptyDb();
+    addReply(c,shared.ok);
+    if (server.bgsavechildpid != -1) {
+        kill(server.bgsavechildpid,SIGKILL);
+        rdbRemoveTempFile(server.bgsavechildpid);
+    }
+    rdbSave(server.dbfilename);
+    server.dirty++;
+}
+
+void delCommand(redisClient *c) {
+    int deleted = 0, j;
+
+    for (j = 1; j < c->argc; j++) {
+        if (dbDelete(c->db,c->argv[j])) {
+            touchWatchedKey(c->db,c->argv[j]);
+            server.dirty++;
+            deleted++;
+        }
+    }
+    addReplyLongLong(c,deleted);
+}
+
+void existsCommand(redisClient *c) {
+    expireIfNeeded(c->db,c->argv[1]);
+    if (dbExists(c->db,c->argv[1])) {
+        addReply(c, shared.cone);
+    } else {
+        addReply(c, shared.czero);
+    }
+}
+
+void selectCommand(redisClient *c) {
+    int id = atoi(c->argv[1]->ptr);
+
+    if (selectDb(c,id) == REDIS_ERR) {
+        addReplySds(c,sdsnew("-ERR invalid DB index\r\n"));
+    } else {
+        addReply(c,shared.ok);
+    }
+}
+
+void randomkeyCommand(redisClient *c) {
+    robj *key;
+
+    if ((key = dbRandomKey(c->db)) == NULL) {
+        addReply(c,shared.nullbulk);
+        return;
+    }
+
+    addReplyBulk(c,key);
+    decrRefCount(key);
+}
+
+void keysCommand(redisClient *c) {
+    dictIterator *di;
+    dictEntry *de;
+    sds pattern = c->argv[1]->ptr;
+    int plen = sdslen(pattern), allkeys;
+    unsigned long numkeys = 0;
+    robj *lenobj = createObject(REDIS_STRING,NULL);
+
+    di = dictGetIterator(c->db->dict);
+    addReply(c,lenobj);
+    decrRefCount(lenobj);
+    allkeys = (pattern[0] == '*' && pattern[1] == '\0');
+    while((de = dictNext(di)) != NULL) {
+        sds key = dictGetEntryKey(de);
+        robj *keyobj;
+
+        if (allkeys || stringmatchlen(pattern,plen,key,sdslen(key),0)) {
+            keyobj = createStringObject(key,sdslen(key));
+            if (expireIfNeeded(c->db,keyobj) == 0) {
+                addReplyBulk(c,keyobj);
+                numkeys++;
+            }
+            decrRefCount(keyobj);
+        }
+    }
+    dictReleaseIterator(di);
+    lenobj->ptr = sdscatprintf(sdsempty(),"*%lu\r\n",numkeys);
+}
+
+void dbsizeCommand(redisClient *c) {
+    addReplySds(c,
+        sdscatprintf(sdsempty(),":%lu\r\n",dictSize(c->db->dict)));
+}
+
+void lastsaveCommand(redisClient *c) {
+    addReplySds(c,
+        sdscatprintf(sdsempty(),":%lu\r\n",server.lastsave));
+}
+
+void typeCommand(redisClient *c) {
+    robj *o;
+    char *type;
+
+    o = lookupKeyRead(c->db,c->argv[1]);
+    if (o == NULL) {
+        type = "+none";
+    } else {
+        switch(o->type) {
+        case REDIS_STRING: type = "+string"; break;
+        case REDIS_LIST: type = "+list"; break;
+        case REDIS_SET: type = "+set"; break;
+        case REDIS_ZSET: type = "+zset"; break;
+        case REDIS_HASH: type = "+hash"; break;
+        default: type = "+unknown"; break;
+        }
+    }
+    addReplySds(c,sdsnew(type));
+    addReply(c,shared.crlf);
+}
+
+void saveCommand(redisClient *c) {
+    if (server.bgsavechildpid != -1) {
+        addReplySds(c,sdsnew("-ERR background save in progress\r\n"));
+        return;
+    }
+    if (rdbSave(server.dbfilename) == REDIS_OK) {
+        addReply(c,shared.ok);
+    } else {
+        addReply(c,shared.err);
+    }
+}
+
+void bgsaveCommand(redisClient *c) {
+    if (server.bgsavechildpid != -1) {
+        addReplySds(c,sdsnew("-ERR background save already in progress\r\n"));
+        return;
+    }
+    if (rdbSaveBackground(server.dbfilename) == REDIS_OK) {
+        char *status = "+Background saving started\r\n";
+        addReplySds(c,sdsnew(status));
+    } else {
+        addReply(c,shared.err);
+    }
+}
+
+void shutdownCommand(redisClient *c) {
+    if (prepareForShutdown() == REDIS_OK)
+        exit(0);
+    addReplySds(c, sdsnew("-ERR Errors trying to SHUTDOWN. Check logs.\r\n"));
+}
+
+void renameGenericCommand(redisClient *c, int nx) {
+    robj *o;
+
+    /* To use the same key as src and dst is probably an error */
+    if (sdscmp(c->argv[1]->ptr,c->argv[2]->ptr) == 0) {
+        addReply(c,shared.sameobjecterr);
+        return;
+    }
+
+    if ((o = lookupKeyWriteOrReply(c,c->argv[1],shared.nokeyerr)) == NULL)
+        return;
+
+    incrRefCount(o);
+    if (dbAdd(c->db,c->argv[2],o) == REDIS_ERR) {
+        if (nx) {
+            decrRefCount(o);
+            addReply(c,shared.czero);
+            return;
+        }
+        dbReplace(c->db,c->argv[2],o);
+    }
+    dbDelete(c->db,c->argv[1]);
+    touchWatchedKey(c->db,c->argv[1]);
+    touchWatchedKey(c->db,c->argv[2]);
+    server.dirty++;
+    addReply(c,nx ? shared.cone : shared.ok);
+}
+
+void renameCommand(redisClient *c) {
+    renameGenericCommand(c,0);
+}
+
+void renamenxCommand(redisClient *c) {
+    renameGenericCommand(c,1);
+}
+
+void moveCommand(redisClient *c) {
+    robj *o;
+    redisDb *src, *dst;
+    int srcid;
+
+    /* Obtain source and target DB pointers */
+    src = c->db;
+    srcid = c->db->id;
+    if (selectDb(c,atoi(c->argv[2]->ptr)) == REDIS_ERR) {
+        addReply(c,shared.outofrangeerr);
+        return;
+    }
+    dst = c->db;
+    selectDb(c,srcid); /* Back to the source DB */
+
+    /* If the user is moving using as target the same
+     * DB as the source DB it is probably an error. */
+    if (src == dst) {
+        addReply(c,shared.sameobjecterr);
+        return;
+    }
+
+    /* Check if the element exists and get a reference */
+    o = lookupKeyWrite(c->db,c->argv[1]);
+    if (!o) {
+        addReply(c,shared.czero);
+        return;
+    }
+
+    /* Try to add the element to the target DB */
+    if (dbAdd(dst,c->argv[1],o) == REDIS_ERR) {
+        addReply(c,shared.czero);
+        return;
+    }
+    incrRefCount(o);
+
+    /* OK! key moved, free the entry in the source DB */
+    dbDelete(src,c->argv[1]);
+    server.dirty++;
+    addReply(c,shared.cone);
+}
+
+/*-----------------------------------------------------------------------------
+ * Expires API
+ *----------------------------------------------------------------------------*/
+
+int removeExpire(redisDb *db, robj *key) {
+    /* An expire may only be removed if there is a corresponding entry in the
+     * main dict. Otherwise, the key will never be freed. */
+    redisAssert(dictFind(db->dict,key->ptr) != NULL);
+    return dictDelete(db->expires,key->ptr) == DICT_OK;
+}
+
+void setExpire(redisDb *db, robj *key, time_t when) {
+    dictEntry *de;
+
+    /* Reuse the sds from the main dict in the expire dict */
+    de = dictFind(db->dict,key->ptr);
+    redisAssert(de != NULL);
+    dictReplace(db->expires,dictGetEntryKey(de),(void*)when);
+}
+
+/* Return the expire time of the specified key, or -1 if no expire
+ * is associated with this key (i.e. the key is non volatile) */
+time_t getExpire(redisDb *db, robj *key) {
+    dictEntry *de;
+
+    /* No expire? return ASAP */
+    if (dictSize(db->expires) == 0 ||
+       (de = dictFind(db->expires,key->ptr)) == NULL) return -1;
+
+    /* The entry was found in the expire dict, this means it should also
+     * be present in the main dict (safety check). */
+    redisAssert(dictFind(db->dict,key->ptr) != NULL);
+    return (time_t) dictGetEntryVal(de);
+}
+
+/* Propagate expires into slaves and the AOF file.
+ * When a key expires in the master, a DEL operation for this key is sent
+ * to all the slaves and the AOF file if enabled.
+ *
+ * This way the key expiry is centralized in one place, and since both
+ * AOF and the master->slave link guarantee operation ordering, everything
+ * will be consistent even if we allow write operations against expiring
+ * keys. */
+void propagateExpire(redisDb *db, robj *key) {
+    struct redisCommand *cmd;
+    robj *argv[2];
+
+    cmd = lookupCommand("del");
+    argv[0] = createStringObject("DEL",3);
+    argv[1] = key;
+    incrRefCount(key);
+
+    if (server.appendonly)
+        feedAppendOnlyFile(cmd,db->id,argv,2);
+    if (listLength(server.slaves))
+        replicationFeedSlaves(server.slaves,db->id,argv,2);
+
+    decrRefCount(argv[0]);
+    decrRefCount(argv[1]);
+}
+
+int expireIfNeeded(redisDb *db, robj *key) {
+    time_t when = getExpire(db,key);
+
+    /* If we are running in the context of a slave, return ASAP:
+     * the slave key expiration is controlled by the master that will
+     * send us synthesized DEL operations for expired keys.
+     *
+     * Still we try to return the right information to the caller, 
+     * that is, 0 if we think the key should be still valid, 1 if
+     * we think the key is expired at this time. */
+    if (server.masterhost != NULL) {
+        return time(NULL) > when;
+    }
+
+    if (when < 0) return 0;
+
+    /* Return when this key has not expired */
+    if (time(NULL) <= when) return 0;
+
+    /* Delete the key */
+    server.stat_expiredkeys++;
+    server.dirty++;
+    propagateExpire(db,key);
+    return dbDelete(db,key);
+}
+
+/*-----------------------------------------------------------------------------
+ * Expires Commands
+ *----------------------------------------------------------------------------*/
+
+void expireGenericCommand(redisClient *c, robj *key, robj *param, long offset) {
+    dictEntry *de;
+    time_t seconds;
+
+    if (getLongFromObjectOrReply(c, param, &seconds, NULL) != REDIS_OK) return;
+
+    seconds -= offset;
+
+    de = dictFind(c->db->dict,key->ptr);
+    if (de == NULL) {
+        addReply(c,shared.czero);
+        return;
+    }
+    if (seconds <= 0) {
+        if (dbDelete(c->db,key)) server.dirty++;
+        addReply(c, shared.cone);
+        touchWatchedKey(c->db,key);
+        return;
+    } else {
+        time_t when = time(NULL)+seconds;
+        setExpire(c->db,key,when);
+        addReply(c,shared.cone);
+        touchWatchedKey(c->db,key);
+        server.dirty++;
+        return;
+    }
+}
+
+void expireCommand(redisClient *c) {
+    expireGenericCommand(c,c->argv[1],c->argv[2],0);
+}
+
+void expireatCommand(redisClient *c) {
+    expireGenericCommand(c,c->argv[1],c->argv[2],time(NULL));
+}
+
+void ttlCommand(redisClient *c) {
+    time_t expire, ttl = -1;
+
+    expire = getExpire(c->db,c->argv[1]);
+    if (expire != -1) {
+        ttl = (expire-time(NULL));
+        if (ttl < 0) ttl = -1;
+    }
+    addReplyLongLong(c,(long long)ttl);
+}
+
+void persistCommand(redisClient *c) {
+    dictEntry *de;
+
+    de = dictFind(c->db->dict,c->argv[1]->ptr);
+    if (de == NULL) {
+        addReply(c,shared.czero);
+    } else {
+        if (removeExpire(c->db,c->argv[1])) {
+            addReply(c,shared.cone);
+            server.dirty++;
+        } else {
+            addReply(c,shared.czero);
+        }
+    }
+}

src/debug.c

@@ -0,0 +1,308 @@
+#include "redis.h"
+#include "sha1.h"   /* SHA1 is used for DEBUG DIGEST */
+
+#include <arpa/inet.h>
+
+/* ================================= Debugging ============================== */
+
+/* Compute the sha1 of string at 's' with 'len' bytes long.
+ * The SHA1 is then xored againt the string pointed by digest.
+ * Since xor is commutative, this operation is used in order to
+ * "add" digests relative to unordered elements.
+ *
+ * So digest(a,b,c,d) will be the same of digest(b,a,c,d) */
+void xorDigest(unsigned char *digest, void *ptr, size_t len) {
+    SHA1_CTX ctx;
+    unsigned char hash[20], *s = ptr;
+    int j;
+
+    SHA1Init(&ctx);
+    SHA1Update(&ctx,s,len);
+    SHA1Final(hash,&ctx);
+
+    for (j = 0; j < 20; j++)
+        digest[j] ^= hash[j];
+}
+
+void xorObjectDigest(unsigned char *digest, robj *o) {
+    o = getDecodedObject(o);
+    xorDigest(digest,o->ptr,sdslen(o->ptr));
+    decrRefCount(o);
+}
+
+/* This function instead of just computing the SHA1 and xoring it
+ * against diget, also perform the digest of "digest" itself and
+ * replace the old value with the new one.
+ *
+ * So the final digest will be:
+ *
+ * digest = SHA1(digest xor SHA1(data))
+ *
+ * This function is used every time we want to preserve the order so
+ * that digest(a,b,c,d) will be different than digest(b,c,d,a)
+ *
+ * Also note that mixdigest("foo") followed by mixdigest("bar")
+ * will lead to a different digest compared to "fo", "obar".
+ */
+void mixDigest(unsigned char *digest, void *ptr, size_t len) {
+    SHA1_CTX ctx;
+    char *s = ptr;
+
+    xorDigest(digest,s,len);
+    SHA1Init(&ctx);
+    SHA1Update(&ctx,digest,20);
+    SHA1Final(digest,&ctx);
+}
+
+void mixObjectDigest(unsigned char *digest, robj *o) {
+    o = getDecodedObject(o);
+    mixDigest(digest,o->ptr,sdslen(o->ptr));
+    decrRefCount(o);
+}
+
+/* Compute the dataset digest. Since keys, sets elements, hashes elements
+ * are not ordered, we use a trick: every aggregate digest is the xor
+ * of the digests of their elements. This way the order will not change
+ * the result. For list instead we use a feedback entering the output digest
+ * as input in order to ensure that a different ordered list will result in
+ * a different digest. */
+void computeDatasetDigest(unsigned char *final) {
+    unsigned char digest[20];
+    char buf[128];
+    dictIterator *di = NULL;
+    dictEntry *de;
+    int j;
+    uint32_t aux;
+
+    memset(final,0,20); /* Start with a clean result */
+
+    for (j = 0; j < server.dbnum; j++) {
+        redisDb *db = server.db+j;
+
+        if (dictSize(db->dict) == 0) continue;
+        di = dictGetIterator(db->dict);
+
+        /* hash the DB id, so the same dataset moved in a different
+         * DB will lead to a different digest */
+        aux = htonl(j);
+        mixDigest(final,&aux,sizeof(aux));
+
+        /* Iterate this DB writing every entry */
+        while((de = dictNext(di)) != NULL) {
+            sds key;
+            robj *keyobj, *o;
+            time_t expiretime;
+
+            memset(digest,0,20); /* This key-val digest */
+            key = dictGetEntryKey(de);
+            keyobj = createStringObject(key,sdslen(key));
+
+            mixDigest(digest,key,sdslen(key));
+
+            /* Make sure the key is loaded if VM is active */
+            o = lookupKeyRead(db,keyobj);
+
+            aux = htonl(o->type);
+            mixDigest(digest,&aux,sizeof(aux));
+            expiretime = getExpire(db,keyobj);
+
+            /* Save the key and associated value */
+            if (o->type == REDIS_STRING) {
+                mixObjectDigest(digest,o);
+            } else if (o->type == REDIS_LIST) {
+                listTypeIterator *li = listTypeInitIterator(o,0,REDIS_TAIL);
+                listTypeEntry entry;
+                while(listTypeNext(li,&entry)) {
+                    robj *eleobj = listTypeGet(&entry);
+                    mixObjectDigest(digest,eleobj);
+                    decrRefCount(eleobj);
+                }
+                listTypeReleaseIterator(li);
+            } else if (o->type == REDIS_SET) {
+                setTypeIterator *si = setTypeInitIterator(o);
+                robj *ele;
+                while((ele = setTypeNext(si)) != NULL) {
+                    xorObjectDigest(digest,ele);
+                    decrRefCount(ele);
+                }
+                setTypeReleaseIterator(si);
+            } else if (o->type == REDIS_ZSET) {
+                zset *zs = o->ptr;
+                dictIterator *di = dictGetIterator(zs->dict);
+                dictEntry *de;
+
+                while((de = dictNext(di)) != NULL) {
+                    robj *eleobj = dictGetEntryKey(de);
+                    double *score = dictGetEntryVal(de);
+                    unsigned char eledigest[20];
+
+                    snprintf(buf,sizeof(buf),"%.17g",*score);
+                    memset(eledigest,0,20);
+                    mixObjectDigest(eledigest,eleobj);
+                    mixDigest(eledigest,buf,strlen(buf));
+                    xorDigest(digest,eledigest,20);
+                }
+                dictReleaseIterator(di);
+            } else if (o->type == REDIS_HASH) {
+                hashTypeIterator *hi;
+                robj *obj;
+
+                hi = hashTypeInitIterator(o);
+                while (hashTypeNext(hi) != REDIS_ERR) {
+                    unsigned char eledigest[20];
+
+                    memset(eledigest,0,20);
+                    obj = hashTypeCurrent(hi,REDIS_HASH_KEY);
+                    mixObjectDigest(eledigest,obj);
+                    decrRefCount(obj);
+                    obj = hashTypeCurrent(hi,REDIS_HASH_VALUE);
+                    mixObjectDigest(eledigest,obj);
+                    decrRefCount(obj);
+                    xorDigest(digest,eledigest,20);
+                }
+                hashTypeReleaseIterator(hi);
+            } else {
+                redisPanic("Unknown object type");
+            }
+            /* If the key has an expire, add it to the mix */
+            if (expiretime != -1) xorDigest(digest,"!!expire!!",10);
+            /* We can finally xor the key-val digest to the final digest */
+            xorDigest(final,digest,20);
+            decrRefCount(keyobj);
+        }
+        dictReleaseIterator(di);
+    }
+}
+
+void debugCommand(redisClient *c) {
+    if (!strcasecmp(c->argv[1]->ptr,"segfault")) {
+        *((char*)-1) = 'x';
+    } else if (!strcasecmp(c->argv[1]->ptr,"reload")) {
+        if (rdbSave(server.dbfilename) != REDIS_OK) {
+            addReply(c,shared.err);
+            return;
+        }
+        emptyDb();
+        if (rdbLoad(server.dbfilename) != REDIS_OK) {
+            addReply(c,shared.err);
+            return;
+        }
+        redisLog(REDIS_WARNING,"DB reloaded by DEBUG RELOAD");
+        addReply(c,shared.ok);
+    } else if (!strcasecmp(c->argv[1]->ptr,"loadaof")) {
+        emptyDb();
+        if (loadAppendOnlyFile(server.appendfilename) != REDIS_OK) {
+            addReply(c,shared.err);
+            return;
+        }
+        redisLog(REDIS_WARNING,"Append Only File loaded by DEBUG LOADAOF");
+        addReply(c,shared.ok);
+    } else if (!strcasecmp(c->argv[1]->ptr,"object") && c->argc == 3) {
+        dictEntry *de = dictFind(c->db->dict,c->argv[2]->ptr);
+        robj *val;
+
+        if (!de) {
+            addReply(c,shared.nokeyerr);
+            return;
+        }
+        val = dictGetEntryVal(de);
+        if (!server.vm_enabled || (val->storage == REDIS_VM_MEMORY ||
+                                   val->storage == REDIS_VM_SWAPPING)) {
+            char *strenc;
+
+            strenc = strEncoding(val->encoding);
+            addReplySds(c,sdscatprintf(sdsempty(),
+                "+Value at:%p refcount:%d "
+                "encoding:%s serializedlength:%lld\r\n",
+                (void*)val, val->refcount,
+                strenc, (long long) rdbSavedObjectLen(val,NULL)));
+        } else {
+            vmpointer *vp = (vmpointer*) val;
+            addReplySds(c,sdscatprintf(sdsempty(),
+                "+Value swapped at: page %llu "
+                "using %llu pages\r\n",
+                (unsigned long long) vp->page,
+                (unsigned long long) vp->usedpages));
+        }
+    } else if (!strcasecmp(c->argv[1]->ptr,"swapin") && c->argc == 3) {
+        lookupKeyRead(c->db,c->argv[2]);
+        addReply(c,shared.ok);
+    } else if (!strcasecmp(c->argv[1]->ptr,"swapout") && c->argc == 3) {
+        dictEntry *de = dictFind(c->db->dict,c->argv[2]->ptr);
+        robj *val;
+        vmpointer *vp;
+
+        if (!server.vm_enabled) {
+            addReplySds(c,sdsnew("-ERR Virtual Memory is disabled\r\n"));
+            return;
+        }
+        if (!de) {
+            addReply(c,shared.nokeyerr);
+            return;
+        }
+        val = dictGetEntryVal(de);
+        /* Swap it */
+        if (val->storage != REDIS_VM_MEMORY) {
+            addReplySds(c,sdsnew("-ERR This key is not in memory\r\n"));
+        } else if (val->refcount != 1) {
+            addReplySds(c,sdsnew("-ERR Object is shared\r\n"));
+        } else if ((vp = vmSwapObjectBlocking(val)) != NULL) {
+            dictGetEntryVal(de) = vp;
+            addReply(c,shared.ok);
+        } else {
+            addReply(c,shared.err);
+        }
+    } else if (!strcasecmp(c->argv[1]->ptr,"populate") && c->argc == 3) {
+        long keys, j;
+        robj *key, *val;
+        char buf[128];
+
+        if (getLongFromObjectOrReply(c, c->argv[2], &keys, NULL) != REDIS_OK)
+            return;
+        for (j = 0; j < keys; j++) {
+            snprintf(buf,sizeof(buf),"key:%lu",j);
+            key = createStringObject(buf,strlen(buf));
+            if (lookupKeyRead(c->db,key) != NULL) {
+                decrRefCount(key);
+                continue;
+            }
+            snprintf(buf,sizeof(buf),"value:%lu",j);
+            val = createStringObject(buf,strlen(buf));
+            dbAdd(c->db,key,val);
+            decrRefCount(key);
+        }
+        addReply(c,shared.ok);
+    } else if (!strcasecmp(c->argv[1]->ptr,"digest") && c->argc == 2) {
+        unsigned char digest[20];
+        sds d = sdsnew("+");
+        int j;
+
+        computeDatasetDigest(digest);
+        for (j = 0; j < 20; j++)
+            d = sdscatprintf(d, "%02x",digest[j]);
+
+        d = sdscatlen(d,"\r\n",2);
+        addReplySds(c,d);
+    } else {
+        addReplySds(c,sdsnew(
+            "-ERR Syntax error, try DEBUG [SEGFAULT|OBJECT <key>|SWAPIN <key>|SWAPOUT <key>|RELOAD]\r\n"));
+    }
+}
+
+void _redisAssert(char *estr, char *file, int line) {
+    redisLog(REDIS_WARNING,"=== ASSERTION FAILED ===");
+    redisLog(REDIS_WARNING,"==> %s:%d '%s' is not true",file,line,estr);
+#ifdef HAVE_BACKTRACE
+    redisLog(REDIS_WARNING,"(forcing SIGSEGV in order to print the stack trace)");
+    *((char*)-1) = 'x';
+#endif
+}
+
+void _redisPanic(char *msg, char *file, int line) {
+    redisLog(REDIS_WARNING,"!!! Software Failure. Press left mouse button to continue");
+    redisLog(REDIS_WARNING,"Guru Meditation: %s #%s:%d",msg,file,line);
+#ifdef HAVE_BACKTRACE
+    redisLog(REDIS_WARNING,"(forcing SIGSEGV in order to print the stack trace)");
+    *((char*)-1) = 'x';
+#endif
+}

src/dict.c

@@ -41,36 +41,16 @@
 #include <stdarg.h>
 #include <assert.h>
 #include <limits.h>
+#include <sys/time.h>
 
 #include "dict.h"
 #include "zmalloc.h"
 
-/* ---------------------------- Utility funcitons --------------------------- */
-
-static void _dictPanic(const char *fmt, ...)
-{
-    va_list ap;
-
-    va_start(ap, fmt);
-    fprintf(stderr, "\nDICT LIBRARY PANIC: ");
-    vfprintf(stderr, fmt, ap);
-    fprintf(stderr, "\n\n");
-    va_end(ap);
-}
-
-/* ------------------------- Heap Management Wrappers------------------------ */
-
-static void *_dictAlloc(size_t size)
-{
-    void *p = zmalloc(size);
-    if (p == NULL)
-        _dictPanic("Out of memory");
-    return p;
-}
-
-static void _dictFree(void *ptr) {
-    zfree(ptr);
-}
+/* Using dictEnableResize() / dictDisableResize() we make possible to
+ * enable/disable resizing of the hash table as needed. This is very important
+ * for Redis, as we use copy-on-write and don't want to move too much memory
+ * around when there is a child performing saving operations. */
+static int dict_can_resize = 1;
 
 /* -------------------------- private prototypes ---------------------------- */
 
@@ -113,7 +93,7 @@ unsigned int dictGenHashFunction(const unsigned char *buf, int len) {
 
 /* Reset an hashtable already initialized with ht_init().
  * NOTE: This function should only called by ht_destroy(). */
-static void _dictReset(dict *ht)
+static void _dictReset(dictht *ht)
 {
     ht->table = NULL;
     ht->size = 0;
@@ -125,104 +105,165 @@ static void _dictReset(dict *ht)
 dict *dictCreate(dictType *type,
         void *privDataPtr)
 {
-    dict *ht = _dictAlloc(sizeof(*ht));
+    dict *d = zmalloc(sizeof(*d));
 
-    _dictInit(ht,type,privDataPtr);
-    return ht;
+    _dictInit(d,type,privDataPtr);
+    return d;
 }
 
 /* Initialize the hash table */
-int _dictInit(dict *ht, dictType *type,
+int _dictInit(dict *d, dictType *type,
         void *privDataPtr)
 {
-    _dictReset(ht);
-    ht->type = type;
-    ht->privdata = privDataPtr;
+    _dictReset(&d->ht[0]);
+    _dictReset(&d->ht[1]);
+    d->type = type;
+    d->privdata = privDataPtr;
+    d->rehashidx = -1;
+    d->iterators = 0;
     return DICT_OK;
 }
 
 /* Resize the table to the minimal size that contains all the elements,
  * but with the invariant of a USER/BUCKETS ration near to <= 1 */
-int dictResize(dict *ht)
+int dictResize(dict *d)
 {
-    int minimal = ht->used;
+    int minimal;
 
+    if (!dict_can_resize || dictIsRehashing(d)) return DICT_ERR;
+    minimal = d->ht[0].used;
     if (minimal < DICT_HT_INITIAL_SIZE)
         minimal = DICT_HT_INITIAL_SIZE;
-    return dictExpand(ht, minimal);
+    return dictExpand(d, minimal);
 }
 
 /* Expand or create the hashtable */
-int dictExpand(dict *ht, unsigned long size)
+int dictExpand(dict *d, unsigned long size)
 {
-    dict n; /* the new hashtable */
-    unsigned long realsize = _dictNextPower(size), i;
+    dictht n; /* the new hashtable */
+    unsigned long realsize = _dictNextPower(size);
 
     /* the size is invalid if it is smaller than the number of
      * elements already inside the hashtable */
-    if (ht->used > size)
+    if (dictIsRehashing(d) || d->ht[0].used > size)
         return DICT_ERR;
 
-    _dictInit(&n, ht->type, ht->privdata);
+    /* Allocate the new hashtable and initialize all pointers to NULL */
     n.size = realsize;
     n.sizemask = realsize-1;
-    n.table = _dictAlloc(realsize*sizeof(dictEntry*));
+    n.table = zcalloc(realsize*sizeof(dictEntry*));
+    n.used = 0;
 
-    /* Initialize all the pointers to NULL */
-    memset(n.table, 0, realsize*sizeof(dictEntry*));
-
-    /* Copy all the elements from the old to the new table:
-     * note that if the old hash table is empty ht->size is zero,
-     * so dictExpand just creates an hash table. */
-    n.used = ht->used;
-    for (i = 0; i < ht->size && ht->used > 0; i++) {
-        dictEntry *he, *nextHe;
-
-        if (ht->table[i] == NULL) continue;
-        
-        /* For each hash entry on this slot... */
-        he = ht->table[i];
-        while(he) {
-            unsigned int h;
-
-            nextHe = he->next;
-            /* Get the new element index */
-            h = dictHashKey(ht, he->key) & n.sizemask;
-            he->next = n.table[h];
-            n.table[h] = he;
-            ht->used--;
-            /* Pass to the next element */
-            he = nextHe;
-        }
+    /* Is this the first initialization? If so it's not really a rehashing
+     * we just set the first hash table so that it can accept keys. */
+    if (d->ht[0].table == NULL) {
+        d->ht[0] = n;
+        return DICT_OK;
     }
-    assert(ht->used == 0);
-    _dictFree(ht->table);
 
-    /* Remap the new hashtable in the old */
-    *ht = n;
+    /* Prepare a second hash table for incremental rehashing */
+    d->ht[1] = n;
+    d->rehashidx = 0;
     return DICT_OK;
 }
 
+/* Performs N steps of incremental rehashing. Returns 1 if there are still
+ * keys to move from the old to the new hash table, otherwise 0 is returned.
+ * Note that a rehashing step consists in moving a bucket (that may have more
+ * thank one key as we use chaining) from the old to the new hash table. */
+int dictRehash(dict *d, int n) {
+    if (!dictIsRehashing(d)) return 0;
+
+    while(n--) {
+        dictEntry *de, *nextde;
+
+        /* Check if we already rehashed the whole table... */
+        if (d->ht[0].used == 0) {
+            zfree(d->ht[0].table);
+            d->ht[0] = d->ht[1];
+            _dictReset(&d->ht[1]);
+            d->rehashidx = -1;
+            return 0;
+        }
+
+        /* Note that rehashidx can't overflow as we are sure there are more
+         * elements because ht[0].used != 0 */
+        while(d->ht[0].table[d->rehashidx] == NULL) d->rehashidx++;
+        de = d->ht[0].table[d->rehashidx];
+        /* Move all the keys in this bucket from the old to the new hash HT */
+        while(de) {
+            unsigned int h;
+
+            nextde = de->next;
+            /* Get the index in the new hash table */
+            h = dictHashKey(d, de->key) & d->ht[1].sizemask;
+            de->next = d->ht[1].table[h];
+            d->ht[1].table[h] = de;
+            d->ht[0].used--;
+            d->ht[1].used++;
+            de = nextde;
+        }
+        d->ht[0].table[d->rehashidx] = NULL;
+        d->rehashidx++;
+    }
+    return 1;
+}
+
+long long timeInMilliseconds(void) {
+    struct timeval tv;
+
+    gettimeofday(&tv,NULL);
+    return (((long long)tv.tv_sec)*1000)+(tv.tv_usec/1000);
+}
+
+/* Rehash for an amount of time between ms milliseconds and ms+1 milliseconds */
+int dictRehashMilliseconds(dict *d, int ms) {
+    long long start = timeInMilliseconds();
+    int rehashes = 0;
+
+    while(dictRehash(d,100)) {
+        rehashes += 100;
+        if (timeInMilliseconds()-start > ms) break;
+    }
+    return rehashes;
+}
+
+/* This function performs just a step of rehashing, and only if there are
+ * not iterators bound to our hash table. When we have iterators in the middle
+ * of a rehashing we can't mess with the two hash tables otherwise some element
+ * can be missed or duplicated.
+ *
+ * This function is called by common lookup or update operations in the
+ * dictionary so that the hash table automatically migrates from H1 to H2
+ * while it is actively used. */
+static void _dictRehashStep(dict *d) {
+    if (d->iterators == 0) dictRehash(d,1);
+}
+
 /* Add an element to the target hash table */
-int dictAdd(dict *ht, void *key, void *val)
+int dictAdd(dict *d, void *key, void *val)
 {
     int index;
     dictEntry *entry;
+    dictht *ht;
+
+    if (dictIsRehashing(d)) _dictRehashStep(d);
 
     /* Get the index of the new element, or -1 if
      * the element already exists. */
-    if ((index = _dictKeyIndex(ht, key)) == -1)
+    if ((index = _dictKeyIndex(d, key)) == -1)
         return DICT_ERR;
 
     /* Allocates the memory and stores key */
-    entry = _dictAlloc(sizeof(*entry));
+    ht = dictIsRehashing(d) ? &d->ht[1] : &d->ht[0];
+    entry = zmalloc(sizeof(*entry));
     entry->next = ht->table[index];
     ht->table[index] = entry;
+    ht->used++;
 
     /* Set the hash entry fields. */
-    dictSetHashKey(ht, entry, key);
-    dictSetHashVal(ht, entry, val);
-    ht->used++;
+    dictSetHashKey(d, entry, key);
+    dictSetHashVal(d, entry, val);
     return DICT_OK;
 }
 
@@ -230,16 +271,16 @@ int dictAdd(dict *ht, void *key, void *val)
  * Return 1 if the key was added from scratch, 0 if there was already an
  * element with such key and dictReplace() just performed a value update
  * operation. */
-int dictReplace(dict *ht, void *key, void *val)
+int dictReplace(dict *d, void *key, void *val)
 {
     dictEntry *entry, auxentry;
 
     /* Try to add the element. If the key
      * does not exists dictAdd will suceed. */
-    if (dictAdd(ht, key, val) == DICT_OK)
+    if (dictAdd(d, key, val) == DICT_OK)
         return 1;
     /* It already exists, get the entry */
-    entry = dictFind(ht, key);
+    entry = dictFind(d, key);
     /* Free the old value and set the new one */
     /* Set the new value and free the old one. Note that it is important
      * to do that in this order, as the value may just be exactly the same
@@ -247,40 +288,45 @@ int dictReplace(dict *ht, void *key, void *val)
      * you want to increment (set), and then decrement (free), and not the
      * reverse. */
     auxentry = *entry;
-    dictSetHashVal(ht, entry, val);
-    dictFreeEntryVal(ht, &auxentry);
+    dictSetHashVal(d, entry, val);
+    dictFreeEntryVal(d, &auxentry);
     return 0;
 }
 
 /* Search and remove an element */
-static int dictGenericDelete(dict *ht, const void *key, int nofree)
+static int dictGenericDelete(dict *d, const void *key, int nofree)
 {
-    unsigned int h;
+    unsigned int h, idx;
     dictEntry *he, *prevHe;
+    int table;
 
-    if (ht->size == 0)
-        return DICT_ERR;
-    h = dictHashKey(ht, key) & ht->sizemask;
-    he = ht->table[h];
+    if (d->ht[0].size == 0) return DICT_ERR; /* d->ht[0].table is NULL */
+    if (dictIsRehashing(d)) _dictRehashStep(d);
+    h = dictHashKey(d, key);
 
-    prevHe = NULL;
-    while(he) {
-        if (dictCompareHashKeys(ht, key, he->key)) {
-            /* Unlink the element from the list */
-            if (prevHe)
-                prevHe->next = he->next;
-            else
-                ht->table[h] = he->next;
-            if (!nofree) {
-                dictFreeEntryKey(ht, he);
-                dictFreeEntryVal(ht, he);
+    for (table = 0; table <= 1; table++) {
+        idx = h & d->ht[table].sizemask;
+        he = d->ht[table].table[idx];
+        prevHe = NULL;
+        while(he) {
+            if (dictCompareHashKeys(d, key, he->key)) {
+                /* Unlink the element from the list */
+                if (prevHe)
+                    prevHe->next = he->next;
+                else
+                    d->ht[table].table[idx] = he->next;
+                if (!nofree) {
+                    dictFreeEntryKey(d, he);
+                    dictFreeEntryVal(d, he);
+                }
+                zfree(he);
+                d->ht[table].used--;
+                return DICT_OK;
             }
-            _dictFree(he);
-            ht->used--;
-            return DICT_OK;
+            prevHe = he;
+            he = he->next;
         }
-        prevHe = he;
-        he = he->next;
+        if (!dictIsRehashing(d)) break;
     }
     return DICT_ERR; /* not found */
 }
@@ -293,8 +339,8 @@ int dictDeleteNoFree(dict *ht, const void *key) {
     return dictGenericDelete(ht,key,1);
 }
 
-/* Destroy an entire hash table */
-int _dictClear(dict *ht)
+/* Destroy an entire dictionary */
+int _dictClear(dict *d, dictht *ht)
 {
     unsigned long i;
 
@@ -305,48 +351,62 @@ int _dictClear(dict *ht)
         if ((he = ht->table[i]) == NULL) continue;
         while(he) {
             nextHe = he->next;
-            dictFreeEntryKey(ht, he);
-            dictFreeEntryVal(ht, he);
-            _dictFree(he);
+            dictFreeEntryKey(d, he);
+            dictFreeEntryVal(d, he);
+            zfree(he);
             ht->used--;
             he = nextHe;
         }
     }
     /* Free the table and the allocated cache structure */
-    _dictFree(ht->table);
+    zfree(ht->table);
     /* Re-initialize the table */
     _dictReset(ht);
     return DICT_OK; /* never fails */
 }
 
 /* Clear & Release the hash table */
-void dictRelease(dict *ht)
+void dictRelease(dict *d)
 {
-    _dictClear(ht);
-    _dictFree(ht);
+    _dictClear(d,&d->ht[0]);
+    _dictClear(d,&d->ht[1]);
+    zfree(d);
 }
 
-dictEntry *dictFind(dict *ht, const void *key)
+dictEntry *dictFind(dict *d, const void *key)
 {
     dictEntry *he;
-    unsigned int h;
+    unsigned int h, idx, table;
 
-    if (ht->size == 0) return NULL;
-    h = dictHashKey(ht, key) & ht->sizemask;
-    he = ht->table[h];
-    while(he) {
-        if (dictCompareHashKeys(ht, key, he->key))
-            return he;
-        he = he->next;
+    if (d->ht[0].size == 0) return NULL; /* We don't have a table at all */
+    if (dictIsRehashing(d)) _dictRehashStep(d);
+    h = dictHashKey(d, key);
+    for (table = 0; table <= 1; table++) {
+        idx = h & d->ht[table].sizemask;
+        he = d->ht[table].table[idx];
+        while(he) {
+            if (dictCompareHashKeys(d, key, he->key))
+                return he;
+            he = he->next;
+        }
+        if (!dictIsRehashing(d)) return NULL;
     }
     return NULL;
 }
 
-dictIterator *dictGetIterator(dict *ht)
-{
-    dictIterator *iter = _dictAlloc(sizeof(*iter));
+void *dictFetchValue(dict *d, const void *key) {
+    dictEntry *he;
 
-    iter->ht = ht;
+    he = dictFind(d,key);
+    return he ? dictGetEntryVal(he) : NULL;
+}
+
+dictIterator *dictGetIterator(dict *d)
+{
+    dictIterator *iter = zmalloc(sizeof(*iter));
+
+    iter->d = d;
+    iter->table = 0;
     iter->index = -1;
     iter->entry = NULL;
     iter->nextEntry = NULL;
@@ -357,10 +417,19 @@ dictEntry *dictNext(dictIterator *iter)
 {
     while (1) {
         if (iter->entry == NULL) {
+            dictht *ht = &iter->d->ht[iter->table];
+            if (iter->index == -1 && iter->table == 0) iter->d->iterators++;
             iter->index++;
-            if (iter->index >=
-                    (signed)iter->ht->size) break;
-            iter->entry = iter->ht->table[iter->index];
+            if (iter->index >= (signed) ht->size) {
+                if (dictIsRehashing(iter->d) && iter->table == 0) {
+                    iter->table++;
+                    iter->index = 0;
+                    ht = &iter->d->ht[1];
+                } else {
+                    break;
+                }
+            }
+            iter->entry = ht->table[iter->index];
         } else {
             iter->entry = iter->nextEntry;
         }
@@ -376,34 +445,45 @@ dictEntry *dictNext(dictIterator *iter)
 
 void dictReleaseIterator(dictIterator *iter)
 {
-    _dictFree(iter);
+    if (!(iter->index == -1 && iter->table == 0)) iter->d->iterators--;
+    zfree(iter);
 }
 
 /* Return a random entry from the hash table. Useful to
  * implement randomized algorithms */
-dictEntry *dictGetRandomKey(dict *ht)
+dictEntry *dictGetRandomKey(dict *d)
 {
-    dictEntry *he;
+    dictEntry *he, *orighe;
     unsigned int h;
     int listlen, listele;
 
-    if (ht->used == 0) return NULL;
-    do {
-        h = random() & ht->sizemask;
-        he = ht->table[h];
-    } while(he == NULL);
+    if (dictSize(d) == 0) return NULL;
+    if (dictIsRehashing(d)) _dictRehashStep(d);
+    if (dictIsRehashing(d)) {
+        do {
+            h = random() % (d->ht[0].size+d->ht[1].size);
+            he = (h >= d->ht[0].size) ? d->ht[1].table[h - d->ht[0].size] :
+                                      d->ht[0].table[h];
+        } while(he == NULL);
+    } else {
+        do {
+            h = random() & d->ht[0].sizemask;
+            he = d->ht[0].table[h];
+        } while(he == NULL);
+    }
 
     /* Now we found a non empty bucket, but it is a linked
      * list and we need to get a random element from the list.
-     * The only sane way to do so is to count the element and
+     * The only sane way to do so is counting the elements and
      * select a random index. */
     listlen = 0;
+    orighe = he;
     while(he) {
         he = he->next;
         listlen++;
     }
     listele = random() % listlen;
-    he = ht->table[h];
+    he = orighe;
     while(listele--) he = he->next;
     return he;
 }
@@ -411,14 +491,16 @@ dictEntry *dictGetRandomKey(dict *ht)
 /* ------------------------- private functions ------------------------------ */
 
 /* Expand the hash table if needed */
-static int _dictExpandIfNeeded(dict *ht)
+static int _dictExpandIfNeeded(dict *d)
 {
     /* If the hash table is empty expand it to the intial size,
      * if the table is "full" dobule its size. */
-    if (ht->size == 0)
-        return dictExpand(ht, DICT_HT_INITIAL_SIZE);
-    if (ht->used == ht->size)
-        return dictExpand(ht, ht->size*2);
+    if (dictIsRehashing(d)) return DICT_OK;
+    if (d->ht[0].size == 0)
+        return dictExpand(d, DICT_HT_INITIAL_SIZE);
+    if (d->ht[0].used >= d->ht[0].size && dict_can_resize)
+        return dictExpand(d, ((d->ht[0].size > d->ht[0].used) ?
+                                    d->ht[0].size : d->ht[0].used)*2);
     return DICT_OK;
 }
 
@@ -437,33 +519,43 @@ static unsigned long _dictNextPower(unsigned long size)
 
 /* Returns the index of a free slot that can be populated with
  * an hash entry for the given 'key'.
- * If the key already exists, -1 is returned. */
-static int _dictKeyIndex(dict *ht, const void *key)
+ * If the key already exists, -1 is returned.
+ *
+ * Note that if we are in the process of rehashing the hash table, the
+ * index is always returned in the context of the second (new) hash table. */
+static int _dictKeyIndex(dict *d, const void *key)
 {
-    unsigned int h;
+    unsigned int h, idx, table;
     dictEntry *he;
 
     /* Expand the hashtable if needed */
-    if (_dictExpandIfNeeded(ht) == DICT_ERR)
+    if (_dictExpandIfNeeded(d) == DICT_ERR)
         return -1;
     /* Compute the key hash value */
-    h = dictHashKey(ht, key) & ht->sizemask;
-    /* Search if this slot does not already contain the given key */
-    he = ht->table[h];
-    while(he) {
-        if (dictCompareHashKeys(ht, key, he->key))
-            return -1;
-        he = he->next;
+    h = dictHashKey(d, key);
+    for (table = 0; table <= 1; table++) {
+        idx = h & d->ht[table].sizemask;
+        /* Search if this slot does not already contain the given key */
+        he = d->ht[table].table[idx];
+        while(he) {
+            if (dictCompareHashKeys(d, key, he->key))
+                return -1;
+            he = he->next;
+        }
+        if (!dictIsRehashing(d)) break;
     }
-    return h;
+    return idx;
 }
 
-void dictEmpty(dict *ht) {
-    _dictClear(ht);
+void dictEmpty(dict *d) {
+    _dictClear(d,&d->ht[0]);
+    _dictClear(d,&d->ht[1]);
+    d->rehashidx = -1;
+    d->iterators = 0;
 }
 
 #define DICT_STATS_VECTLEN 50
-void dictPrintStats(dict *ht) {
+static void _dictPrintStatsHt(dictht *ht) {
     unsigned long i, slots = 0, chainlen, maxchainlen = 0;
     unsigned long totchainlen = 0;
     unsigned long clvector[DICT_STATS_VECTLEN];
@@ -507,6 +599,28 @@ void dictPrintStats(dict *ht) {
     }
 }
 
+void dictPrintStats(dict *d) {
+    _dictPrintStatsHt(&d->ht[0]);
+    if (dictIsRehashing(d)) {
+        printf("-- Rehashing into ht[1]:\n");
+        _dictPrintStatsHt(&d->ht[1]);
+    }
+}
+
+void dictEnableResize(void) {
+    dict_can_resize = 1;
+}
+
+void dictDisableResize(void) {
+    dict_can_resize = 0;
+}
+
+#if 0
+
+/* The following are just example hash table types implementations.
+ * Not useful for Redis so they are commented out.
+ */
+
 /* ----------------------- StringCopy Hash Table Type ------------------------*/
 
 static unsigned int _dictStringCopyHTHashFunction(const void *key)
@@ -514,10 +628,10 @@ static unsigned int _dictStringCopyHTHashFunction(const void *key)
     return dictGenHashFunction(key, strlen(key));
 }
 
-static void *_dictStringCopyHTKeyDup(void *privdata, const void *key)
+static void *_dictStringDup(void *privdata, const void *key)
 {
     int len = strlen(key);
-    char *copy = _dictAlloc(len+1);
+    char *copy = zmalloc(len+1);
     DICT_NOTUSED(privdata);
 
     memcpy(copy, key, len);
@@ -525,17 +639,6 @@ static void *_dictStringCopyHTKeyDup(void *privdata, const void *key)
     return copy;
 }
 
-static void *_dictStringKeyValCopyHTValDup(void *privdata, const void *val)
-{
-    int len = strlen(val);
-    char *copy = _dictAlloc(len+1);
-    DICT_NOTUSED(privdata);
-
-    memcpy(copy, val, len);
-    copy[len] = '\0';
-    return copy;
-}
-
 static int _dictStringCopyHTKeyCompare(void *privdata, const void *key1,
         const void *key2)
 {
@@ -544,47 +647,41 @@ static int _dictStringCopyHTKeyCompare(void *privdata, const void *key1,
     return strcmp(key1, key2) == 0;
 }
 
-static void _dictStringCopyHTKeyDestructor(void *privdata, void *key)
+static void _dictStringDestructor(void *privdata, void *key)
 {
     DICT_NOTUSED(privdata);
 
-    _dictFree((void*)key); /* ATTENTION: const cast */
-}
-
-static void _dictStringKeyValCopyHTValDestructor(void *privdata, void *val)
-{
-    DICT_NOTUSED(privdata);
-
-    _dictFree((void*)val); /* ATTENTION: const cast */
+    zfree(key);
 }
 
 dictType dictTypeHeapStringCopyKey = {
-    _dictStringCopyHTHashFunction,        /* hash function */
-    _dictStringCopyHTKeyDup,              /* key dup */
-    NULL,                               /* val dup */
-    _dictStringCopyHTKeyCompare,          /* key compare */
-    _dictStringCopyHTKeyDestructor,       /* key destructor */
-    NULL                                /* val destructor */
+    _dictStringCopyHTHashFunction, /* hash function */
+    _dictStringDup,                /* key dup */
+    NULL,                          /* val dup */
+    _dictStringCopyHTKeyCompare,   /* key compare */
+    _dictStringDestructor,         /* key destructor */
+    NULL                           /* val destructor */
 };
 
 /* This is like StringCopy but does not auto-duplicate the key.
  * It's used for intepreter's shared strings. */
 dictType dictTypeHeapStrings = {
-    _dictStringCopyHTHashFunction,        /* hash function */
-    NULL,                               /* key dup */
-    NULL,                               /* val dup */
-    _dictStringCopyHTKeyCompare,          /* key compare */
-    _dictStringCopyHTKeyDestructor,       /* key destructor */
-    NULL                                /* val destructor */
+    _dictStringCopyHTHashFunction, /* hash function */
+    NULL,                          /* key dup */
+    NULL,                          /* val dup */
+    _dictStringCopyHTKeyCompare,   /* key compare */
+    _dictStringDestructor,         /* key destructor */
+    NULL                           /* val destructor */
 };
 
 /* This is like StringCopy but also automatically handle dynamic
  * allocated C strings as values. */
 dictType dictTypeHeapStringCopyKeyValue = {
-    _dictStringCopyHTHashFunction,        /* hash function */
-    _dictStringCopyHTKeyDup,              /* key dup */
-    _dictStringKeyValCopyHTValDup,        /* val dup */
-    _dictStringCopyHTKeyCompare,          /* key compare */
-    _dictStringCopyHTKeyDestructor,       /* key destructor */
-    _dictStringKeyValCopyHTValDestructor, /* val destructor */
+    _dictStringCopyHTHashFunction, /* hash function */
+    _dictStringDup,                /* key dup */
+    _dictStringDup,                /* val dup */
+    _dictStringCopyHTKeyCompare,   /* key compare */
+    _dictStringDestructor,         /* key destructor */
+    _dictStringDestructor,         /* val destructor */
 };
+#endif

src/dict.h

@@ -57,17 +57,26 @@ typedef struct dictType {
     void (*valDestructor)(void *privdata, void *obj);
 } dictType;
 
-typedef struct dict {
+/* This is our hash table structure. Every dictionary has two of this as we
+ * implement incremental rehashing, for the old to the new table. */
+typedef struct dictht {
     dictEntry **table;
-    dictType *type;
     unsigned long size;
     unsigned long sizemask;
     unsigned long used;
+} dictht;
+
+typedef struct dict {
+    dictType *type;
     void *privdata;
+    dictht ht[2];
+    int rehashidx; /* rehashing not in progress if rehashidx == -1 */
+    int iterators; /* number of iterators currently running */
 } dict;
 
 typedef struct dictIterator {
-    dict *ht;
+    dict *d;
+    int table;
     int index;
     dictEntry *entry, *nextEntry;
 } dictIterator;
@@ -76,57 +85,63 @@ typedef struct dictIterator {
 #define DICT_HT_INITIAL_SIZE     4
 
 /* ------------------------------- Macros ------------------------------------*/
-#define dictFreeEntryVal(ht, entry) \
-    if ((ht)->type->valDestructor) \
-        (ht)->type->valDestructor((ht)->privdata, (entry)->val)
+#define dictFreeEntryVal(d, entry) \
+    if ((d)->type->valDestructor) \
+        (d)->type->valDestructor((d)->privdata, (entry)->val)
 
-#define dictSetHashVal(ht, entry, _val_) do { \
-    if ((ht)->type->valDup) \
-        entry->val = (ht)->type->valDup((ht)->privdata, _val_); \
+#define dictSetHashVal(d, entry, _val_) do { \
+    if ((d)->type->valDup) \
+        entry->val = (d)->type->valDup((d)->privdata, _val_); \
     else \
         entry->val = (_val_); \
 } while(0)
 
-#define dictFreeEntryKey(ht, entry) \
-    if ((ht)->type->keyDestructor) \
-        (ht)->type->keyDestructor((ht)->privdata, (entry)->key)
+#define dictFreeEntryKey(d, entry) \
+    if ((d)->type->keyDestructor) \
+        (d)->type->keyDestructor((d)->privdata, (entry)->key)
 
-#define dictSetHashKey(ht, entry, _key_) do { \
-    if ((ht)->type->keyDup) \
-        entry->key = (ht)->type->keyDup((ht)->privdata, _key_); \
+#define dictSetHashKey(d, entry, _key_) do { \
+    if ((d)->type->keyDup) \
+        entry->key = (d)->type->keyDup((d)->privdata, _key_); \
     else \
         entry->key = (_key_); \
 } while(0)
 
-#define dictCompareHashKeys(ht, key1, key2) \
-    (((ht)->type->keyCompare) ? \
-        (ht)->type->keyCompare((ht)->privdata, key1, key2) : \
+#define dictCompareHashKeys(d, key1, key2) \
+    (((d)->type->keyCompare) ? \
+        (d)->type->keyCompare((d)->privdata, key1, key2) : \
         (key1) == (key2))
 
-#define dictHashKey(ht, key) (ht)->type->hashFunction(key)
+#define dictHashKey(d, key) (d)->type->hashFunction(key)
 
 #define dictGetEntryKey(he) ((he)->key)
 #define dictGetEntryVal(he) ((he)->val)
-#define dictSlots(ht) ((ht)->size)
-#define dictSize(ht) ((ht)->used)
+#define dictSlots(d) ((d)->ht[0].size+(d)->ht[1].size)
+#define dictSize(d) ((d)->ht[0].used+(d)->ht[1].used)
+#define dictIsRehashing(ht) ((ht)->rehashidx != -1)
 
 /* API */
 dict *dictCreate(dictType *type, void *privDataPtr);
-int dictExpand(dict *ht, unsigned long size);
-int dictAdd(dict *ht, void *key, void *val);
-int dictReplace(dict *ht, void *key, void *val);
-int dictDelete(dict *ht, const void *key);
-int dictDeleteNoFree(dict *ht, const void *key);
-void dictRelease(dict *ht);
-dictEntry * dictFind(dict *ht, const void *key);
-int dictResize(dict *ht);
-dictIterator *dictGetIterator(dict *ht);
+int dictExpand(dict *d, unsigned long size);
+int dictAdd(dict *d, void *key, void *val);
+int dictReplace(dict *d, void *key, void *val);
+int dictDelete(dict *d, const void *key);
+int dictDeleteNoFree(dict *d, const void *key);
+void dictRelease(dict *d);
+dictEntry * dictFind(dict *d, const void *key);
+void *dictFetchValue(dict *d, const void *key);
+int dictResize(dict *d);
+dictIterator *dictGetIterator(dict *d);
 dictEntry *dictNext(dictIterator *iter);
 void dictReleaseIterator(dictIterator *iter);
-dictEntry *dictGetRandomKey(dict *ht);
-void dictPrintStats(dict *ht);
+dictEntry *dictGetRandomKey(dict *d);
+void dictPrintStats(dict *d);
 unsigned int dictGenHashFunction(const unsigned char *buf, int len);
-void dictEmpty(dict *ht);
+void dictEmpty(dict *d);
+void dictEnableResize(void);
+void dictDisableResize(void);
+int dictRehash(dict *d, int n);
+int dictRehashMilliseconds(dict *d, int ms);
 
 /* Hash table types */
 extern dictType dictTypeHeapStringCopyKey;

src/fmacros.h

@@ -2,7 +2,13 @@
 #define _REDIS_FMACRO_H
 
 #define _BSD_SOURCE
+
+#ifdef __linux__
+#define _XOPEN_SOURCE 700
+#else
 #define _XOPEN_SOURCE
+#endif
+
 #define _LARGEFILE_SOURCE
 #define _FILE_OFFSET_BITS 64
 

src/intset.c

@@ -0,0 +1,422 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "intset.h"
+#include "zmalloc.h"
+
+/* Note that these encodings are ordered, so:
+ * INTSET_ENC_INT16 < INTSET_ENC_INT32 < INTSET_ENC_INT64. */
+#define INTSET_ENC_INT16 (sizeof(int16_t))
+#define INTSET_ENC_INT32 (sizeof(int32_t))
+#define INTSET_ENC_INT64 (sizeof(int64_t))
+
+/* Return the required encoding for the provided value. */
+static uint8_t _intsetValueEncoding(int64_t v) {
+    if (v < INT32_MIN || v > INT32_MAX)
+        return INTSET_ENC_INT64;
+    else if (v < INT16_MIN || v > INT16_MAX)
+        return INTSET_ENC_INT32;
+    return INTSET_ENC_INT16;
+}
+
+/* Return the value at pos, given an encoding. */
+static int64_t _intsetGetEncoded(intset *is, int pos, uint8_t enc) {
+    if (enc == INTSET_ENC_INT64)
+        return ((int64_t*)is->contents)[pos];
+    else if (enc == INTSET_ENC_INT32)
+        return ((int32_t*)is->contents)[pos];
+    return ((int16_t*)is->contents)[pos];
+}
+
+/* Return the value at pos, using the configured encoding. */
+static int64_t _intsetGet(intset *is, int pos) {
+    return _intsetGetEncoded(is,pos,is->encoding);
+}
+
+/* Set the value at pos, using the configured encoding. */
+static void _intsetSet(intset *is, int pos, int64_t value) {
+    if (is->encoding == INTSET_ENC_INT64)
+        ((int64_t*)is->contents)[pos] = value;
+    else if (is->encoding == INTSET_ENC_INT32)
+        ((int32_t*)is->contents)[pos] = value;
+    else
+        ((int16_t*)is->contents)[pos] = value;
+}
+
+/* Create an empty intset. */
+intset *intsetNew(void) {
+    intset *is = zmalloc(sizeof(intset));
+    is->encoding = INTSET_ENC_INT16;
+    is->length = 0;
+    return is;
+}
+
+/* Resize the intset */
+static intset *intsetResize(intset *is, uint32_t len) {
+    uint32_t size = len*is->encoding;
+    is = zrealloc(is,sizeof(intset)+size);
+    return is;
+}
+
+/* Search for the position of "value". Return 1 when the value was found and
+ * sets "pos" to the position of the value within the intset. Return 0 when
+ * the value is not present in the intset and sets "pos" to the position
+ * where "value" can be inserted. */
+static uint8_t intsetSearch(intset *is, int64_t value, uint32_t *pos) {
+    int min = 0, max = is->length-1, mid = -1;
+    int64_t cur = -1;
+
+    /* The value can never be found when the set is empty */
+    if (is->length == 0) {
+        if (pos) *pos = 0;
+        return 0;
+    } else {
+        /* Check for the case where we know we cannot find the value,
+         * but do know the insert position. */
+        if (value > _intsetGet(is,is->length-1)) {
+            if (pos) *pos = is->length;
+            return 0;
+        } else if (value < _intsetGet(is,0)) {
+            if (pos) *pos = 0;
+            return 0;
+        }
+    }
+
+    while(max >= min) {
+        mid = (min+max)/2;
+        cur = _intsetGet(is,mid);
+        if (value > cur) {
+            min = mid+1;
+        } else if (value < cur) {
+            max = mid-1;
+        } else {
+            break;
+        }
+    }
+
+    if (value == cur) {
+        if (pos) *pos = mid;
+        return 1;
+    } else {
+        if (pos) *pos = min;
+        return 0;
+    }
+}
+
+/* Upgrades the intset to a larger encoding and inserts the given integer. */
+static intset *intsetUpgradeAndAdd(intset *is, int64_t value) {
+    uint8_t curenc = is->encoding;
+    uint8_t newenc = _intsetValueEncoding(value);
+    int length = is->length;
+    int prepend = value < 0 ? 1 : 0;
+
+    /* First set new encoding and resize */
+    is->encoding = newenc;
+    is = intsetResize(is,is->length+1);
+
+    /* Upgrade back-to-front so we don't overwrite values.
+     * Note that the "prepend" variable is used to make sure we have an empty
+     * space at either the beginning or the end of the intset. */
+    while(length--)
+        _intsetSet(is,length+prepend,_intsetGetEncoded(is,length,curenc));
+
+    /* Set the value at the beginning or the end. */
+    if (prepend)
+        _intsetSet(is,0,value);
+    else
+        _intsetSet(is,is->length,value);
+    is->length++;
+    return is;
+}
+
+static void intsetMoveTail(intset *is, uint32_t from, uint32_t to) {
+    void *src, *dst;
+    uint32_t bytes = is->length-from;
+    if (is->encoding == INTSET_ENC_INT64) {
+        src = (int64_t*)is->contents+from;
+        dst = (int64_t*)is->contents+to;
+        bytes *= sizeof(int64_t);
+    } else if (is->encoding == INTSET_ENC_INT32) {
+        src = (int32_t*)is->contents+from;
+        dst = (int32_t*)is->contents+to;
+        bytes *= sizeof(int32_t);
+    } else {
+        src = (int16_t*)is->contents+from;
+        dst = (int16_t*)is->contents+to;
+        bytes *= sizeof(int16_t);
+    }
+    memmove(dst,src,bytes);
+}
+
+/* Insert an integer in the intset */
+intset *intsetAdd(intset *is, int64_t value, uint8_t *success) {
+    uint8_t valenc = _intsetValueEncoding(value);
+    uint32_t pos;
+    if (success) *success = 1;
+
+    /* Upgrade encoding if necessary. If we need to upgrade, we know that
+     * this value should be either appended (if > 0) or prepended (if < 0),
+     * because it lies outside the range of existing values. */
+    if (valenc > is->encoding) {
+        /* This always succeeds, so we don't need to curry *success. */
+        return intsetUpgradeAndAdd(is,value);
+    } else {
+        /* Abort if the value is already present in the set.
+         * This call will populate "pos" with the right position to insert
+         * the value when it cannot be found. */
+        if (intsetSearch(is,value,&pos)) {
+            if (success) *success = 0;
+            return is;
+        }
+
+        is = intsetResize(is,is->length+1);
+        if (pos < is->length) intsetMoveTail(is,pos,pos+1);
+    }
+
+    _intsetSet(is,pos,value);
+    is->length++;
+    return is;
+}
+
+/* Delete integer from intset */
+intset *intsetRemove(intset *is, int64_t value, uint8_t *success) {
+    uint8_t valenc = _intsetValueEncoding(value);
+    uint32_t pos;
+    if (success) *success = 0;
+
+    if (valenc <= is->encoding && intsetSearch(is,value,&pos)) {
+        /* We know we can delete */
+        if (success) *success = 1;
+
+        /* Overwrite value with tail and update length */
+        if (pos < (is->length-1)) intsetMoveTail(is,pos+1,pos);
+        is = intsetResize(is,is->length-1);
+        is->length--;
+    }
+    return is;
+}
+
+/* Determine whether a value belongs to this set */
+uint8_t intsetFind(intset *is, int64_t value) {
+    uint8_t valenc = _intsetValueEncoding(value);
+    return valenc <= is->encoding && intsetSearch(is,value,NULL);
+}
+
+/* Return random member */
+int64_t intsetRandom(intset *is) {
+    return _intsetGet(is,rand()%is->length);
+}
+
+/* Sets the value to the value at the given position. When this position is
+ * out of range the function returns 0, when in range it returns 1. */
+uint8_t intsetGet(intset *is, uint32_t pos, int64_t *value) {
+    if (pos < is->length) {
+        *value = _intsetGet(is,pos);
+        return 1;
+    }
+    return 0;
+}
+
+/* Return intset length */
+uint32_t intsetLen(intset *is) {
+    return is->length;
+}
+
+#ifdef INTSET_TEST_MAIN
+#include <sys/time.h>
+
+void intsetRepr(intset *is) {
+    int i;
+    for (i = 0; i < is->length; i++) {
+        printf("%lld\n", (uint64_t)_intsetGet(is,i));
+    }
+    printf("\n");
+}
+
+void error(char *err) {
+    printf("%s\n", err);
+    exit(1);
+}
+
+void ok(void) {
+    printf("OK\n");
+}
+
+long long usec(void) {
+    struct timeval tv;
+    gettimeofday(&tv,NULL);
+    return (((long long)tv.tv_sec)*1000000)+tv.tv_usec;
+}
+
+#define assert(_e) ((_e)?(void)0:(_assert(#_e,__FILE__,__LINE__),exit(1)))
+void _assert(char *estr, char *file, int line) {
+    printf("\n\n=== ASSERTION FAILED ===\n");
+    printf("==> %s:%d '%s' is not true\n",file,line,estr);
+}
+
+intset *createSet(int bits, int size) {
+    uint64_t mask = (1<<bits)-1;
+    uint64_t i, value;
+    intset *is = intsetNew();
+
+    for (i = 0; i < size; i++) {
+        if (bits > 32) {
+            value = (rand()*rand()) & mask;
+        } else {
+            value = rand() & mask;
+        }
+        is = intsetAdd(is,value,NULL);
+    }
+    return is;
+}
+
+void checkConsistency(intset *is) {
+    int i;
+
+    for (i = 0; i < (is->length-1); i++) {
+        if (is->encoding == INTSET_ENC_INT16) {
+            int16_t *i16 = (int16_t*)is->contents;
+            assert(i16[i] < i16[i+1]);
+        } else if (is->encoding == INTSET_ENC_INT32) {
+            int32_t *i32 = (int32_t*)is->contents;
+            assert(i32[i] < i32[i+1]);
+        } else {
+            int64_t *i64 = (int64_t*)is->contents;
+            assert(i64[i] < i64[i+1]);
+        }
+    }
+}
+
+int main(int argc, char **argv) {
+    uint8_t success;
+    int i;
+    intset *is;
+    sranddev();
+
+    printf("Value encodings: "); {
+        assert(_intsetValueEncoding(-32768) == INTSET_ENC_INT16);
+        assert(_intsetValueEncoding(+32767) == INTSET_ENC_INT16);
+        assert(_intsetValueEncoding(-32769) == INTSET_ENC_INT32);
+        assert(_intsetValueEncoding(+32768) == INTSET_ENC_INT32);
+        assert(_intsetValueEncoding(-2147483648) == INTSET_ENC_INT32);
+        assert(_intsetValueEncoding(+2147483647) == INTSET_ENC_INT32);
+        assert(_intsetValueEncoding(-2147483649) == INTSET_ENC_INT64);
+        assert(_intsetValueEncoding(+2147483648) == INTSET_ENC_INT64);
+        assert(_intsetValueEncoding(-9223372036854775808ull) == INTSET_ENC_INT64);
+        assert(_intsetValueEncoding(+9223372036854775807ull) == INTSET_ENC_INT64);
+        ok();
+    }
+
+    printf("Basic adding: "); {
+        is = intsetNew();
+        is = intsetAdd(is,5,&success); assert(success);
+        is = intsetAdd(is,6,&success); assert(success);
+        is = intsetAdd(is,4,&success); assert(success);
+        is = intsetAdd(is,4,&success); assert(!success);
+        ok();
+    }
+
+    printf("Large number of random adds: "); {
+        int inserts = 0;
+        is = intsetNew();
+        for (i = 0; i < 1024; i++) {
+            is = intsetAdd(is,rand()%0x800,&success);
+            if (success) inserts++;
+        }
+        assert(is->length == inserts);
+        checkConsistency(is);
+        ok();
+    }
+
+    printf("Upgrade from int16 to int32: "); {
+        is = intsetNew();
+        is = intsetAdd(is,32,NULL);
+        assert(is->encoding == INTSET_ENC_INT16);
+        is = intsetAdd(is,65535,NULL);
+        assert(is->encoding == INTSET_ENC_INT32);
+        assert(intsetFind(is,32));
+        assert(intsetFind(is,65535));
+        checkConsistency(is);
+
+        is = intsetNew();
+        is = intsetAdd(is,32,NULL);
+        assert(is->encoding == INTSET_ENC_INT16);
+        is = intsetAdd(is,-65535,NULL);
+        assert(is->encoding == INTSET_ENC_INT32);
+        assert(intsetFind(is,32));
+        assert(intsetFind(is,-65535));
+        checkConsistency(is);
+        ok();
+    }
+
+    printf("Upgrade from int16 to int64: "); {
+        is = intsetNew();
+        is = intsetAdd(is,32,NULL);
+        assert(is->encoding == INTSET_ENC_INT16);
+        is = intsetAdd(is,4294967295,NULL);
+        assert(is->encoding == INTSET_ENC_INT64);
+        assert(intsetFind(is,32));
+        assert(intsetFind(is,4294967295));
+        checkConsistency(is);
+
+        is = intsetNew();
+        is = intsetAdd(is,32,NULL);
+        assert(is->encoding == INTSET_ENC_INT16);
+        is = intsetAdd(is,-4294967295,NULL);
+        assert(is->encoding == INTSET_ENC_INT64);
+        assert(intsetFind(is,32));
+        assert(intsetFind(is,-4294967295));
+        checkConsistency(is);
+        ok();
+    }
+
+    printf("Upgrade from int32 to int64: "); {
+        is = intsetNew();
+        is = intsetAdd(is,65535,NULL);
+        assert(is->encoding == INTSET_ENC_INT32);
+        is = intsetAdd(is,4294967295,NULL);
+        assert(is->encoding == INTSET_ENC_INT64);
+        assert(intsetFind(is,65535));
+        assert(intsetFind(is,4294967295));
+        checkConsistency(is);
+
+        is = intsetNew();
+        is = intsetAdd(is,65535,NULL);
+        assert(is->encoding == INTSET_ENC_INT32);
+        is = intsetAdd(is,-4294967295,NULL);
+        assert(is->encoding == INTSET_ENC_INT64);
+        assert(intsetFind(is,65535));
+        assert(intsetFind(is,-4294967295));
+        checkConsistency(is);
+        ok();
+    }
+
+    printf("Stress lookups: "); {
+        long num = 100000, size = 10000;
+        int i, bits = 20;
+        long long start;
+        is = createSet(bits,size);
+        checkConsistency(is);
+
+        start = usec();
+        for (i = 0; i < num; i++) intsetSearch(is,rand() % ((1<<bits)-1),NULL);
+        printf("%ld lookups, %ld element set, %lldusec\n",num,size,usec()-start);
+    }
+
+    printf("Stress add+delete: "); {
+        int i, v1, v2;
+        is = intsetNew();
+        for (i = 0; i < 0xffff; i++) {
+            v1 = rand() % 0xfff;
+            is = intsetAdd(is,v1,NULL);
+            assert(intsetFind(is,v1));
+
+            v2 = rand() % 0xfff;
+            is = intsetRemove(is,v2,NULL);
+            assert(!intsetFind(is,v2));
+        }
+        checkConsistency(is);
+        ok();
+    }
+}
+#endif

src/intset.h

@@ -0,0 +1,19 @@
+#ifndef __INTSET_H
+#define __INTSET_H
+#include <stdint.h>
+
+typedef struct intset {
+    uint32_t encoding;
+    uint32_t length;
+    int8_t contents[];
+} intset;
+
+intset *intsetNew(void);
+intset *intsetAdd(intset *is, int64_t value, uint8_t *success);
+intset *intsetRemove(intset *is, int64_t value, uint8_t *success);
+uint8_t intsetFind(intset *is, int64_t value);
+int64_t intsetRandom(intset *is);
+uint8_t intsetGet(intset *is, uint32_t pos, int64_t *value);
+uint32_t intsetLen(intset *is);
+
+#endif // __INTSET_H

src/linenoise.c

@@ -0,0 +1,471 @@
+/* linenoise.c -- guerrilla line editing library against the idea that a
+ * line editing lib needs to be 20,000 lines of C code.
+ *
+ * You can find the latest source code at:
+ * 
+ *   http://github.com/antirez/linenoise
+ *
+ * Does a number of crazy assumptions that happen to be true in 99.9999% of
+ * the 2010 UNIX computers around.
+ *
+ * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * References:
+ * - http://invisible-island.net/xterm/ctlseqs/ctlseqs.html
+ * - http://www.3waylabs.com/nw/WWW/products/wizcon/vt220.html
+ *
+ * Todo list:
+ * - Switch to gets() if $TERM is something we can't support.
+ * - Filter bogus Ctrl+<char> combinations.
+ * - Win32 support
+ *
+ * Bloat:
+ * - Completion?
+ * - History search like Ctrl+r in readline?
+ *
+ * List of escape sequences used by this program, we do everything just
+ * with three sequences. In order to be so cheap we may have some
+ * flickering effect with some slow terminal, but the lesser sequences
+ * the more compatible.
+ *
+ * CHA (Cursor Horizontal Absolute)
+ *    Sequence: ESC [ n G
+ *    Effect: moves cursor to column n
+ *
+ * EL (Erase Line)
+ *    Sequence: ESC [ n K
+ *    Effect: if n is 0 or missing, clear from cursor to end of line
+ *    Effect: if n is 1, clear from beginning of line to cursor
+ *    Effect: if n is 2, clear entire line
+ *
+ * CUF (CUrsor Forward)
+ *    Sequence: ESC [ n C
+ *    Effect: moves cursor forward of n chars
+ * 
+ */
+
+#include "fmacros.h"
+
+#include <termios.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/ioctl.h>
+#include <unistd.h>
+
+#define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100
+#define LINENOISE_MAX_LINE 4096
+static char *unsupported_term[] = {"dumb","cons25",NULL};
+
+static struct termios orig_termios; /* in order to restore at exit */
+static int rawmode = 0; /* for atexit() function to check if restore is needed*/
+static int atexit_registered = 0; /* register atexit just 1 time */
+static int history_max_len = LINENOISE_DEFAULT_HISTORY_MAX_LEN;
+static int history_len = 0;
+char **history = NULL;
+
+static void linenoiseAtExit(void);
+int linenoiseHistoryAdd(const char *line);
+
+static int isUnsupportedTerm(void) {
+    char *term = getenv("TERM");
+    int j;
+
+    if (term == NULL) return 0;
+    for (j = 0; unsupported_term[j]; j++)
+        if (!strcasecmp(term,unsupported_term[j])) return 1;
+    return 0;
+}
+
+static void freeHistory(void) {
+    if (history) {
+        int j;
+
+        for (j = 0; j < history_len; j++)
+            free(history[j]);
+        free(history);
+    }
+}
+
+static int enableRawMode(int fd) {
+    struct termios raw;
+
+    if (!isatty(STDIN_FILENO)) goto fatal;
+    if (!atexit_registered) {
+        atexit(linenoiseAtExit);
+        atexit_registered = 1;
+    }
+    if (tcgetattr(fd,&orig_termios) == -1) goto fatal;
+
+    raw = orig_termios;  /* modify the original mode */
+    /* input modes: no break, no CR to NL, no parity check, no strip char,
+     * no start/stop output control. */
+    raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON);
+    /* output modes - disable post processing */
+    raw.c_oflag &= ~(OPOST);
+    /* control modes - set 8 bit chars */
+    raw.c_cflag |= (CS8);
+    /* local modes - choing off, canonical off, no extended functions,
+     * no signal chars (^Z,^C) */
+    raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG);
+    /* control chars - set return condition: min number of bytes and timer.
+     * We want read to return every single byte, without timeout. */
+    raw.c_cc[VMIN] = 1; raw.c_cc[VTIME] = 0; /* 1 byte, no timer */
+
+    /* put terminal in raw mode after flushing */
+    if (tcsetattr(fd,TCSAFLUSH,&raw) < 0) goto fatal;
+    rawmode = 1;
+    return 0;
+
+fatal:
+    errno = ENOTTY;
+    return -1;
+}
+
+static void disableRawMode(int fd) {
+    /* Don't even check the return value as it's too late. */
+    if (rawmode && tcsetattr(fd,TCSAFLUSH,&orig_termios) != -1)
+        rawmode = 0;
+}
+
+/* At exit we'll try to fix the terminal to the initial conditions. */
+static void linenoiseAtExit(void) {
+    disableRawMode(STDIN_FILENO);
+    freeHistory();
+}
+
+static int getColumns(void) {
+    struct winsize ws;
+
+    if (ioctl(1, TIOCGWINSZ, &ws) == -1) return 80;
+    return ws.ws_col;
+}
+
+static void refreshLine(int fd, const char *prompt, char *buf, size_t len, size_t pos, size_t cols) {
+    char seq[64];
+    size_t plen = strlen(prompt);
+    
+    while((plen+pos) >= cols) {
+        buf++;
+        len--;
+        pos--;
+    }
+    while (plen+len > cols) {
+        len--;
+    }
+
+    /* Cursor to left edge */
+    snprintf(seq,64,"\x1b[0G");
+    if (write(fd,seq,strlen(seq)) == -1) return;
+    /* Write the prompt and the current buffer content */
+    if (write(fd,prompt,strlen(prompt)) == -1) return;
+    if (write(fd,buf,len) == -1) return;
+    /* Erase to right */
+    snprintf(seq,64,"\x1b[0K");
+    if (write(fd,seq,strlen(seq)) == -1) return;
+    /* Move cursor to original position. */
+    snprintf(seq,64,"\x1b[0G\x1b[%dC", (int)(pos+plen));
+    if (write(fd,seq,strlen(seq)) == -1) return;
+}
+
+static int linenoisePrompt(int fd, char *buf, size_t buflen, const char *prompt) {
+    size_t plen = strlen(prompt);
+    size_t pos = 0;
+    size_t len = 0;
+    size_t cols = getColumns();
+    int history_index = 0;
+
+    buf[0] = '\0';
+    buflen--; /* Make sure there is always space for the nulterm */
+
+    /* The latest history entry is always our current buffer, that
+     * initially is just an empty string. */
+    linenoiseHistoryAdd("");
+    
+    if (write(fd,prompt,plen) == -1) return -1;
+    while(1) {
+        char c;
+        int nread;
+        char seq[2];
+
+        nread = read(fd,&c,1);
+        if (nread <= 0) return len;
+        switch(c) {
+        case 13:    /* enter */
+        case 4:     /* ctrl-d */
+            history_len--;
+            free(history[history_len]);
+            return (len == 0 && c == 4) ? -1 : (int)len;
+        case 3:     /* ctrl-c */
+            errno = EAGAIN;
+            return -1;
+        case 127:   /* backspace */
+        case 8:     /* ctrl-h */
+            if (pos > 0 && len > 0) {
+                memmove(buf+pos-1,buf+pos,len-pos);
+                pos--;
+                len--;
+                buf[len] = '\0';
+                refreshLine(fd,prompt,buf,len,pos,cols);
+            }
+            break;
+        case 20:    /* ctrl-t */
+            if (pos > 0 && pos < len) {
+                int aux = buf[pos-1];
+                buf[pos-1] = buf[pos];
+                buf[pos] = aux;
+                if (pos != len-1) pos++;
+                refreshLine(fd,prompt,buf,len,pos,cols);
+            }
+            break;
+        case 2:     /* ctrl-b */
+            goto left_arrow;
+        case 6:     /* ctrl-f */
+            goto right_arrow;
+        case 16:    /* ctrl-p */
+            seq[1] = 65;
+            goto up_down_arrow;
+        case 14:    /* ctrl-n */
+            seq[1] = 66;
+            goto up_down_arrow;
+            break;
+        case 27:    /* escape sequence */
+            if (read(fd,seq,2) == -1) break;
+            if (seq[0] == 91 && seq[1] == 68) {
+left_arrow:
+                /* left arrow */
+                if (pos > 0) {
+                    pos--;
+                    refreshLine(fd,prompt,buf,len,pos,cols);
+                }
+            } else if (seq[0] == 91 && seq[1] == 67) {
+right_arrow:
+                /* right arrow */
+                if (pos != len) {
+                    pos++;
+                    refreshLine(fd,prompt,buf,len,pos,cols);
+                }
+            } else if (seq[0] == 91 && (seq[1] == 65 || seq[1] == 66)) {
+up_down_arrow:
+                /* up and down arrow: history */
+                if (history_len > 1) {
+                    /* Update the current history entry before to
+                     * overwrite it with tne next one. */
+                    free(history[history_len-1-history_index]);
+                    history[history_len-1-history_index] = strdup(buf);
+                    /* Show the new entry */
+                    history_index += (seq[1] == 65) ? 1 : -1;
+                    if (history_index < 0) {
+                        history_index = 0;
+                        break;
+                    } else if (history_index >= history_len) {
+                        history_index = history_len-1;
+                        break;
+                    }
+                    strncpy(buf,history[history_len-1-history_index],buflen);
+                    buf[buflen] = '\0';
+                    len = pos = strlen(buf);
+                    refreshLine(fd,prompt,buf,len,pos,cols);
+                }
+            }
+            break;
+        default:
+            if (len < buflen) {
+                if (len == pos) {
+                    buf[pos] = c;
+                    pos++;
+                    len++;
+                    buf[len] = '\0';
+                    if (plen+len < cols) {
+                        /* Avoid a full update of the line in the
+                         * trivial case. */
+                        if (write(fd,&c,1) == -1) return -1;
+                    } else {
+                        refreshLine(fd,prompt,buf,len,pos,cols);
+                    }
+                } else {
+                    memmove(buf+pos+1,buf+pos,len-pos);
+                    buf[pos] = c;
+                    len++;
+                    pos++;
+                    buf[len] = '\0';
+                    refreshLine(fd,prompt,buf,len,pos,cols);
+                }
+            }
+            break;
+        case 21: /* Ctrl+u, delete the whole line. */
+            buf[0] = '\0';
+            pos = len = 0;
+            refreshLine(fd,prompt,buf,len,pos,cols);
+            break;
+        case 11: /* Ctrl+k, delete from current to end of line. */
+            buf[pos] = '\0';
+            len = pos;
+            refreshLine(fd,prompt,buf,len,pos,cols);
+            break;
+        case 1: /* Ctrl+a, go to the start of the line */
+            pos = 0;
+            refreshLine(fd,prompt,buf,len,pos,cols);
+            break;
+        case 5: /* ctrl+e, go to the end of the line */
+            pos = len;
+            refreshLine(fd,prompt,buf,len,pos,cols);
+            break;
+        }
+    }
+    return len;
+}
+
+static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) {
+    int fd = STDIN_FILENO;
+    int count;
+
+    if (buflen == 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!isatty(STDIN_FILENO)) {
+        if (fgets(buf, buflen, stdin) == NULL) return -1;
+        count = strlen(buf);
+        if (count && buf[count-1] == '\n') {
+            count--;
+            buf[count] = '\0';
+        }
+    } else {
+        if (enableRawMode(fd) == -1) return -1;
+        count = linenoisePrompt(fd, buf, buflen, prompt);
+        disableRawMode(fd);
+        printf("\n");
+    }
+    return count;
+}
+
+char *linenoise(const char *prompt) {
+    char buf[LINENOISE_MAX_LINE];
+    int count;
+
+    if (isUnsupportedTerm()) {
+        size_t len;
+
+        printf("%s",prompt);
+        fflush(stdout);
+        if (fgets(buf,LINENOISE_MAX_LINE,stdin) == NULL) return NULL;
+        len = strlen(buf);
+        while(len && (buf[len-1] == '\n' || buf[len-1] == '\r')) {
+            len--;
+            buf[len] = '\0';
+        }
+        return strdup(buf);
+    } else {
+        count = linenoiseRaw(buf,LINENOISE_MAX_LINE,prompt);
+        if (count == -1) return NULL;
+        return strdup(buf);
+    }
+}
+
+/* Using a circular buffer is smarter, but a bit more complex to handle. */
+int linenoiseHistoryAdd(const char *line) {
+    char *linecopy;
+
+    if (history_max_len == 0) return 0;
+    if (history == NULL) {
+        history = malloc(sizeof(char*)*history_max_len);
+        if (history == NULL) return 0;
+        memset(history,0,(sizeof(char*)*history_max_len));
+    }
+    linecopy = strdup(line);
+    if (!linecopy) return 0;
+    if (history_len == history_max_len) {
+        free(history[0]);
+        memmove(history,history+1,sizeof(char*)*(history_max_len-1));
+        history_len--;
+    }
+    history[history_len] = linecopy;
+    history_len++;
+    return 1;
+}
+
+int linenoiseHistorySetMaxLen(int len) {
+    char **new;
+
+    if (len < 1) return 0;
+    if (history) {
+        int tocopy = history_len;
+
+        new = malloc(sizeof(char*)*len);
+        if (new == NULL) return 0;
+        if (len < tocopy) tocopy = len;
+        memcpy(new,history+(history_max_len-tocopy), sizeof(char*)*tocopy);
+        free(history);
+        history = new;
+    }
+    history_max_len = len;
+    if (history_len > history_max_len)
+        history_len = history_max_len;
+    return 1;
+}
+
+/* Save the history in the specified file. On success 0 is returned
+ * otherwise -1 is returned. */
+int linenoiseHistorySave(char *filename) {
+    FILE *fp = fopen(filename,"w");
+    int j;
+    
+    if (fp == NULL) return -1;
+    for (j = 0; j < history_len; j++)
+        fprintf(fp,"%s\n",history[j]);
+    fclose(fp);
+    return 0;
+}
+
+/* Load the history from the specified file. If the file does not exist
+ * zero is returned and no operation is performed.
+ *
+ * If the file exists and the operation succeeded 0 is returned, otherwise
+ * on error -1 is returned. */
+int linenoiseHistoryLoad(char *filename) {
+    FILE *fp = fopen(filename,"r");
+    char buf[LINENOISE_MAX_LINE];
+    
+    if (fp == NULL) return -1;
+
+    while (fgets(buf,LINENOISE_MAX_LINE,fp) != NULL) {
+        char *p;
+        
+        p = strchr(buf,'\r');
+        if (!p) p = strchr(buf,'\n');
+        if (p) *p = '\0';
+        linenoiseHistoryAdd(buf);
+    }
+    fclose(fp);
+    return 0;
+}

src/linenoise.h

@@ -1,5 +1,9 @@
-/*
- * Copyright (c) 2009-2010, Salvatore Sanfilippo <antirez at gmail dot com>
+/* linenoise.h -- guerrilla line editing library against the idea that a
+ * line editing lib needs to be 20,000 lines of C code.
+ *
+ * See linenoise.c for more information.
+ *
+ * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -27,49 +31,13 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-#ifndef __REDIS_H__
-#define __REDIS_H__
+#ifndef __LINENOISE_H
+#define __LINENOISE_H
 
-enum
-{
-  REG_GS = 0,
-# define REG_GS		REG_GS
-  REG_FS,
-# define REG_FS		REG_FS
-  REG_ES,
-# define REG_ES		REG_ES
-  REG_DS,
-# define REG_DS		REG_DS
-  REG_EDI,
-# define REG_EDI	REG_EDI
-  REG_ESI,
-# define REG_ESI	REG_ESI
-  REG_EBP,
-# define REG_EBP	REG_EBP
-  REG_ESP,
-# define REG_ESP	REG_ESP
-  REG_EBX,
-# define REG_EBX	REG_EBX
-  REG_EDX,
-# define REG_EDX	REG_EDX
-  REG_ECX,
-# define REG_ECX	REG_ECX
-  REG_EAX,
-# define REG_EAX	REG_EAX
-  REG_TRAPNO,
-# define REG_TRAPNO	REG_TRAPNO
-  REG_ERR,
-# define REG_ERR	REG_ERR
-  REG_EIP,
-# define REG_EIP	REG_EIP
-  REG_CS,
-# define REG_CS		REG_CS
-  REG_EFL,
-# define REG_EFL	REG_EFL
-  REG_UESP,
-# define REG_UESP	REG_UESP
-  REG_SS
-# define REG_SS	REG_SS
-};
+char *linenoise(const char *prompt);
+int linenoiseHistoryAdd(const char *line);
+int linenoiseHistorySetMaxLen(int len);
+int linenoiseHistorySave(char *filename);
+int linenoiseHistoryLoad(char *filename);
 
-#endif
+#endif /* __LINENOISE_H */