Compare commits

..
249 Commits
Author SHA1 Message Date
Roshan KhatriandMadelyn Olson 45d824c003 Fix Tcl 9 compatibility in cluster packet test (#3251)
Remove -encoding binary from fconfigure as it is no longer supported in
Tcl 9 (Fedora Rawhide and Daily / test-fedoralatest-jemalloc
(pull_request)). -translation binary alone is sufficient.
```
https://github.com/valkey-io/valkey/actions/runs/22324297258/job/64590589777?pr=3225
===== Start of server log (pid 17617) =====

### Starting server for test 
17617:M 23 Feb 2026 21:21:47.846 # WARNING Memory overcommit must be enabled! Without it, a background save or replication may fail under low memory condition. Being disabled, it can also cause failures without low memory condition, see https://github.com/jemalloc/jemalloc/issues/1328. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect.
17617:M 23 Feb 2026 21:21:47.846 * oO0OoO0OoO0Oo Valkey is starting oO0OoO0OoO0Oo
17617:M 23 Feb 2026 21:21:47.846 * Valkey version=9.0.3, bits=64, commit=00000000, modified=0, pid=17617, just started
17617:M 23 Feb 2026 21:21:47.846 * Configuration loaded
17617:M 23 Feb 2026 21:21:47.847 * monotonic clock: POSIX clock_gettime
                .+^+.                                                
            .+#########+.                                            
        .+########+########+.           Valkey 9.0.3 (00000000/0) 64 bit
    .+########+'     '+########+.                                    
 .########+'     .+.     '+########.    Running in cluster mode
 |####+'     .+#######+.     '+####|    Port: 25121
 |###|   .+###############+.   |###|    PID: 17617                     
 |###|   |#####*'' ''*#####|   |###|                                 
 |###|   |####'  .-.  '####|   |###|                                 
 |###|   |###(  (@@@)  )###|   |###|          https://valkey.io/      
 |###|   |####.  '-'  .####|   |###|                                 
 |###|   |#####*.   .*#####|   |###|                                 
 |###|   '+#####|   |#####+'   |###|                                 
 |####+.     +##|   |#+'     .+####|                                 
 '#######+   |##|        .+########'                                 
    '+###|   |##|    .+########+'                                    
        '|   |####+########+'                                        
             +#########+'                                            
                '+v+'                                                

17617:M 23 Feb 2026 21:21:47.849 * No cluster configuration found, I'm 5501916ccdf76dee6b652cab10402cab3a8f9152
17617:M 23 Feb 2026 21:21:47.852 * Server initialized
17617:M 23 Feb 2026 21:21:47.852 * Ready to accept connections tcp
17617:M 23 Feb 2026 21:21:47.852 * Ready to accept connections unix
17617:M 23 Feb 2026 21:21:47.963 - Accepted 127.0.0.1:37791
17617:M 23 Feb 2026 21:21:47.963 - Client closed connection id=2 addr=127.0.0.1:37791 laddr=127.0.0.1:25121 fd=12 name= age=0 idle=0 flags=N capa= db=0 sub=0 psub=0 ssub=0 multi=-1 watch=0 qbuf=0 qbuf-free=20474 argv-mem=0 multi-mem=0 rbs=16384 rbp=16384 obl=0 oll=0 omem=0 tot-mem=37504 events=r cmd=ping user=default redir=-1 resp=2 lib-name= lib-ver= tot-net-in=7 tot-net-out=7 tot-cmds=1
17617:M 23 Feb 2026 21:21:47.969 - Accepted 127.0.0.1:39251
17617:M 23 Feb 2026 21:21:47.970 * configEpoch set to 1 via CLUSTER SET-CONFIG-EPOCH
17617:M 23 Feb 2026 21:21:47.975 # Missing implement of connection type tls
17617:M 23 Feb 2026 21:21:47.976 # DEBUG LOG: ========== I am primary 0 ==========
17617:M 23 Feb 2026 21:21:49.872 * Cluster state changed: ok
### Starting test Packet with missing gossip messages don't cause invalid read in tests/unit/cluster/packet.tcl
17617:M 23 Feb 2026 21:21:49.879 - Accepting cluster node connection from 127.0.0.1:33701
17617:M 23 Feb 2026 21:21:49.880 - Client closed connection id=3 addr=127.0.0.1:39251 laddr=127.0.0.1:25121 fd=12 name= age=2 idle=0 flags=N capa= db=0 sub=0 psub=0 ssub=0 multi=-1 watch=0 qbuf=0 qbuf-free=20474 argv-mem=0 multi-mem=0 rbs=1024 rbp=426 obl=0 oll=0 omem=0 tot-mem=22144 events=r cmd=cluster|info user=default redir=-1 resp=2 lib-name= lib-ver= tot-net-in=1363 tot-net-out=17324 tot-cmds=47
17617:signal-handler (1771881709) Received SIGTERM scheduling shutdown...
17617:M 23 Feb 2026 21:21:49.973 * User requested shutdown...
17617:M 23 Feb 2026 21:21:49.973 * Removing the pid file.
17617:M 23 Feb 2026 21:21:49.973 * Saving the cluster configuration file before exiting.
17617:M 23 Feb 2026 21:21:49.978 * Removing the unix socket file.
17617:M 23 Feb 2026 21:21:49.978 # Valkey is now ready to exit, bye bye...
===== End of server log (pid 17617) =====


===== Start of server stderr log (pid 17617) =====


===== End of server stderr log (pid 17617) =====

[exception]: Executing test client: unknown encoding "binary": No longer supported.
	please use either "-translation binary" or "-encoding iso8859-1".
unknown encoding "binary": No longer supported.
	please use either "-translation binary" or "-encoding iso8859-1"
    while executing
"fconfigure $sock -translation binary -encoding binary -buffering none -blocking 1"
    ("uplevel" body line 18)
    invoked from within
"uplevel 1 $code"
    (procedure "test" line 62)
    invoked from within
"test "Packet with missing gossip messages don't cause invalid read" {
        set base_port [srv 0 port]
        set cluster_port [expr {$base_port + ..."
    ("uplevel" body line 2)
    invoked from within
"uplevel 1 $code"
    (procedure "cluster_setup" line 41)
    invoked from within
"cluster_setup 1 0 1 continuous_slot_allocation default_replica_allocation {
    test "Packet with missing gossip messages don't cause invalid read" {
..."
    ("uplevel" body line 1)
    invoked from within
"uplevel 1 $code "
    (procedure "start_server" line 2)
    invoked from within
"start_server {overrides {cluster-enabled yes cluster-ping-interval 100 cluster-node-timeout 3000 cluster-databases 16 cluster-slot-stats-enabled yes} ..."
    ("uplevel" body line 1)
    invoked from within
"uplevel 1 $code"
    (procedure "start_multiple_servers" line 5)
    invoked from within
"start_multiple_servers $node_count $options $code"
    (procedure "start_cluster" line 17)
    invoked from within
"start_cluster 1 0 {tags {external:skip cluster tls:skip}} {
    test "Packet with missing gossip messages don't cause invalid read" {
        set base..."
    (file "tests/unit/cluster/packet.tcl" line 84)
    invoked from within
"source $path"
    (procedure "execute_test_file" line 4)
    invoked from within
"execute_test_file $data"
    (procedure "test_client_main" line 10)
    invoked from within
"test_client_main $::test_server_port "
Killing still running Valkey server 10700
Killing still running Valkey server 11659
Killing still running Valkey server 11705
Killing still running Valkey server 12735
Killing still running Valkey server 12775
Killing still running Valkey server 12808
Killing still running Valkey server 12858
Killing still running Valkey server 12901
Killing still running Valkey server 12940
Killing still running Valkey server 12972
Killing still running Valkey server 13004
Killing still running Valkey server 13036
Killing still running Valkey server 13073
Killing still running Valkey server 14952
Killing still running Valkey server 16631
Killing still running Valkey server 16670
Killing still running Valkey server 16686
Killing still running Valkey server 16702
Killing still running Valkey server 16718
Killing still running Valkey server 16734
Killing still running Valkey server 16753
Killing still running Valkey server 16771
Killing still running Valkey server 17094
Killing still running Valkey server 17112
Killing still running Valkey server 17133
Killing still running Valkey server 17262
Killing still running Valkey server 17278
Killing still running Valkey server 17316
Killing still running Valkey server 17349
Killing still running Valkey server 17379
Killing still running Valkey server 17410
Killing still running Valkey server 17430
Killing still running Valkey server 17443
Killing still running Valkey server 17459
Killing still running Valkey server 17478
Killing still running Valkey server 17491
Killing still running Valkey server 17507
Killing still running Valkey server 17526
Killing still running Valkey server 17545
Killing still running Valkey server 17561
Killing still running Valkey server 17577
Killing still running Valkey server 17716
Killing still running Valkey server 17778
Killing still running Valkey server 17825
Killing still running Valkey server 17848
Killing still running Valkey server 17896
Killing still running Valkey server 17931
Killing still running Valkey server 17953
Killing still running Valkey server 17973
Killing still running Valkey server 17991
Killing still running Valkey server 18010
Killing still running Valkey server 18064
Killing still running Valkey server 18082
Killing still running Valkey server 18134
Killing still running Valkey server 18118
Killing still running Valkey server 18150
Killing still running Valkey server 18166
Killing still running Valkey server 18188
Killing still running Valkey server 18209
```

Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
Roshan KhatriandMadelyn Olson 4506261fbf Updates version.h and adds release notes
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
Roshan KhatriandMadelyn Olson c8d443e369 Fix uninitialized variable warning in test_rax.c
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
Roshan KhatriandMadelyn Olson 8ebfd1c68b Fix for [CVE-2025-67733] RESP Protocol Injectiton via Lua error_reply
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
Roshan KhatriandMadelyn Olson caf17be808 Fix for [CVE-2026-21863] Remote DoS with malformed Valkey Cluster bus message
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
2026-02-23 19:51:42 -08:00
Sachin Venkatesha MurthyandMadelyn Olson 380e6178ad [BUG][issue-2797] Block module unload when ACL references module subcommands (#3160)
Summary:

Prevent MODULE UNLOAD when ACL rules reference a module subcommand.
Avoids crash during ACL recompute after module removal.
Adds coverage for subcommand ACL rules.
Tests:

make test
Fixes #2797

---------

Signed-off-by: sachinvmurthy <sachin.murthy97@gmail.com>
Signed-off-by: Sachin Venkatesha Murthy <sachin.murthy97@gmail.com>
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
Jacob MurphyandMadelyn Olson 8fd84df7f8 Fix bug causing no response flush sometimes when IO threads are busy (#3205)
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
Roshan KhatriandMadelyn Olson 5747c0ed71 Spelling corrections, no critical changes
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
Rain ValentineandMadelyn Olson 7a79a3e814 Update and pin github actions to full SHAs for supply chain security (#3185)
Updates to latest versions for each of the github actions used.

Pinning prevents an attack where the upstream action dependency is
compromised and the "v4" tag for example gets edited to point to a
malicious version. We already do this for most checkout actions in our
workflows.

---------

Signed-off-by: Rain Valentine <rsg000@gmail.com>
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
Daniil KashapovandMadelyn Olson 1fc0f68932 Fix server assert on ACL LOAD and resetchannels (#3182)
If the client executing `ACL LOAD` is a user that is removed from ACL
file, it frees itself while still executing the command. Same happens
for user while removing it's channel permissions.
This causes `c->conn` pointer to become `null` and later
`serverAssert(c->conn)` fails for that client on write in
`prepareClientToWrite()`.
Solution is to flag current client to be closed after command completes
and reply is written, later this client is freed async.

Resolves #3181

---------

Signed-off-by: Daniil Kashapov <daniil.kashapov.ykt@gmail.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
Vitah LinandMadelyn Olson b8f07b95de Fix 'Client output buffer hard limit is enforced' test causing infinite loop on macOS 15.4 (#1940)
This PR fixes an issue in the CI test for client-output-buffer-limit,
which was causing an infinite loop when running on macOS 15.4.

### Problem

This test start two clients, R and R1:
```c
R1 subscribe foo
R publish foo bar
```

When R executes `PUBLISH foo bar`, the server first stores the message
`bar` in R1‘s buf. Only when the space in buf is insufficient does it
call `_addReplyProtoToList`.
Inside this function, `closeClientOnOutputBufferLimitReached` is invoked
to check whether the client’s R1 output buffer has reached its
configured limit.
On macOS 15.4, because the server writes to the client at a high speed,
R1’s buf never gets full. As a result,
`closeClientOnOutputBufferLimitReached` in the test is never triggered,
causing the test to never exit and fall into an infinite loop.

### Fixed

I changed `r publish foo bar` to `r publish foo [string repeat bar 50]`
to ensure the buffer is filled, which correctly reproduces the scenario
where omem increases.

Signed-off-by: vitah <vitahlin@gmail.com>
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
Zhijun LiaoandMadelyn Olson a94d8dffe5 Skip flaky test cases in client-eviction.tcl when in TLS mode (#3151)
Closes #3146

The following two test cases are flaky

- `evict clients only until below limit` - uses exact math expecting
exactly half the clients evicted
- `evict clients in right order (large to small)` - uses exact math
expecting specific clients evicted in order

It's fine to skip them in TLS because the core logic being tested
(client eviction) doesn't change based on TLS vs non-TLS.

The `decrease maxmemory-clients causes client eviction` test case could
potentially be flaky as well (has not shown flakiness on CI yet), but
since it has more tolerant assertion: `connected_clients > 0 &&
connected_clients < $client_count`, I think it's okay not to bother
skipping it.

Other test cases are not flaky because they use large thresholds or
check binary outcomes (yes/no eviction), not exact counts.

Signed-off-by: Zhijun <dszhijun@gmail.com>
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
Daniil KashapovandMadelyn Olson 78f8f9a7aa Add waits in test-ubuntu-reclaim-cache (#3134)
After #3103 time sensitive `test-ubuntu-reclaim-cache` started to fail
because now startup always includes 30ms of calibration of HW clock,
that's why we get this output:

```
Run echo "test SAVE doesn't increase cache"
test SAVE doesn't increase cache
2460491776
Could not connect to Valkey at 127.0.0.1:8080: Connection refused
```

Added waits for server to start, locally run, it helps

---------

Signed-off-by: Daniil Kashapov <daniil.kashapov.ykt@gmail.com>
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
BinbinandMadelyn Olson f8ec00fb35 Fix discarded-qualifiers warnings reported by fedorarawhide (#2874)
fedorarawhide CI reports these warnings:
```
networking.c: In function 'afterErrorReply':
networking.c:821:30: error: initialization discards 'const' qualifier from pointer target type [-Werror=discarded-qualifiers]
  821 |             char *spaceloc = memchr(s, ' ', len < 32 ? len : 32);
```

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
Harkrishn PatroandMadelyn Olson 834e85b50a [flaky-failure-fix] Increase the cluster-node-timeout to have longer delay between failover of each shard (#2793)
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
13470f7828 Fix: ltrim should not call signalModifiedKey when no elements are removed (#2787)
There’s an issue with the LTRIM command. When LTRIM does not actually
modify the key — for example, with `LTRIM key 0 -1` — the server.dirty
counter is not updated because both ltrim and rtrim values are 0. As a
result, the command is not propagated. However, `signalModifiedKey` is
still called regardless of whether server.dirty changes. This behavior
is unexpected and can cause a mismatch between the source and target
during propagation, since the LTRIM command is not sent.

Signed-off-by: Harry Lin <harrylhl@amazon.com>
Co-authored-by: Harry Lin <harrylhl@amazon.com>
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
Roshan KhatriandMadelyn Olson b9c86573e1 Deflake Psync established within grace period (#2743)
increased the wait time to a total of 10 seconds where we check the log
for `Done loading RDB` message

Fixes https://github.com/valkey-io/valkey/issues/2694

CI run (100 times):
https://github.com/roshkhatri/valkey/actions/runs/18576201712/job/52961907806

Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
496820dabf Fix used_memory_dataset underflow due to miscalculated used_memory_overhead (#3005)
The metric `used_memory_dataset` turned into an insanely large number close to 2^64 (actually
overflowed negative value), as reported in #2994.

When server starts, the global variable `server.initial_memory_usage` is used to record a memory
baseline in InitServerLast. This `server.initial_memory_usage` has clearly included initial database
memory, since databases are created in initServer.

In function getMemoryOverheadData, the `mem_total` is firstly assigned the baseline, which includes
initial database memory. And then all extra memory usage of databases are added to mem_total. The initial
database memory are therefore counted TWICE.

This eventually caused wrongly larger `used_memory_overhead`. For a database with only a couple of keys,
the `used_memory_overhead` is easily larger than `used_memory` and causes an overflowed `used_memory_dataset`.

In function getMemoryOverheadData(), kvstores without any allocated hashtable are ignored from calculation:
```c
if (db == NULL || !kvstoreNumAllocatedHashtables(db->keys)) continue;
```

However, even the kvstore has no allocated hashtable, there are still some memory allocated by kvstoreCreate(),
including `hashtable_size_index`, which can be larger than 128 KiB.

On the contrary, this caused wrongly smaller `used_memory_overhead` for an empty database. When we insert only
ONE key to the database, the database is suddenly taken into account, and `used_memory_overhead` will increase
(for `used_memory_dataset` decrease) by more than 128 KiB due to the single key insertion.

Signed-off-by: Ace Breakpoint <chemistudio@gmail.com>
Signed-off-by: bpint <chemistudio@gmail.com>
Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
BinbinandMadelyn Olson b966e8550e Fix chained replica crash when doing dual channel replication (#2983)
There is a crash in freeReplicationBacklog:
```
Discarding previously cached primary state.
ASSERTION FAILED
'listLength(server.replicas) == 0' is not true
freeReplicationBacklog
```

The reason is that during dual channel operation, the RDB channel is protected.
In the chained replica case, `disconnectReplicas` is called to disconnect all
replica clients, but since the RDB channel is protected, `freeClient` does not
actually free the replica client. Later, we encounter an assertion failure in
`freeReplicationBacklog`.
```
void replicationAttachToNewPrimary(void) {
    /* Replica starts to apply data from new primary, we must discard the cached
     * primary structure. */
    serverAssert(server.primary == NULL);
    replicationDiscardCachedPrimary();

    /* Cancel any in progress imports (we will now use the primary's) */
    clusterCleanSlotImportsOnFullSync();

    disconnectReplicas();     /* Force our replicas to resync with us as well. */
    freeReplicationBacklog(); /* Don't allow our chained replicas to PSYNC. */
}
```

Dual channel replication was introduced in #60.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
Vitah LinandMadelyn Olson fae80f0f0b Upgrade macos version in actions (#2920)
GitHub has deprecated older macOS runners, and macos-13 is no longer supported.

1. The latest version of cross-platform-actions/action does allow
running on ubuntu-latest (Linux runner) and does not strictly require macOS.
2. Previously, cross-platform-actions/action@v0.22.0 used runs-on:
macos-13. I checked the latest version of cross-platform-actions, and
the official examples now use runs-on: ubuntu. I think we can switch from macOS to Ubuntu.

---------

Signed-off-by: Vitah Lin <vitahlin@gmail.com>
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
Sarthak AggarwalandMadelyn Olson 6584c6534e Fixes test-freebsd workflow in daily (package lang/tclX) (#2832)
This PR fixes the freebsd daily job that has been failing consistently
for the last days with the error "pkg: No packages available to install
matching 'lang/tclx' have been found in the repositories".

The package name is corrected from `lang/tclx` to `lang/tclX`. The
lowercase version worked previously but appears to have stopped working
in an update of freebsd's pkg tool to 2.4.x.

Example of failed job:

https://github.com/valkey-io/valkey/actions/runs/19282092345/job/55135193499

Signed-off-by: Sarthak Aggarwal <sarthagg@amazon.com>
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
Sarthak AggarwalandMadelyn Olson eeb9a7b12c Log test details at the end when the test times out (#2276)
Resolves #2267

Timed out test gets logged at the end of the test run.

```
!!! WARNING The following tests failed:

*** [TIMEOUT]: WAIT should not acknowledge 2 additional copies of the data in tests/unit/wait.tcl
Cleanup: may take some time... OK
```

Signed-off-by: Sarthak Aggarwal <sarthagg@amazon.com>
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
47eebd733d Fix test_reclaimFilePageCache to avoid tmpfs (#1379)
Avoid tmpfs as fadvise(FADV_DONTNEED) has no effect on memory-backed
filesystems.

Fixes https://github.com/valkey-io/valkey/issues/897

---------

Signed-off-by: Ran Shidlansik <ranshid@amazon.com>
Signed-off-by: ranshid <88133677+ranshid@users.noreply.github.com>
Co-authored-by: ranshid <88133677+ranshid@users.noreply.github.com>
Co-authored-by: Ran Shidlansik <ranshid@amazon.com>
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2026-02-23 19:51:42 -08:00
029e35ac15 [Backport 8.0] Allow TCL 9.0 for tests (#1673) (#2813)
Makes our tests possible to run with TCL 9.

The latest Fedora now has TCL 9.0 and it's working now, including the
TCL TLS package. (This wasn't working earlier due to some packaging
errors for TCL packages in Fedora, which have been fixed now.)

This PR also removes the custom compilation of TCL 8 used in our Daily
jobs and uses the system default TCL version instead. The TCL version
depends on the OS. For the latest Fedora, you get 9.0, for macOS you get
8.5 and for most other OSes you get 8.6.

The checks for TCL 8.7 are removed, because 8.7 doesn't exist. It was
never released.

Backport of #1673.

Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-11-06 23:44:25 +01:00
2f27253c4f Release notes for 8.0.6
updated release notes and version file

Signed-off-by: Rain Valentine <rsg000@gmail.com>
Co-authored-by: Harkrishn Patro <bunty.hari@gmail.com>
2025-10-03 13:27:36 -07:00
Madelyn OlsonandGitHub 702292426f Fix format issues with CVE fix (#2682)
The CVE fixes had a formatting and external test issue that wasn't
caught because private branches don't run those CI steps. 

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
2025-10-03 10:33:18 -07:00
Madelyn OlsonandGitHub 9846a31111 Merge commit from fork
Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
2025-10-03 06:32:24 -07:00
BinbinandViktor Söderqvist 06207cfbd0 Minor fix for dual rdb channel connection conn error log (#2658)
This should be server.repl_rdb_transfer_s

Signed-off-by: Binbin <binloveplay1314@qq.com>
2025-10-01 16:04:34 +02:00
Zeroday BYTEandViktor Söderqvist a022b6e653 Fix unsigned difference expression compared to zero (#2101)
https://github.com/valkey-io/valkey/blob/8509a063425a674688d26d2b8c641fda00889b23/src/networking.c#L886-L886

Fix the issue need to ensure that the subtraction `prev->size -
prev->used` does not underflow. This can be achieved by explicitly
checking that `prev->used` is less than `prev->size` before performing
the subtraction. This approach avoids relying on unsigned arithmetic and
ensures the logic is clear and robust.

The specific changes are:
1. Replace the condition `prev->size - prev->used > 0` with `prev->used
< prev->size`.
2. This change ensures that the logic checks whether there is remaining
space in the buffer without risking underflow.

**References**
[INT02-C. Understand integer conversion
rules](https://wiki.sei.cmu.edu/confluence/display/c/INT02-C.+Understand+integer+conversion+rules)
[CWE-191](https://cwe.mitre.org/data/definitions/191.html)


---

Signed-off-by: Zeroday BYTE <github@zerodaysec.org>
2025-10-01 16:04:34 +02:00
7d3e31074a Fix accounting for dual channel RDB bytes in replication stats (#2602) (backport in #2616)
Backport of #2602. Resolves #2545.

---------

Signed-off-by: Sarthak Aggarwal <sarthagg@amazon.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-09-23 12:59:19 +02:00
Nikhil MangloreandViktor Söderqvist b0b4acb7fb Add release notes for 8.0.5 and update version file
Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Signed-off-by: Nikhil Manglore <nmanglor@amazon.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
BinbinandViktor Söderqvist 5dae072fc7 Fix pre-size hashtables per slot when reading RDB files (#2466)
When reading RDB files with information about the number of keys per cluster slot,
we need to create the dicts if they don't exist.

Currently, when processing RDB slot-info, our expand has no effect because the dict
does not exist (we initialize it only when we need it).

We also update kvstoreExpand to use the kvstoreDictExpand to make
sure there is only one code path. Also see #1199 for more details.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2025-08-22 15:30:45 +02:00
Ted LyngmoandViktor Söderqvist 8fc1403f7f Fix assumptions that pthread functions set errno (#2526)
pthread functions return the error instead of setting errno.

Fixes #2525

Signed-off-by: Ted Lyngmo <ted@lyncon.se>
2025-08-22 15:30:45 +02:00
Harkrishn PatroandViktor Söderqvist aba0aed27c Converge divergent shard-id persisted in nodes.conf to primary's shard id (#2174)
Fixes #2171

Handle divergent shard-id across primary and replica from nodes.conf and
reconcile all the nodes in the shard to the primary node's shard-id.

---------

Signed-off-by: Harkrishn Patro <harkrisp@amazon.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
BinbinandViktor Söderqvist 8b6c61555c Fix client tracking memory overhead calculation (#2360)
This should be + instread of *, otherwise it does not make any sense.
Otherwise we would have to calculate 20 more bytes for each prefix rax
node in 64 bits build.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
f00cd89bc9 Ensure empty error tables in scripts don't crash Valkey (#2229)
When calling the command `EVAL error{} 0`, Valkey crashes with the
following stack trace. This patch ensures we never leave the
`err_info.msg` field null when we fail to extract a proper error
message.

```
=== VALKEY BUG REPORT START: Cut & paste starting from here ===
2595901:M 18 Jun 2025 01:20:12.917 # valkey 8.1.2 crashed by signal: 11, si_code: 1
2595901:M 18 Jun 2025 01:20:12.917 # Accessing address: (nil)
2595901:M 18 Jun 2025 01:20:12.917 # Crashed running the instruction at: 0x726f8e57ed1d

------ STACK TRACE ------
EIP:
/usr/lib/libc.so.6(+0x16ed1d) [0x726f8e57ed1d]

2595905 bio_aof
/usr/lib/libc.so.6(+0x9de22) [0x726f8e4ade22]
/usr/lib/libc.so.6(+0x91fda) [0x726f8e4a1fda]
/usr/lib/libc.so.6(+0x9264c) [0x726f8e4a264c]
/usr/lib/libc.so.6(pthread_cond_wait+0x14e) [0x726f8e4a4d1e]
valkey-server *:6379(bioProcessBackgroundJobs+0x1b4) [0x6530abb46db4]
/usr/lib/libc.so.6(+0x957eb) [0x726f8e4a57eb]
/usr/lib/libc.so.6(+0x11918c) [0x726f8e52918c]

2595904 bio_close_file
/usr/lib/libc.so.6(+0x9de22) [0x726f8e4ade22]
/usr/lib/libc.so.6(+0x91fda) [0x726f8e4a1fda]
/usr/lib/libc.so.6(+0x9264c) [0x726f8e4a264c]
/usr/lib/libc.so.6(pthread_cond_wait+0x14e) [0x726f8e4a4d1e]
valkey-server *:6379(bioProcessBackgroundJobs+0x1b4) [0x6530abb46db4]
/usr/lib/libc.so.6(+0x957eb) [0x726f8e4a57eb]
/usr/lib/libc.so.6(+0x11918c) [0x726f8e52918c]

2595901 valkey-server *
/usr/lib/libc.so.6(+0x3def0) [0x726f8e44def0]
/usr/lib/libc.so.6(+0x16ed1d) [0x726f8e57ed1d]
valkey-server *:6379(sdscatfmt+0x894) [0x6530abaa24a4]
valkey-server *:6379(luaCallFunction+0x39a) [0x6530abbc66ea]
valkey-server *:6379(+0x1a0992) [0x6530abbc6992]
valkey-server *:6379(scriptingEngineCallFunction+0x98) [0x6530abbc1298]
valkey-server *:6379(+0x11ff55) [0x6530abb45f55]
valkey-server *:6379(call+0x174) [0x6530aba94454]
valkey-server *:6379(processCommand+0x93d) [0x6530aba958dd]
valkey-server *:6379(processCommandAndResetClient+0x21) [0x6530abaa9d11]
valkey-server *:6379(processInputBuffer+0xe3) [0x6530abaaee83]
valkey-server *:6379(readQueryFromClient+0x65) [0x6530abaaef55]
valkey-server *:6379(+0x18e31a) [0x6530abbb431a]
valkey-server *:6379(aeProcessEvents+0x24a) [0x6530aba790ca]
valkey-server *:6379(aeMain+0x2d) [0x6530aba7938d]
valkey-server *:6379(main+0x3f6) [0x6530aba6e7b6]
/usr/lib/libc.so.6(+0x276b5) [0x726f8e4376b5]
/usr/lib/libc.so.6(__libc_start_main+0x89) [0x726f8e437769]
valkey-server *:6379(_start+0x25) [0x6530aba70235]

2595906 bio_lazy_free
/usr/lib/libc.so.6(+0x9de22) [0x726f8e4ade22]
/usr/lib/libc.so.6(+0x91fda) [0x726f8e4a1fda]
/usr/lib/libc.so.6(+0x9264c) [0x726f8e4a264c]
/usr/lib/libc.so.6(pthread_cond_wait+0x14e) [0x726f8e4a4d1e]
valkey-server *:6379(bioProcessBackgroundJobs+0x1b4) [0x6530abb46db4]
/usr/lib/libc.so.6(+0x957eb) [0x726f8e4a57eb]
/usr/lib/libc.so.6(+0x11918c) [0x726f8e52918c]

4/4 expected stacktraces.

------ STACK TRACE DONE ------

------ REGISTERS ------
2595901:M 18 Jun 2025 01:20:12.920 #
RAX:0000000000000000 RBX:0000726f8dd35663
RCX:0000000000000000 RDX:0000000000000000
RDI:0000000000000000 RSI:0000000000000010
RBP:00007ffc2b821a80 RSP:00007ffc2b821938
R8 :000000000000000c R9 :00006530abc111b8
R10:0000000000000001 R11:0000000000000003
R12:00006530abc49adc R13:00006530abc111b7
R14:0000000000000001 R15:0000000000000001
RIP:0000726f8e57ed1d EFL:0000000000010283
CSGSFS:002b000000000033
2595901:M 18 Jun 2025 01:20:12.921 * hide-user-data-from-log is on, skip logging stack content to avoid spilling user data.

------ INFO OUTPUT ------
redis_version:7.2.4
server_name:valkey
valkey_version:8.1.2
valkey_release_stage:ga
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:38d65aa7b4148d2c
server_mode:standalone
os:Linux 6.14.6-arch1-1 x86_64
arch_bits:64
monotonic_clock:POSIX clock_gettime
multiplexing_api:epoll
gcc_version:15.1.1
process_id:2595901
process_supervised:no
run_id:a0b75f67a217a81142f17553028c010e86c1ee80
tcp_port:6379
server_time_usec:1750209612917634
uptime_in_seconds:16
uptime_in_days:0
hz:10
configured_hz:10
clients_hz:10
lru_clock:5379148
executable:/home/fusl/valkey-server
config_file:
io_threads_active:0
availability_zone:
listener0:name=tcp,bind=*,bind=-::*,port=6379

connected_clients:1
cluster_connections:0
maxclients:10000
client_recent_max_input_buffer:0
client_recent_max_output_buffer:0
blocked_clients:0
tracking_clients:0
pubsub_clients:0
watching_clients:0
clients_in_timeout_table:0
total_watched_keys:0
total_blocking_keys:0
total_blocking_keys_on_nokey:0
paused_reason:none
paused_actions:none
paused_timeout_milliseconds:0

used_memory:911824
used_memory_human:890.45K
used_memory_rss:15323136
used_memory_rss_human:14.61M
used_memory_peak:911824
used_memory_peak_human:890.45K
used_memory_peak_perc:100.29%
used_memory_overhead:892232
used_memory_startup:891824
used_memory_dataset:19592
used_memory_dataset_perc:97.96%
allocator_allocated:1845952
allocator_active:1986560
allocator_resident:6672384
allocator_muzzy:0
total_system_memory:67323842560
total_system_memory_human:62.70G
used_memory_lua:34816
used_memory_vm_eval:34816
used_memory_lua_human:34.00K
used_memory_scripts_eval:184
number_of_cached_scripts:1
number_of_functions:0
number_of_libraries:0
used_memory_vm_functions:33792
used_memory_vm_total:68608
used_memory_vm_total_human:67.00K
used_memory_functions:224
used_memory_scripts:408
used_memory_scripts_human:408B
maxmemory:0
maxmemory_human:0B
maxmemory_policy:noeviction
allocator_frag_ratio:1.00
allocator_frag_bytes:0
allocator_rss_ratio:3.36
allocator_rss_bytes:4685824
rss_overhead_ratio:2.30
rss_overhead_bytes:8650752
mem_fragmentation_ratio:17.18
mem_fragmentation_bytes:14431168
mem_not_counted_for_evict:0
mem_replication_backlog:0
mem_total_replication_buffers:0
mem_clients_slaves:0
mem_clients_normal:0
mem_cluster_links:0
mem_aof_buffer:0
mem_allocator:jemalloc-5.3.0
mem_overhead_db_hashtable_rehashing:0
active_defrag_running:0
lazyfree_pending_objects:0
lazyfreed_objects:0

loading:0
async_loading:0
current_cow_peak:0
current_cow_size:0
current_cow_size_age:0
current_fork_perc:0.00
current_save_keys_processed:0
current_save_keys_total:0
rdb_changes_since_last_save:0
rdb_bgsave_in_progress:0
rdb_last_save_time:1750209596
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:-1
rdb_current_bgsave_time_sec:-1
rdb_saves:0
rdb_last_cow_size:0
rdb_last_load_keys_expired:0
rdb_last_load_keys_loaded:0
aof_enabled:0
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:-1
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_rewrites:0
aof_rewrites_consecutive_failures:0
aof_last_write_status:ok
aof_last_cow_size:0
module_fork_in_progress:0
module_fork_last_cow_size:0

total_connections_received:1
total_commands_processed:0
instantaneous_ops_per_sec:0
total_net_input_bytes:34
total_net_output_bytes:0
total_net_repl_input_bytes:0
total_net_repl_output_bytes:0
instantaneous_input_kbps:0.00
instantaneous_output_kbps:0.00
instantaneous_input_repl_kbps:0.00
instantaneous_output_repl_kbps:0.00
rejected_connections:0
sync_full:0
sync_partial_ok:0
sync_partial_err:0
expired_keys:0
expired_stale_perc:0.00
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:0
evicted_keys:0
evicted_clients:0
evicted_scripts:0
total_eviction_exceeded_time:0
current_eviction_exceeded_time:0
keyspace_hits:0
keyspace_misses:0
pubsub_channels:0
pubsub_patterns:0
pubsubshard_channels:0
latest_fork_usec:0
total_forks:0
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
total_active_defrag_time:0
current_active_defrag_time:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:0
total_error_replies:0
dump_payload_sanitizations:0
total_reads_processed:1
total_writes_processed:0
io_threaded_reads_processed:0
io_threaded_writes_processed:0
io_threaded_freed_objects:0
io_threaded_accept_processed:0
io_threaded_poll_processed:0
io_threaded_total_prefetch_batches:0
io_threaded_total_prefetch_entries:0
client_query_buffer_limit_disconnections:0
client_output_buffer_limit_disconnections:0
reply_buffer_shrinks:0
reply_buffer_expands:0
eventloop_cycles:170
eventloop_duration_sum:17739
eventloop_duration_cmd_sum:0
instantaneous_eventloop_cycles_per_sec:9
instantaneous_eventloop_duration_usec:99
acl_access_denied_auth:0
acl_access_denied_cmd:0
acl_access_denied_key:0
acl_access_denied_channel:0

role:master
connected_slaves:0
replicas_waiting_psync:0
master_failover_state:no-failover
master_replid:d35a0bb7979f490a60174bb363524431d7eb2428
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:0
second_repl_offset:-1
repl_backlog_active:0
repl_backlog_size:10485760
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0

used_cpu_sys:0.012543
used_cpu_user:0.016853
used_cpu_sys_children:0.000000
used_cpu_user_children:0.000000
used_cpu_sys_main_thread:0.012440
used_cpu_user_main_thread:0.016714

cluster_enabled:0

------ CLIENT LIST OUTPUT ------
id=2 addr=127.0.0.1:41372 laddr=127.0.0.1:6379 fd=10 name=*redacted* age=0 idle=0 flags=N capa= db=0 sub=0 psub=0 ssub=0 multi=-1 watch=0 qbuf=0 qbuf-free=0 argv-mem=12 multi-mem=0 rbs=16384 rbp=16384 obl=0 oll=0 omem=0 tot-mem=17060 events=r cmd=eval user=*redacted* redir=-1 resp=2 lib-name= lib-ver= tot-net-in=34 tot-net-out=0 tot-cmds=0

------ CURRENT CLIENT INFO ------
id=2 addr=127.0.0.1:41372 laddr=127.0.0.1:6379 fd=10 name=*redacted* age=0 idle=0 flags=N capa= db=0 sub=0 psub=0 ssub=0 multi=-1 watch=0 qbuf=0 qbuf-free=0 argv-mem=12 multi-mem=0 rbs=16384 rbp=16384 obl=0 oll=0 omem=0 tot-mem=17060 events=r cmd=eval user=*redacted* redir=-1 resp=2 lib-name= lib-ver= tot-net-in=34 tot-net-out=0 tot-cmds=0
argc: 3
argv[0]: "eval"
argv[1]: 7 bytes
argv[2]: 1 bytes

------ EXECUTING CLIENT INFO ------
id=2 addr=127.0.0.1:41372 laddr=127.0.0.1:6379 fd=10 name=*redacted* age=0 idle=0 flags=N capa= db=0 sub=0 psub=0 ssub=0 multi=-1 watch=0 qbuf=0 qbuf-free=0 argv-mem=12 multi-mem=0 rbs=16384 rbp=16384 obl=0 oll=0 omem=0 tot-mem=17060 events=r cmd=eval user=*redacted* redir=-1 resp=2 lib-name= lib-ver= tot-net-in=34 tot-net-out=0 tot-cmds=0
argc: 3
argv[0]: "eval"
argv[1]: 7 bytes
argv[2]: 1 bytes

------ MODULES INFO OUTPUT ------

------ CONFIG DEBUG OUTPUT ------
repl-diskless-load disabled
debug-context ""
sanitize-dump-payload no
lazyfree-lazy-user-del yes
lazyfree-lazy-server-del yes
import-mode no
lazyfree-lazy-user-flush yes
list-compress-depth 0
dual-channel-replication-enabled no
repl-diskless-sync yes
activedefrag no
lazyfree-lazy-expire yes
io-threads 1
replica-read-only yes
client-query-buffer-limit 1gb
slave-read-only yes
lazyfree-lazy-eviction yes
proto-max-bulk-len 512mb

------ FAST MEMORY TEST ------
2595901:M 18 Jun 2025 01:20:12.921 # Bio worker thread #0 terminated
2595901:M 18 Jun 2025 01:20:12.921 # Bio worker thread #1 terminated
2595901:M 18 Jun 2025 01:20:12.921 # Bio worker thread #2 terminated
*** Preparing to test memory region 6530abce2000 (212992 bytes)
*** Preparing to test memory region 726f8af7f000 (2621440 bytes)
*** Preparing to test memory region 726f8b200000 (8388608 bytes)
*** Preparing to test memory region 726f8ba00000 (4194304 bytes)
*** Preparing to test memory region 726f8bffe000 (8388608 bytes)
*** Preparing to test memory region 726f8c7ff000 (8388608 bytes)
*** Preparing to test memory region 726f8d000000 (8388608 bytes)
*** Preparing to test memory region 726f8dc00000 (4194304 bytes)
*** Preparing to test memory region 726f8e290000 (16384 bytes)
*** Preparing to test memory region 726f8e3d2000 (20480 bytes)
*** Preparing to test memory region 726f8e5f8000 (32768 bytes)
*** Preparing to test memory region 726f8eb58000 (12288 bytes)
*** Preparing to test memory region 726f8eb5c000 (16384 bytes)
*** Preparing to test memory region 726f8ed63000 (4096 bytes)
*** Preparing to test memory region 726f8eef2000 (397312 bytes)
*** Preparing to test memory region 726f8efc7000 (4096 bytes)
.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O
Fast memory test PASSED, however your memory can still be broken. Please run a memory test for several hours if possible.

------ DUMPING CODE AROUND EIP ------
Symbol: (null) (base: (nil))
Module: /usr/lib/libc.so.6 (base 0x726f8e410000)
$ xxd -r -p /tmp/dump.hex /tmp/dump.bin
$ objdump --adjust-vma=(nil) -D -b binary -m i386:x86-64 /tmp/dump.bin
------

=== VALKEY BUG REPORT END. Make sure to include from START to END. ===
```

---------

Signed-off-by: Fusl <fusl@meo.ws>
Signed-off-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Viktor Söderqvist ac490dcf77 Fix defrag timeouts in 8.0
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Ran ShidlansikandViktor Söderqvist 03b0bd8fa6 update build-debian-old to use Bullseye instead of EOL buster (#2345)
Signed-off-by: Ran Shidlansik <ranshid@amazon.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Viktor Söderqvist e8723d4fdc Disable clang-format in 8.0 to simplify backporting
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Sarthak AggarwalandViktor Söderqvist 70ca43bc00 Try to stabilize aof test (#2399)
Based on @enjoy-binbin's suggestion on #1611, I made the change to find
the available port. The test has been passing in the daily tests in my
local repo.

Resolved #1611

Signed-off-by: Sarthak Aggarwal <sarthagg@amazon.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Viktor Söderqvist 8ccae197ba Test coverage for ECHO for reply schema validation (#1549)
After #1545 disabled some tests for reply schema validation, we now have
another issue that ECHO is not covered.

```
WARNING! The following commands were not hit at all:
  echo
ERROR! at least one command was not hit by the tests
```

This patch adds a test case for ECHO in the unit/other test suite. I
haven't checked if there are more commands that aren't covered.

Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
BinbinandViktor Söderqvist be5ba25244 Avoid timing issue in diskless-load-swapdb test (#1077)
Since we paused the primary node earlier, the replica may enter
cluster down due to primary node pfail. Here set allow read to
prevent subsequent read errors.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Madelyn OlsonandViktor Söderqvist 0b48ed5a58 Fix undefined behavior defined by ASAN (#1451)
Asan now supports making sure you are passing in the correct pointer
type, which seems useful but we can't support it since we pass in an
incorrect pointer in several places. This is most commonly done with
generic free functions, where we simply cast it to the correct type.

It's not a lot of code to clean up, so it seems appropriate to cleanup
instead of disabling the check.

---------

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
515a0b8388 Disable lazy free in defrag test to fix 32bit daily failure (#1370)
Signed-off-by: Zvi Schneider <zvi.schneider22@gmail.com>
Co-authored-by: Zvi Schneider <zvi.schneider22@gmail.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Viktor Söderqvist 24d27578e5 Skip CLI tests with reply schema validation (#1545)
The commands used in valkey-cli tests are not important the reply schema
validation. Skip them to avoid the problem if tests hanging. This has
failed lately in the daily job:

```
[TIMEOUT]: clients state report follows.
sock55fedcc19be0 => (IN PROGRESS) valkey-cli pubsub mode with single standard channel subscription
Killing still running Valkey server 33357
```

These test cases use a special valkey-cli command `:get pubsub` command,
which is an internal command to valkey-cli rather than a Valkey server
command. This command hangs when compiled with with logreqres enabled.
Easy solution is to skip the tests in this setup.

The test cases were introduced in #1432.

Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
BinbinandViktor Söderqvist b64c60dba8 Fix the election was reset wrongly before failover epoch was obtained (#1339)
After #1009, we will reset the election when we received
a claim with an equal or higher epoch since a node can win
an election in the past.

But we need to consider the time before the node actually
obtains the failover_auth_epoch. The failover_auth_epoch
default is 0, so before the node actually get the failover
epoch, we might wrongly reset the election.

This is probably harmless, but will produce misleading log
output and may delay election by a cron cycle or beforesleep.
Now we will only reset the election when a node is actually
obtains the failover epoch.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
BinbinandViktor Söderqvist 9481a5b466 Fix empty primary may have dirty slots data due to bad migration (#1285)
If we become an empty primary for some reason, we still need to
check if we need to delete dirty slots, because we may have dirty
slots data left over from a bad migration. Like the target node forcibly
executes CLUSTER SETSLOT NODE to take over the slot without
performing key migration.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Vitaly ArbuzovandViktor Söderqvist 71f4ce8ce1 Add retries on CLUSTERDOWN in tests and fix sanitizer error
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Vitaly ArbuzovandViktor Söderqvist 03315727a1 fix uninitialized variable warning
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Vitaly ArbuzovandViktor Söderqvist 6e3d2dd745 formatting
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Vitaly ArbuzovandViktor Söderqvist 0477c0544f Fixes
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
BinbinandViktor Söderqvist 64c86859da Make manual failover reset the on-going election to promote failover (#1274)
If a manual failover got timed out, like the election don't get the
enough votes, since we have a auth_timeout and a auth_retry_time, a
new manual failover will not be able to proceed on the replica side.

Like if we initiate a new manual failover after a election timed out,
we will pause the primary, but on the replica side, due to retry_time,
replica does not trigger the new election and the manual failover will
eventually time out.

In this case, if we initiate manual failover again and there is an
ongoing election, we will reset it so that the replica can initiate
a new election at the manual failover's request.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
BinbinandViktor Söderqvist 4b7e29d75c Trigger the election as soon as possible when doing a forced manual failover (#1067)
In CLUSTER FAILOVER FORCE case, we will set mf_can_start to
1 and wait for a cron to trigger the election. We can also set a
CLUSTER_TODO_HANDLE_MANUALFAILOVER flag so that we
can start the election as soon as possible instead of waiting for
the cron, so that we won't have a 100ms delay (clusterCron).

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
BinbinandViktor Söderqvist 501b09c71f Minor log fixes when failover auth denied due to slot epoch (#1341)
The old reqEpoch mainly refers to requestCurrentEpoch, see:
```
    if (requestCurrentEpoch < server.cluster->currentEpoch) {
        serverLog(LL_WARNING, "Failover auth denied to %.40s (%s): reqEpoch (%llu) < curEpoch(%llu)", node->name,
                  node->human_nodename, (unsigned long long)requestCurrentEpoch,
                  (unsigned long long)server.cluster->currentEpoch);
        return;
    }
```

And in here we refer to requestConfigEpoch, it's a bit misleading,
so change it to reqConfigEpoch to make it clear.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Madelyn OlsonandViktor Söderqvist c9031cf46a Add a filter option to drop all cluster packets (#1252)
A minor debugging change that helped in the investigation of
https://github.com/valkey-io/valkey/issues/1251. Basically there are
some edge cases where we want to fully isolate a note from receiving
packets, but can't suspend the process because we need it to continue
sending outbound traffic. So, added a filter for that.

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
BinbinandViktor Söderqvist 295c2efc01 Add packet-drop to fix the new flaky failover test (#2196)
The new test was added in #2178, obviously there may be
pending reads in the connection, so there may be a race
in the DROP-CLUSTER-PACKET-FILTER part causing the test
to fail. Add CLOSE-CLUSTER-LINK-ON-PACKET-DROP to ensure
that the replica does not process the packet.

Signed-off-by: Binbin <binloveplay1314@qq.com>
(cherry picked from commit b9ffae9e39)
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
BinbinandViktor Söderqvist 89dcc11ce2 Fix cluster myself CLUSTER SLOTS/NODES wrong port after updating port/tls-port (#2186)
When modifying port or tls-port through config set, we need to call
clusterUpdateMyselfAnnouncedPorts to update myself's port, otherwise
CLUSTER SLOTS/NODES will be old information from myself's perspective.

In addition, in some places, such as clusterUpdateMyselfAnnouncedPorts
and clusterUpdateMyselfIp, beforeSleep save is added so that the
new ip info can be updated to nodes.conf.

Remove clearCachedClusterSlotsResponse in updateClusterAnnouncedPort
since now we add beforeSleep save in clusterUpdateMyselfAnnouncedPorts,
and it will call clearCachedClusterSlotsResponse.

Signed-off-by: Binbin <binloveplay1314@qq.com>
(cherry picked from commit 21ad38a616)
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
BinbinandViktor Söderqvist 8d5ec04ff2 Fix replica can't finish failover when config epoch is outdated (#2178)
When the primary changes the config epoch and then down immediately,
the replica may not update the config epoch in time. Although we will
broadcast the change in cluster (see #1813), there may be a race in
the network or in the code. In this case, the replica will never finish
the failover since other primaries will refuse to vote because the
replica's slot config epoch is old.

We need a way to allow the replica can finish the failover in this case.

When the primary refuses to vote because the replica's config epoch is
less than the dead primary's config epoch, it can send an UPDATE packet
to the replica to inform the replica about the dead primary. The UPDATE
message contains information about the dead primary's config epoch and
owned slots. The failover will time out, but later the replica can try
again with the updated config epoch and it can succeed.

Fixes #2169.

---------

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Harkrishn Patro <bunty.hari@gmail.com>
Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Co-authored-by: Harkrishn Patro <bunty.hari@gmail.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
(cherry picked from commit 3739d4c96b)
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Madelyn OlsonandViktor Söderqvist 3c30e98891 Incorporate Redis CVE for CVE-2025-27151 (#2146)
Resolves https://github.com/valkey-io/valkey/issues/2145

Incorporate the CVE patch that was sent to us by Redis Ltd.

---------

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: Ping Xie <pingxie@outlook.com>
(cherry picked from commit 36c31c1d40)
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Madelyn OlsonandViktor Söderqvist 8bb805419f Correctly cast the extension lengths (#2144)
Correctly use a 32 bit integer for accumulating the length of ping
extensions.

The current code may accidentally truncate the length of an
extension that is greater than 64kb and fail the validation check. We
don't currently emit any extensions that are this length, but if we were
to do so in the future we might have issues with older nodes (without
this fix) will silently drop packets from newer nodes. We should
backport this to all versions.

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
(cherry picked from commit 2668ca6534)
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Viktor Söderqvist dbb98f0668 Detect SSL_new() returning NULL in outgoing connections (#2140)
When creating an outgoing TLS connection, we don't check if `SSL_new()`
returned NULL.

Without this patch, the check was done only for incoming connections in
`connCreateAcceptedTLS()`. This patch moves the check to
`createTLSConnection()` which is used both for incoming and outgoing
connections.

This check makes sure we fail the connection before going any further,
e.g. when `connCreate()` is followed by `connConnect()`, the latter
returns `C_ERR` which is commonly detected where outgoing connections
are established, such where a replica connects to a primary.

```c
int connectWithPrimary(void) {
    server.repl_transfer_s = connCreate(connTypeOfReplication());
    if (connConnect(server.repl_transfer_s, server.primary_host, server.primary_port, server.bind_source_addr,
                    server.repl_mptcp, syncWithPrimary) == C_ERR) {
        serverLog(LL_WARNING, "Unable to connect to PRIMARY: %s", connGetLastError(server.repl_transfer_s));
        connClose(server.repl_transfer_s);
        server.repl_transfer_s = NULL;
        return C_ERR;
    }
```

For a more thorough explanation, see
https://github.com/valkey-io/valkey/issues/1939#issuecomment-2912177877.

Might fix #1939.

Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
(cherry picked from commit d9d61283c1)
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
BinbinandViktor Söderqvist 0777a5c256 CLIENT UNBLOCK should't be able to unpause paused clients (#2117)
When a client is blocked by something like `CLIENT PAUSE`, we should not
allow `CLIENT UNBLOCK timeout` to unblock it, since some blocking types
does not has the timeout callback, it will trigger a panic in the core,
people should use `CLIENT UNPAUSE` to unblock it.

Also using `CLIENT UNBLOCK error` is not right, it will return a UNBLOCKED
error to the command, people don't expect a `SET` command to get an error.

So in this commit, in these cases, we will return 0 to `CLIENT UNBLOCK`
to indicate the unblock is fail. The reason is that we assume that if
a command doesn't expect to be timedout, it also doesn't expect to be
unblocked by `CLIENT UNBLOCK`.

The old behavior of the following command will trigger panic in timeout
and get UNBLOCKED error in error. Under the new behavior, client unblock
will get the result of 0.
```
client 1> client pause 100000 write
client 2> set x x

client 1> client unblock 2 timeout
or
client 1> client unblock 2 error
```

Potentially breaking change, previously allowed `CLIENT UNBLOCK error`.
Fixes #2111.

Signed-off-by: Binbin <binloveplay1314@qq.com>
(cherry picked from commit 4c6f9dfd15)
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Jacob MurphyandViktor Söderqvist 37fddb0f5f Free module context even if there was no content written in auxsave2 (#2132)
When a module saves to the Aux metadata with aux_save2 callback, the
module context is not freed if the module didn't save anything in the
callback.

https://github.com/valkey-io/valkey/blob/8.1.1/src/rdb.c#L1277-L1284.
Note that we return 0, however we should be doing the cleanup done on
https://github.com/valkey-io/valkey/blob/8.1.1/src/rdb.c#L1300-L1303
still.

Fixes #2125

Signed-off-by: Jacob Murphy <jkmurphy@google.com>
(cherry picked from commit af9223ff9f)
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
uriyageandViktor Söderqvist 893dd278e2 Fix bad slot used in sharded pubsub unsubscribe (#2137)
This commit fixes two issues in `pubsubUnsubscribeChannel` that could
lead to memory corruption:

1. When calculating the slot for a channel, we were using getKeySlot()
which might use the current_client's slot if available. This is
problematic when a client kills another client (e.g., via CLIENT KILL
command) as the slot won't match the channel's actual slot.

2. The `found` variable was not initialized to `NULL`, causing the
serverAssert to potentially pass incorrectly when the hashtable lookup
failed, leading to memory corruption in subsequent operations.

The fix:
- Calculate the slot directly from the channel name using keyHashSlot()
instead of relying on the current client's slot
- Initialize 'found' to NULL

Added a test case that reproduces the issue by having one client kill
another client that is subscribed to a sharded pubsub channel during a
transaction.

Crash log (After initializing the variable 'found' to null, without
initialization, memory corruption could occur):
```
VALKEY BUG REPORT START: Cut & paste starting from here ===
59707:M 24 May 2025 23:10:40.429 # === ASSERTION FAILED CLIENT CONTEXT ===
59707:M 24 May 2025 23:10:40.429 # client->flags = 108086391057154048
59707:M 24 May 2025 23:10:40.429 # client->conn = fd=11
59707:M 24 May 2025 23:10:40.429 # client->argc = 0
59707:M 24 May 2025 23:10:40.429 # === RECURSIVE ASSERTION FAILED ===
59707:M 24 May 2025 23:10:40.429 # ==> pubsub.c:348 'found' is not true

------ STACK TRACE ------

Backtrace:
0   valkey-server                       0x0000000104974054 _serverAssertWithInfo + 112
1   valkey-server                       0x000000010496c7fc pubsubUnsubscribeChannel + 268
2   valkey-server                       0x000000010496cea0 pubsubUnsubscribeAllChannelsInternal + 216
3   valkey-server                       0x000000010496c2e0 pubsubUnsubscribeShardAllChannels + 76
4   valkey-server                       0x000000010496c1d4 freeClientPubSubData + 60
5   valkey-server                       0x00000001048f3cbc freeClient + 792
6   valkey-server                       0x0000000104900870 clientKillCommand + 356
7   valkey-server                       0x00000001048d1790 call + 428
8   valkey-server                       0x000000010496ef4c execCommand + 872
9   valkey-server                       0x00000001048d1790 call + 428
10  valkey-server                       0x00000001048d3a44 processCommand + 5056
11  valkey-server                       0x00000001048fdc20 processCommandAndResetClient + 64
12  valkey-server                       0x00000001048fdeac processInputBuffer + 276
13  valkey-server                       0x00000001048f2ff0 readQueryFromClient + 148
14  valkey-server                       0x0000000104a182e8 callHandler + 60
15  valkey-server                       0x0000000104a1731c connSocketEventHandler + 488
16  valkey-server                       0x00000001048b5e80 aeProcessEvents + 820
17  valkey-server                       0x00000001048b6598 aeMain + 64
18  valkey-server                       0x00000001048dcecc main + 4084
19  dyld                                0x0000000186b34274 start + 2840
````

---------

Signed-off-by: Uri Yagelnik <uriy@amazon.com>
(cherry picked from commit 91b2dcc233)
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Ran ShidlansikandViktor Söderqvist db450a6775 Only mark the client reprocessing flag when unblocked on keys (#2109)
When we refactored the blocking framework we introduced the client
reprocessing infrastructure. In cases the client was blocked on keys, it
will attempt to reprocess the command. One challenge was to keep track
of the command timeout, since we are reprocessing and do not want to
re-register the client with a fresh timeout each time. The solution was
to consider the client reprocessing flag when the client is
blockedOnKeys:

```
if (!c->flag.reprocessing_command) {
        /* If the client is re-processing the command, we do not set the timeout
         * because we need to retain the client's original timeout. */
        c->bstate->timeout = timeout;
    }
```

However, this introduced a new issue. There are cases where the client
will consecutive blocking of different types for example:
```
CLIENT PAUSE 10000 ALL
BZPOPMAX zset 1
```
would have the client blocked on the zset endlessly if nothing will be
written to it.

**Credits to @uriyage for locating this with his fuzzer testing**

The suggested solution is to only flag the client when it is
specifically unblocked on keys.

---------

Signed-off-by: Ran Shidlansik <ranshid@amazon.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
(cherry picked from commit c9ec3d63d6)
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-08-22 15:30:45 +02:00
Ran Shidlansik 1a035ce30e Release notes for 8.0.4
Signed-off-by: Ran Shidlansik <ranshid@amazon.com>
2025-07-07 10:22:40 +03:00
Ran Shidlansik eef11d8c76 retry accept on transient errors (CVE-2025-48367) (#2315)
Signed-off-by: Ran Shidlansik <ranshid@amazon.com>
2025-07-07 10:22:40 +03:00
Madelyn OlsonandRan Shidlansik c5655a5c68 Apply fixed for CVE-2025-32023 (#2314)
Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
2025-07-07 10:22:40 +03:00
Jacob MurphyandGitHub 0453426f87 Release notes for 8.0.3 (#1998)
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 13:41:26 -07:00
uriyageandGitHub d297b5bba7 Add output buffer limiting for unauthenticated clients (#1993)
This commit introduces a mechanism to track client authentication state
with a new `ever_authenticated` flag. It refactors client authentication
handling by adding a `clientSetUser` function that properly sets both
the authenticated and `ever_authenticated` flags.

The implementation limits output buffer size for clients that have never
been authenticated.

Added tests to verify the output buffer limiting behavior for
unauthenticated clients.

---------

Signed-off-by: Uri Yagelnik <uriy@amazon.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-04-23 22:30:56 +02:00
Madelyn OlsonandViktor Söderqvist 486ee3d11f Remove readability refactor for failover auth to fix clang warning (#1481)
As part of #1463, I made a small refactor between the PR and the daily
test I submitted to try to improve readability by adding a function to
abstract the extraction of the message types. However, that change
apparently caused GCC to throw another warning, so reverting the
abstraction on just one line.

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
Harkrishn PatroandViktor Söderqvist 8a6e68b0bc Update upload artifacts to v4 (#1539)
Fixes #1538

Signed-off-by: Harkrishn Patro <harkrisp@amazon.com>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
Vitah LinandViktor Söderqvist f9e41c182f Fix CI latest fedora: install awk and tcl8, build tcltls from source (#1965)
Fix two problems in fedora CI jobs:

1. Install awk where missing. It's required for building jemalloc.
2. Fix problems with TCL, required for running the tests. Fedora comes
   with TCL 9 by default, but the TLS package 'tcltls' isn't built for TCL 9.
   Install 'tcl8' and build 'tcltls' from source.

---------

Signed-off-by: vitah <vitahlin@gmail.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
Harkrishn PatroandViktor Söderqvist 16b22a7cd2 Avoid shard id update of replica if not matching with primary shard id (#573)
During cluster setup, the shard id gets established through extensions data propagation and if the engine crashes/restarts while the reconciliation of shard id is in progress, there is a possibility of corrupted config file with temporary shard id stored in the cluster. With this fix, a replica sharing a temporary shard id is ignored and allows the cluster bus to converge for only one shard id for primary and it's replicas.

Signed-off-by: Harkrishn Patro <harkrisp@amazon.com>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
nesty92andViktor Söderqvist 1a6dc2890d Fix incorrect lag reported in XINFO GROUPS (#1952)
When a tombstone is created after the last_id of a consumer group.

The consumer group lag is reported in the reply of `XINFO GROUPS mystream`.

Close: #1951

Signed-off-by: Ernesto Alejandro Santana Hidalgo <ernesto.alejandrosantana@gmail.com>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
Jacob MurphyandViktor Söderqvist 5b82acd40d Fix flaky shutdown tests caused by timing issue (#1960)
This started failing after
https://github.com/valkey-io/valkey/commit/57cd9a51d9c7d8cf9c4551bd1f2af79a0f279eeb,
since processing of SHUTDOWN might happen after the CLIENT INFO is
retrieved. Now we use wait_for_condition to make sure CLIENT INFO is
retried in the case that SHUTDOWN is not processed yet.

Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
BinbinandViktor Söderqvist db1f491be4 Ignore stale gossip packets that arrive out of order (#1777)
There is a failure in the daily:
```
=== ASSERTION FAILED ===
==> cluster_legacy.c:6588 'primary->replicaof == ((void *)0)' is not true
```

This is the logs:
```
- i am fd4318562665b4490ccc86e7f7988017cf960371 and myself become a replica,
- 63c0167232dae95cdcc0a1568cd5368ac3b99f5 is the new primary
27867:M 24 Feb 2025 00:19:11.011 * Failover auth granted to 763c0167232dae95cdcc0a1568cd5368ac3b99f5 () for epoch 9
27867:M 24 Feb 2025 00:19:11.039 * Configuration change detected. Reconfiguring myself as a replica of node 763c0167232dae95cdcc0a1568cd5368ac3b99f5 () in shard c5f6b2a9c74cabd4d1e54d1130dc9cb9419bf76f
27867:S 24 Feb 2025 00:19:11.039 * Before turning into a replica, using my own primary parameters to synthesize a cached primary: I may be able to synchronize with the new primary with just a partial transfer.
27867:S 24 Feb 2025 00:19:11.039 * Connecting to PRIMARY 127.0.0.1:23654
27867:S 24 Feb 2025 00:19:11.039 * PRIMARY <-> REPLICA sync started

- in here myself got an stale message, but we still process the packet and cause this issue
27867:S 24 Feb 2025 00:19:11.040 * Ignore stale message from 763c0167232dae95cdcc0a1568cd5368ac3b99f5 () in shard c5f6b2a9c74cabd4d1e54d1130dc9cb9419bf76f; gossip config epoch: 8, current config epoch: 9
27867:S 24 Feb 2025 00:19:11.040 * Node 763c0167232dae95cdcc0a1568cd5368ac3b99f5 () is now a replica of node fd4318562665b4490ccc86e7f7988017cf960371 () in shard c5f6b2a9c74cabd4d1e54d1130dc9cb9419bf76f
```

We can see myself got a stale message, but we still process it, and
changed
the role and cause a primary replica chain loop.

The reason is that, this text is copy from
https://github.com/valkey-io/valkey/pull/651.

In some rare case, slot config updates (via either PING/PONG or UPDATE)
can be delivered out of order as illustrated below:
```
1. To keep the discussion simple, let's assume we have 2 shards, shard a
   and shard b. Let's also assume there are two slots in total with shard
   a owning slot 1 and shard b owning slot 2.
2. Shard a has two nodes: primary A and replica A*; shard b has primary
   B and replica B*.
3. A manual failover was initiated on A* and A* just wins the election.
4. A* announces to the world that it now owns slot 1 using PING messages.
   These PING messages are queued in the outgoing buffer to every other
   node in the cluster, namely, A, B, and B*.
5. Keep in mind that there is no ordering in the delivery of these PING
   messages. For the stale PING message to appear, we need the following
   events in the exact order as they are laid out.
a. An old PING message before A* becomes the new primary is still queued
   in A*'s outgoing buffer to A. This later becomes the stale message,
   which says A* is a replica of A. It is followed by A*'s election
   winning announcement PING message.
b. B or B* processes A's election winning announcement PING message
   and sets slots[1]=A*.
c. A sends a PING message to B (or B*). Since A hasn't learnt that A*
   wins the election, it claims that it owns slot 1 but with a lower
   epoch than B has on slot 1. This leads to B sending an UPDATE to
   A directly saying A* is the new owner of slot 1 with a higher epoch.
d. A receives the UPDATE from B and executes clusterUpdateSlotsConfigWith.
   A now realizes that it is a replica of A* hence setting myself->replicaof
   to A*.
e. Finally, the pre-failover PING message queued up in A*'s outgoing
   buffer to A is delivered and processed, out of order though, to A.
f. This stale PING message creates the replication loop
```

Closes #1015.

---------

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
Jacob MurphyandViktor Söderqvist 0a155596f2 Fix panic in primary when blocking shutdown after previous block with timeout (#1948)
When previous timeout isn't zeroed out, shutdown will use the previously
executed blocking commands timeout. Since shutdown is not expected to
have timeouts, this causes a serverPanic.

Without this fix, this scenario leads to a server panic:

1. Call any command that uses blockClient with a timeout (this could be
BLPOP or something that blocks implicitly like CLUSTER SETSLOT)
2. On the same client, call SHUTDOWN with some pending replication data,
which will trigger shutdown to be blocked while we try to catch the
replica up

Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
Katie HollyandViktor Söderqvist ae110706ab Fix cluster slot stats assertion during promotion of replica (#1950)
This affects only the configuration `cluster-slot-stats-enabled yes`.

For the first command a newly promoted primary receives, we send a
`SELECT` command to all replicas via `replicationFeedReplicas`. This
calls a chain of functions (`feedReplicationBufferWithObject` ->
`feedReplicationBuffer` ->
`clusterSlotStatsIncrNetworkBytesOutForReplication` ->
`clusterSlotStatsUpdateNetworkBytesOutForReplication`) that increments
the per-slot `network_bytes_out` by `len * listLength(server.replicas)`.
However, if the new primary receives a command before any replicas
connected yet, `server.replicas` is still empty, causing the increment
to add zero.

Right after, we call `clusterSlotStatsDecrNetworkBytesOutForReplication`
to remove the `SELECT` overhead from the per-slot stats. The assert in
`clusterSlotStatsUpdateNetworkBytesOutForReplication` was using
(basically) `>= -len` instead of `>= -(len *
listLength(server.replicas))`, so it could fail if the previous
increment was zero because we never included the `server.replicas`
length in the assertion.

This patch makes the addition and subtraction use the same computed
value (`rlen = len * listLength(server.replicas)`) in both the stats
update and the assert, ensuring the logic remains consistent when
replicas are promoted.

Fixes #1912

Signed-off-by: Fusl <fusl@meo.ws>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
Arthur LeeandViktor Söderqvist bcf1bd30bf fix: add samples to stream object consumer trees (#1825)
Use sampling when calculating the memory usage of consumers as part of
streams in MEMORY USAGE and MEMORY STATS.

Without this fix, all consumers are traversed, which is slow if there
are very many consumers.

Fixes #1824

Signed-off-by: arthur.lee <liziang.arthur@bytedance.com>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
82b3206bea Fix RANDOMKEY infinite loop during CLIENT PAUSE (#1850)
When the `client pause write` is set and all the keys in the server
are expired keys, executing the `randomkey` command will lead to an
infinite loop.

The reason is that expired keys are not deleted in this case. Limit
the number of tries and return an expired key after the max number
tries in this case.

Closes #1848.

---------

Signed-off-by: li-benson <1260437731@qq.com>
Signed-off-by: youngmore <youngmore1024@outlook.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Co-authored-by: youngmore <youngmore1024@outlook.com>
Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
0ba88a4919 Fix ACL LOAD crash on replica since the primary client don't has a user (#1842)
Check for NULL before accessing users of a client during the `ACL LOAD`
command.
This prevents the command causing a segmentation fault in the replica.

Fixes #1832.

---------

Signed-off-by: Bogdan Petre <bogdan.petre@aiven.io>
Signed-off-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
BinbinandViktor Söderqvist 41670c1678 Fix temp file leak druing replication error handling (#1721)
Before actually entering REPL_STATE_TRANSFER, we usually have
some other things to do, such as registering the ae handler, etc.
If an error occurs at this time, we may leak the previously opened
temp file.

This commit adds a new cleanupTransferResources function to do the
cleanup, avoiding code duplication.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
338f9161c8 Fix error "SSL routines::bad length" when connTLSWrite is called second time with smaller buffer (#1737)
Issue described in #1135

When call to `connTLSWrite` fails on next attempt `connTLSWritev` should
provide buffer with at least same amount of bytes,
but if first call was with buffer smaller than
`NET_MAX_WRITES_PER_EVENT` then buffer gets larger and exceed
`NET_MAX_WRITES_PER_EVENT`, `connTLSWritev` will not combine `iov` into
one buffer resulting into chance that first element of `iov` is smaller
than last failed call to `connTLSWrite` and causing `SSL routines::bad
length`.

This change force combining `iov`s into one buffer if first element of
`iov` is smaller than last failed write by `connTLSWrite`

---------

Signed-off-by: Marek Zoremba <marek@janeasystems.com>
Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Signed-off-by: ranshid <ranshid@amazon.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: ranshid <ranshid@amazon.com>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
6afce8157d Fix raxRemove crash at memcpy() due to key size exceeds max Rax size (#1722)
Fix raxRemove crash at memcpy() (line 1181) due to key size exceeds
`RAX_NODE_MAX_SIZE`. Note that this could happen when key size was more
than 512MB if we allow it by increasing the default
`proto-max-bulk-len`. The crash could happen when we recompress the rax
after removing a key due to expiry or DEL while memcpy() merge the key
that exceed 512MB limit. While the counting phase has the size check,
the actual compress logic is missing it which lead to this crash.

---------

Signed-off-by: Ram Prasad Voleti <ramvolet@amazon.com>
Co-authored-by: Ram Prasad Voleti <ramvolet@amazon.com>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
BinbinandViktor Söderqvist 2a0767b6c6 Fix module LatencyAddSample still work when latency-monitor-threshold is 0 (#1541)
When latency-monitor-threshold is set to 0, it means the latency monitor
is disabled, and in VM_LatencyAddSample, we wrote the condition
incorrectly, causing us to record latency when latency was turned off.

This bug was introduced in the very first day, see 144277c, it was merged
in 2019.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
Madelyn OlsonandViktor Söderqvist c1b986d854 Resolve bounds checks on cluster_legacy.c (#1463)
We are getting a number of errors like:
```
array subscript ‘clusterMsg[0]’ is partly outside array bounds of ‘unsigned char[2272]’
```

Which is basically GCC telling us that we have an object which is longer
than the underlying storage of the allocation. We actually do this a
lot, but GCC is generally not aware of how big the underlying allocation
is, so it doesn't throw this error. We are specifically getting this
error because the msgBlock can be of variable length depending on the
type of message, but GCC assumes it's the longest one possible. The
solution I went with here was make the message type optional, so that it
wasn't included in the size. I think this also makes some sense, since
it's really just a helper for us to easily cast the object around.

I considered disabling this error, but it is generally pretty useful
since it can catch real issues. Another solution would be to
over-allocate to the largest possible object, which could hurt
performance as we initialize it to zero.

Results:
https://github.com/madolson/valkey/actions/runs/12423414811/job/34686899884

This is a slightly cleaned up version of
https://github.com/valkey-io/valkey/pull/1439. I thought I had another
strategy but alas, it didn't work out.

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
BinbinandViktor Söderqvist 05cc6c3e33 Use fake client flag to replace not conn check (#1198)
The fake client flag was introduced in #1063,
we want this to replace all !conn fake client checks.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
2025-04-23 20:59:38 +02:00
525b86fa43 [Backport 8.0] Moved build-release automation to valkey-release-automation (#1988)
All the Valkey Release automation has been moved and setup here:
https://github.com/valkey-io/valkey-release-automation

The release automation will now be triggered from the
[trigger-build-release.yml](https://github.com/valkey-io/valkey/blob/unstable/.github/workflows/trigger-build-release.yml)
workflow.

Need to remove these files to stop duplicating push to the s3 bucket. PR
(#1977)

---------

Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
Signed-off-by: Nikhil Manglore <nmanglor@amazon.com>
Signed-off-by: Jacob Murphy <jkmurphy@google.com>
Co-authored-by: Roshan Khatri <117414976+roshkhatri@users.noreply.github.com>
2025-04-22 17:39:54 -07:00
45902d648c [Backport] Fix cluster info sent stats for message with light header (#1563) (#1571)
This issue affected only two message types (CLUSTERMSG_TYPE_PUBLISH and
CLUSTERMSG_TYPE_PUBLISHSHARD) because they used a light message header,
which caused the CLUSTER INFO stats to miss sent/received message
information for those types.

---------

Cherry pick to 8.0 branch

Signed-off-by: Harkrishn Patro <harkrisp@amazon.com>
Signed-off-by: Harkrishn Patro <bunty.hari@gmail.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
2025-04-15 16:51:44 +02:00
PierreandGitHub 35377a6b61 Backport "Fix memory leak in forgotten node ping ext code path" to 8.0 (#1575)
When processing a cluster bus PING extension, there is a memory leak
when adding a new key to the `nodes_black_list` dict. We now make sure
to free the key `sds` if the dict did not take ownership of it.

Backport of #1574 from unstable.

Signed-off-by: Pierre Turin <pieturin@amazon.com>
2025-04-15 16:47:58 +02:00
8a5f5e9467 Backport: Optimize RDB load performance and fix cluster mode resizing on replica side (#1199) (#1328)
This PR addresses two issues:

1. Performance Degradation Fix - Resolves a significant performance
issue during RDB load on replica nodes.
- The problem was causing replicas to rehash multiple times during the
load process. Local testing demonstrated up to 50% degradation in BGSAVE
time.
- The problem occurs when the replica tries to expand pre-created slot
dictionaries. This operation fails quietly, resulting in undetected
performance issues.
- This fix aims to optimize the RDB load process and restore expected
performance levels.

2. Bug fix when reading `RDB_OPCODE_RESIZEDB` in Valkey 8.0 cluster
mode-
- Use the shard's master slots count when processing this opcode, as
`clusterNodeCoversSlot` is not initialized for the currently syncing
replica.
- Previously, this problem went unnoticed because `RDB_OPCODE_RESIZEDB`
had no practical impact (due to 1).

These improvements will enhance overall system performance and ensure
smoother upgrades to Valkey 8.0 in the future.

Testing:
- Conducted local tests to verify the performance improvement during RDB
load.
- Verified that ignoring `RDB_OPCODE_RESIZEDB` does not negatively
impact functionality in the current version.



(cherry picked from commit f60ecc80b5)

Signed-off-by: naglera <anagler123@gmail.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
2025-04-15 16:43:28 +02:00
Roshan KhatriandViktor Söderqvist c59c36a9da Migrate binaries build to ARM github runners
Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2025-04-15 09:41:30 +02:00
Nikhil MangloreandViktor Söderqvist 7526d45dfe Update correct repository name for automation trigger workflow (#1855)
Update automation repository name to `valkey-release-automation`

Signed-off-by: Nikhil Manglore <nmanglor@amazon.com>
2025-04-15 09:41:30 +02:00
a1629e6779 Trigger post-release tasks in Valkey for a new release (#1830)
This trigger-build-release.yml file will be used to automate the release
process as described in #1397. It will trigger the post release task after a new release

---------

Signed-off-by: Nikhil Manglore <nmanglor@amazon.com>
Signed-off-by: Nikhil Manglore <nikhilmanglore9@gmail.com>
Co-authored-by: Roshan Khatri <117414976+roshkhatri@users.noreply.github.com>
2025-04-15 09:41:30 +02:00
9dc97134dd Minor cleanup in defragEntryFinishCbForKeys (#1562)
Minor cleanup, fast path reduces the number of lines of code
and improves readability.

Signed-off-by: Nadav Levanoni <nadavl@amazon.com>
Co-authored-by: Nadav Levanoni <nadavl@amazon.com>
2025-01-15 14:01:25 +08:00
Madelyn Olson 44852a1fad Update release notes with final security information
Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
2025-01-07 13:14:48 -08:00
Madelyn OlsonandYaacovHazan 1acdd8425b Fix LUA garbage collector (CVE-2024-46981) (#1513)
Reset GC state before closing the lua VM to prevent user data to be
wrongly freed while still might be used on destructor callbacks.

Created and publish by Redis in their OSS branch.

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: YaacovHazan <yaacov.hazan@redis.com>
2025-01-07 13:14:48 -08:00
Madelyn OlsonandYaacovHazan 23b6e337d7 Fix Read/Write key pattern selector (CVE-2024-51741) (#1514)
The explanation on the original commit was wrong. Key based access must
have a `~` in order to correctly configure whey key prefixes to apply
the selector to. If this is missing, a server assert will be triggered
later.

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: YaacovHazan <yaacov.hazan@redis.com>
2025-01-07 13:14:48 -08:00
Madelyn Olson 7033e36ceb Update info.tcl test to revert client output limits sooner (#1462)
We set the client output buffer limits to 10 bytes, and then execute
`info stats` which produces more than 10 bytes of output, which can
cause that command to throw an error.

I'm not sure why it wasn't consistently erroring before, might have been
some change related to the ubuntu upgrade though.

Issues related to ubuntu-tls are hopefully resolved now.

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
2025-01-07 13:14:48 -08:00
BinbinandMadelyn Olson 55f73f7d7b Try to stabilize the failover call in the slot migration test (#1078)
The CI report replica will return the error when performing CLUSTER
FAILOVER:
```
-ERR Master is down or failed, please use CLUSTER FAILOVER FORCE
```

This may because the primary state is fail or the cluster connection
is disconnected during the primary pause. In this PR, we added some
waits in wait_for_role, if the role is replica, we will wait for the
replication link and the cluster link to be ok.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2025-01-07 13:14:48 -08:00
ranshidandMadelyn Olson bcd13d37a9 change the container image to ubuntu:plucky (#1359)
Our fortify workflow is running on ubuntu lunar container that is EOL
since [January 25, 2024(January 25,
2024](https://lists.ubuntu.com/archives/ubuntu-announce/2024-January/000298.html).
This case cause the workflow to fail during update actions like:
```
apt-get update && apt-get install -y make gcc-13
  update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-1[3](https://github.com/valkey-io/valkey/actions/runs/12021130026/job/33547460209#step:5:3) 100
  make all-with-unit-tests CC=gcc OPT=-O3 SERVER_CFLAGS='-Werror -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3'
  shell: sh -e {0}
Ign:1 http://security.ubuntu.com/ubuntu lunar-security InRelease
Err:2 http://security.ubuntu.com/ubuntu lunar-security Release
  [4](https://github.com/valkey-io/valkey/actions/runs/12021130026/job/33547460209#step:5:4)04  Not Found [IP: 91.189.91.82 80]
Ign:3 http://archive.ubuntu.com/ubuntu lunar InRelease
Ign:4 http://archive.ubuntu.com/ubuntu lunar-updates InRelease
Ign:[5](https://github.com/valkey-io/valkey/actions/runs/12021130026/job/33547460209#step:5:5) http://archive.ubuntu.com/ubuntu lunar-backports InRelease
Err:[6](https://github.com/valkey-io/valkey/actions/runs/12021130026/job/33547460209#step:5:7) http://archive.ubuntu.com/ubuntu lunar Release
  404  Not Found [IP: 185.125.190.81 80]
Err:7 http://archive.ubuntu.com/ubuntu lunar-updates Release
  404  Not Found [IP: 185.125.190.81 80]
Err:8 http://archive.ubuntu.com/ubuntu lunar-backports Release
  404  Not Found [IP: 185.125.190.81 80]
Reading package lists...
E: The repository 'http://security.ubuntu.com/ubuntu lunar-security Release' does not have a Release file.
E: The repository 'http://archive.ubuntu.com/ubuntu lunar Release' does not have a Release file.
E: The repository 'http://archive.ubuntu.com/ubuntu lunar-updates Release' does not have a Release file.
E: The repository 'http://archive.ubuntu.com/ubuntu lunar-backports Release' does not have a Release file.
update-alternatives: error: alternative path /usr/bin/gcc-[13](https://github.com/valkey-io/valkey/actions/runs/12021130026/job/33547460209#step:5:14) doesn't exist
Error: Process completed with exit code 2.
```

example:
https://github.com/valkey-io/valkey/actions/runs/12021130026/job/33547460209

This pr uses the latest stable ubuntu image release
[plucky](https://hub.docker.com/layers/library/ubuntu/plucky/images/sha256-dc4565c7636f006c26d54c988faae576465e825ea349fef6fd3af6bf5100e8b6?context=explore)

Signed-off-by: Ran Shidlansik <ranshid@amazon.com>
2025-01-07 13:14:48 -08:00
Madelyn Olson 918c635f38 Remove test that relies on functionality from unstable
Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
2025-01-07 13:14:48 -08:00
Madelyn Olson e63c09fccb Add release notes for 8.0.2
Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
2025-01-07 13:14:48 -08:00
f97fb6ca98 Add scoped RDB loading context and immediate abort flag (#1173)
This PR introduces a new mechanism for temporarily changing the
server's loading_rio context during RDB loading operations. The new
`RDB_SCOPED_LOADING_RIO` macro allows for a scoped change of the
`server.loading_rio` value, ensuring that it's automatically restored
to its original value when the scope ends.

Introduces a dedicated flag to `rio` to signal immediate abort,
preventing
potential use-after-free scenarios during replication disconnection in
dual-channel load. This ensures proper termination of
`rdbLoadRioWithLoadingCtx`
when replication is cancelled due to connection loss on main connection.

Fixes https://github.com/valkey-io/valkey/issues/1152

---------

Signed-off-by: naglera <anagler123@gmail.com>
Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Signed-off-by: Amit Nagler <58042354+naglera@users.noreply.github.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: ranshid <88133677+ranshid@users.noreply.github.com>
2025-01-07 13:14:48 -08:00
BinbinandMadelyn Olson 1b8c8c8c99 Reset repl_down_since to zero only on state change (#1149)
We should reset repl_down_since only on state change, in the
current code, if the rdb channel in the dual channel is normal,
that is, rdb is loaded normally, but the psync channel is
abnormal, we will set repl_down_since 0 here. If the primary
is down at this time, the replica may be abnormal when calculating
data_age in cluster failover, since repl_state != REPL_STATE_CONNECTED,
this causes the replica to be unable to initiate an election due
to the old data_age.

In dualChannelSyncHandleRdbLoadCompletion, if the psync channel
is not established, the function will return. We will set repl_state
to REPL_STATE_CONNECTED and set repl_down_since to 0 in
dualChannelSyncSuccess, that is, in establishPrimaryConnection.

See also ce8caa867c for more details.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2025-01-07 13:14:48 -08:00
722f542d4b Optimize RDB load performance and fix cluster mode resizing on replica side (#1199)
This PR addresses two issues:

1. Performance Degradation Fix - Resolves a significant performance
issue during RDB load on replica nodes.
- The problem was causing replicas to rehash multiple times during the
load process. Local testing demonstrated up to 50% degradation in BGSAVE
time.
- The problem occurs when the replica tries to expand pre-created slot
dictionaries. This operation fails quietly, resulting in undetected
performance issues.
- This fix aims to optimize the RDB load process and restore expected
performance levels.

2. Bug fix when reading `RDB_OPCODE_RESIZEDB` in Valkey 8.0 cluster
mode-
- Use the shard's master slots count when processing this opcode, as
`clusterNodeCoversSlot` is not initialized for the currently syncing
replica.
- Previously, this problem went unnoticed because `RDB_OPCODE_RESIZEDB`
had no practical impact (due to 1).

These improvements will enhance overall system performance and ensure
smoother upgrades to Valkey 8.0 in the future.

Testing:
- Conducted local tests to verify the performance improvement during RDB
load.
- Verified that ignoring `RDB_OPCODE_RESIZEDB` does not negatively
impact functionality in the current version.

Signed-off-by: naglera <anagler123@gmail.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
2025-01-07 13:14:48 -08:00
BinbinandMadelyn Olson 32ab751d39 Avoid double close on repl_transfer_fd (#1349)
The code is ok before 1a6dc8459a,
but now we will set server.repl_transfer_fd right after dfd was
initiated, and in here we have a double close error since dfd and
server.repl_transfer_fd are the same fd.

Also move the declaration of dfd/maxtries to a small scope to avoid
the confusion since they are only used in this code.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2025-01-07 13:14:48 -08:00
cbfdfaa6bf Use the correct command proc for the LOOKUP_NOTOUCH exception in lookupKey (#1499)
When looking up a key in no-touch mode, `LOOKUP_NOTOUCH` is set to avoid
updating the last access time in `lookupKey`. An exception must be made
for the `TOUCH` command which must always update the key.

When called from a script, `server.executing_client` will point to the
`TOUCH` command, while `server.current_client` will point to e.g. an
`EVAL` command. So, we must use the former to find out the currently
executing command if defined.

This fix addresses the issue where TOUCH wasn't updating key access
times when called from scripts like EVAL.

Fixes #1498

Signed-off-by: Simon Baatz <gmbnomis@gmail.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
2025-01-07 13:14:48 -08:00
ad9dc0d5c4 Fix restore replica output bytes stat update (#1486)
This PR fixes the missing stat update for `total_net_repl_output_bytes`
that was removed during the refactoring in PR #758. The metric was not
being updated when writing to replica connections.

Changes:
- Restored the stat update in postWriteToClient for replica connections
- Added integration test to verify the metric is properly updated

Signed-off-by: Uri Yagelnik <uriy@amazon.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
2025-01-07 13:14:48 -08:00
BinbinandMadelyn Olson d7b655a918 Fix incorrect cache_memory reset in functionsLibCtxClear (#1255)
functionsLibCtxClear should clear the provided lib_ctx parameter,
not the static variable curr_functions_lib_ctx, as this contradicts
the function's intended purpose.

The impact i guess is minor, like in some unhappy paths (diskless load
fails, function restore fails?), we will mess up the functions_caches
field, which is used in used_memory_functions / used_memory_scripts
fileds in INFO.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2025-01-07 13:14:48 -08:00
Wen HuiandMadelyn Olson 997a123baf Revert "Decline unsubscribe related command in non-subscribed mode" (#1265)
This PR goal is to revert the changes on PR
https://github.com/valkey-io/valkey/pull/759

Recently, we got some reports that in Valkey 8.0 the PR
https://github.com/valkey-io/valkey/pull/759 (Decline unsubscribe
related command in non-subscribed mode) causes break change.
(https://github.com/valkey-io/valkey/issues/1228)

Although from my thought, call commands "unsubscribeCommand",
"sunsubscribeCommand", "punsubscribeCommand" in request-response mode
make no sense. This is why I created PR
https://github.com/valkey-io/valkey/pull/759

But breaking change is always no good, @valkey-io/core-team How do you
think we revert this PR code changes?

Signed-off-by: hwware <wen.hui.ware@gmail.com>
2025-01-07 13:14:48 -08:00
b2a5c59fba Fix LRU crash when getting too many random lua scripts (#1310)
### Problem
Valkey stores scripts in a dictionary (lua_scripts) keyed by their SHA1
hashes, but it needs a way to know which scripts are least recently
used. It uses an LRU list (lua_scripts_lru_list) to keep track of
scripts in usage order. When the list reaches a maximum length, Valkey
evicts the oldest scripts to free memory in both the list and
dictionary. The problem here is that the sds from the LRU list can be
pointing to already freed/moved memory by active defrag that the sds in
the dictionary used to point to. It results in assertion error at [this
line](https://github.com/valkey-io/valkey/blob/unstable/src/eval.c#L519)

### Solution
If we duplicate the sds when adding it to the LRU list, we can create an
independent copy of the script identifier (sha). This duplication
ensures that the sha string in the LRU list remains stable and
unaffected by any defragmentation that could alter or free the original
sds. In addition, dictUnlink doesn't require exact pointer
match([ref](https://github.com/valkey-io/valkey/blob/unstable/src/eval.c#L71-L78))
so this change makes sense to unlink the right dictEntry with the copy
of the sds.

### Reproduce
To reproduce it with tcl test:
1. Disable je_get_defrag_hint in defrag.c to trigger defrag often
2. Execute test script
```
start_server {tags {"auth external:skip"}} {

    test {Regression for script LRU crash} {
        r config set activedefrag yes
        r config set active-defrag-ignore-bytes 1
        r config set active-defrag-threshold-lower 0
        r config set active-defrag-threshold-upper 1
        r config set active-defrag-cycle-min 99
        r config set active-defrag-cycle-max 99

        for {set i 0} {$i < 100000} {incr i} {
            r eval "return $i" 0
        }
        after 5000;
    }
}
```


### Crash info
Crash report:
```
=== REDIS BUG REPORT START: Cut & paste starting from here ===
14044:M 12 Nov 2024 14:51:27.054 # === ASSERTION FAILED ===
14044:M 12 Nov 2024 14:51:27.054 # ==> eval.c:556 'de' is not true

------ STACK TRACE ------

Backtrace:
/usr/bin/redis-server 127.0.0.1:6379 [cluster](luaDeleteFunction+0x148)[0x723708]
/usr/bin/redis-server 127.0.0.1:6379 [cluster](luaCreateFunction+0x26c)[0x724450]
/usr/bin/redis-server 127.0.0.1:6379 [cluster](evalCommand+0x2bc)[0x7254dc]
/usr/bin/redis-server 127.0.0.1:6379 [cluster](call+0x574)[0x5b8d14]
/usr/bin/redis-server 127.0.0.1:6379 [cluster](processCommand+0xc84)[0x5b9b10]
/usr/bin/redis-server 127.0.0.1:6379 [cluster](processCommandAndResetClient+0x11c)[0x6db63c]
/usr/bin/redis-server 127.0.0.1:6379 [cluster](processInputBuffer+0x1b0)[0x6dffd4]
/usr/bin/redis-server 127.0.0.1:6379 [cluster][0x6bd968]
/usr/bin/redis-server 127.0.0.1:6379 [cluster][0x659634]
/usr/bin/redis-server 127.0.0.1:6379 [cluster](amzTLSEventHandler+0x194)[0x6588d8]
/usr/bin/redis-server 127.0.0.1:6379 [cluster][0x750c88]
/usr/bin/redis-server 127.0.0.1:6379 [cluster](aeProcessEvents+0x228)[0x757fa8]
/usr/bin/redis-server 127.0.0.1:6379 [cluster](redisMain+0x478)[0x7786b8]
/lib64/libc.so.6(__libc_start_main+0xe4)[0xffffa7763da4]
/usr/bin/redis-server 127.0.0.1:6379 [cluster][0x5ad3b0]
```
Defrag info:
```
mem_fragmentation_ratio:1.18
mem_fragmentation_bytes:47229992
active_defrag_hits:20561
active_defrag_misses:5878518
active_defrag_key_hits:77
active_defrag_key_misses:212
total_active_defrag_time:29009
```

### Test:
Run the test script to push 100,000 scripts to ensure the LRU list keeps
500 maximum length without any crash.
```
27489:M 14 Nov 2024 20:56:41.583 * LRU List length: 500
27489:M 14 Nov 2024 20:56:41.583 * LRU List length: 500
27489:M 14 Nov 2024 20:56:41.584 * LRU List length: 500
27489:M 14 Nov 2024 20:56:41.584 * LRU List length: 500
27489:M 14 Nov 2024 20:56:41.584 * LRU List length: 500
27489:M 14 Nov 2024 20:56:41.584 * LRU List length: 500
27489:M 14 Nov 2024 20:56:41.584 * LRU List length: 500
27489:M 14 Nov 2024 20:56:41.584 * LRU List length: 500
27489:M 14 Nov 2024 20:56:41.584 * LRU List length: 500
27489:M 14 Nov 2024 20:56:41.584 * LRU List length: 500
27489:M 14 Nov 2024 20:56:41.584 * LRU List length: 500
27489:M 14 Nov 2024 20:56:41.584 * LRU List length: 500
27489:M 14 Nov 2024 20:56:41.584 * LRU List length: 500
[ok]: Regression for script LRU crash (6811 ms)
[1/1 done]: unit/test (7 seconds)
```

---------

Signed-off-by: Seungmin Lee <sungming@amazon.com>
Signed-off-by: Seungmin Lee <155032684+sungming2@users.noreply.github.com>
Co-authored-by: Seungmin Lee <sungming@amazon.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
2025-01-07 13:14:48 -08:00
Roshan KhatriandMadelyn Olson 74fd7fd2f9 Workflow changes to fix old release binaries (#1461)
- Moves `build-config.json` to workflow dir to build old versions with
new configs.
- Enables contributors to test release Wf on private repo by adding
`github.event_name == 'workflow_dispatch' ||`

---------

Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2025-01-07 13:14:48 -08:00
BinbinandMadelyn Olson 4dd7161c53 Fix wrong file name in build-release-packages.yml (#1437)
Introduced in #1363, the file name does not match.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2025-01-07 13:14:48 -08:00
Roshan KhatriandMadelyn Olson eac6b29a50 Fix the secrete for test bucket. (#1447)
We have set the secret as `AWS_S3_TEST_BUCKET` for test bucket and I
missed it in the initial review.

Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
2025-01-07 13:14:48 -08:00
Vu DiepandMadelyn Olson 5e007f50bd Use configure-aws-credentials workflow instead of passing secret_access_key (#1363)
## Summary
This PR fixes #1346 where we can get rid of the long term credentials by
using OpenID Connect. OpenID Connect (OIDC) allows your GitHub Actions
workflows to access resources in Amazon Web Services (AWS), without
needing to store the AWS credentials as long-lived GitHub secrets.

---------

Signed-off-by: vudiep411 <vdiep@amazon.com>
2025-01-07 13:14:48 -08:00
BinbinandMadelyn Olson ee53ecb186 Skip build-release-packages CI job in forks (#1438)
The CI job was introduced in #1363, we should skip it in forks.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2025-01-07 13:14:48 -08:00
muelstefamznandMadelyn Olson 053186fa39 Trim free space from inline command argument strings to avoid excess memory usage (#1213)
The command argument strings created while parsing inline commands (see
`processInlineBuffer()`) can contain free capacity. Since some commands
,such as `SET`, store these strings in the database, that free capacity
increases the memory usage. In the worst case, it could double the
memory usage.

This only occurs if the inline command format is used. The argument
strings are built by appending character by character in
`sdssplitargs()`. Regular RESP commands are not affected.

This change trims the strings within `processInlineBuffer()`.

### Why `trimStringObjectIfNeeded()` within `object.c` is not solving
this?

When the command argument string is packed into an object,
`trimStringObjectIfNeeded()` is called.

This does only trim the string if it is larger than
`PROTO_MBULK_BIG_ARG` (32kB), as only strings larger than this would
ever need trimming if the command it sent using the bulk string format.

We could modify this condition, but that would potentially have a
performance impact on commands using the bulk format. Since those make
up for the vast majority of executed commands, limiting this change to
inline commands seems prudent.

### Experiment Results

* 1 million `SET [key] [value]` commands
* Random keys (16 bytes)
* 600 bytes values

Memory usage without this change:

```
used_memory:1089327888
used_memory_human:1.01G
used_memory_rss:1131696128
used_memory_rss_human:1.05G
used_memory_peak:1089348264
used_memory_peak_human:1.01G
used_memory_peak_perc:100.00%
used_memory_overhead:49302800
used_memory_startup:911808
used_memory_dataset:1040025088
used_memory_dataset_perc:95.55%
```

Memory usage with this change:
```
used_memory:705327888
used_memory_human:672.65M
used_memory_rss:718802944
used_memory_rss_human:685.50M
used_memory_peak:705348256
used_memory_peak_human:672.67M
used_memory_peak_perc:100.00%
used_memory_overhead:49302800
used_memory_startup:911808
used_memory_dataset:656025088
used_memory_dataset_perc:93.13%
```

If the same experiment is repeated using the normal RESP array of bulk
string format (`*3\r\n$3\r\nSET\r\n...`) then the memory usage is 672MB
with and without of this change.

If a replica is attached, its memory usage is 672MB with and without
this change, since the replication link never uses inline commands.

Signed-off-by: Stefan Mueller <muelstef@amazon.com>
2025-01-07 13:14:48 -08:00
BinbinandMadelyn Olson 15c749ce31 Fix SORT GET to ignore special pattern # in cluster slot check (#1182)
This special pattern '#' is used to get the element itself,
it does not actually participate in the slot check.

In this case, passing `GET #` will cause '#' to participate
in the slot check, causing the command to get an
`pattern may be in different slots` error.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2025-01-07 13:14:48 -08:00
90dee49093 Add 'WithDictIndex' expiry API and update RANDOMKEY command (#1155)
https://github.com/valkey-io/valkey/issues/1145

First part of a two-step effort to add `WithSlot` API for expiry. This
PR is to fix a crash that occurs when a RANDOMKEY uses a different slot
than the cached slot of a client during a multi-exec.

The next part will be to utilize the new API as an optimization to
prevent duplicate work when calculating the slot for a key.

---------

Signed-off-by: Nadav Levanoni <nadavl@amazon.com>
Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: Nadav Levanoni <nadavl@amazon.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
2025-01-07 13:14:48 -08:00
BinbinandMadelyn Olson 7334d4ae36 Fix FUNCTION KILL error message being displayed as SCRIPT KILL (#1171)
The client that was killed by FUNCTION KILL received a reply of
SCRIPT KILL and the server log also showed SCRIPT KILL.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2025-01-07 13:14:48 -08:00
f91035e0ef Fix empty response for ACL CAT category subcommand for module defined categories (#1140)
The module commands which were added to acl categories were getting
skipped when `ACL CAT category` command was executed.

This PR fixes the bug.
Before:
```
127.0.0.1:6379> ACL CAT foocategory
(empty array)
```
After:
```
127.0.0.1:6379> ACL CAT foocategory
aclcheck.module.command.test.add.new.aclcategories
```

---------

Signed-off-by: Roshan Khatri <rvkhatri@amazon.com>
Co-authored-by: Harkrishn Patro <bunty.hari@gmail.com>
2025-01-07 13:14:48 -08:00
729ee95762 Fix primary crash when processing dirty slots during shutdown wait / failover wait / client pause (#1131)
We have an assert in propagateNow. If the primary node receives a
CLUSTER UPDATE such as dirty slots during SIGTERM waitting or during
a manual failover pausing or during a client pause, the delKeysInSlot
call will trigger this assert and cause primary crash.

In this case, we added a new server_del_keys_in_slot state just like
client_pause_in_transaction to track the state to avoid the assert
in propagateNow, the dirty slots will be deleted in the end without
affecting the data consistency.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2025-01-07 13:14:48 -08:00
ranshidandGitHub 6c41b29644 Fix for when active rehashing kvstore dictionaries are replaced by defrag (#1512)
The kvstore operates the active rehashing by traversing a list of
dictionaries which were registered to it when they started rehashing.
The problem is that active defrag may realloc some of the dictionary
structures while they are registered on the list.where the dictionary 
might be relocated in dictDefragTables.

The Solution is to make sure we update the rehashing list node withy the
new(or not) dictionary pointer after applying the defrag function.

Signed-off-by: Ran Shidlansik <ranshid@amazon.com>
2025-01-06 13:36:33 -08:00
Madelyn OlsonandGitHub ba0a3b0137 Apply security fixes for CVEs (#1114)
Apply the security fixes for the release.

(CVE-2024-31449) Lua library commands may lead to stack overflow and
potential RCE.
(CVE-2024-31227) Potential Denial-of-service due to malformed ACL
selectors.
(CVE-2024-31228) Potential Denial-of-service due to unbounded pattern
matching.

---------

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
2024-10-02 14:09:21 -07:00
Melroy van den BergandViktor Söderqvist c7952082d0 Build binary releases with systemd support (#1107)
- Add systemd support to the build artifact tarballs, so people can use
it under systemd compatible distros. As discussed here:
https://github.com/orgs/valkey-io/discussions/1103#discussioncomment-10815549.
Adding `libsystemd-dev` to install and add `USE_SYSTEMD=yes` to the
build.
- Cleanup & bring the arm & x86 workflow files in-sync. It was a bit of
a mess ;) (removing `jq wget awscli` from the 'Tarball' step)

Signed-off-by: Melroy van den Berg <melroy@melroy.org>
2024-10-02 20:10:55 +02:00
Melroy van den BergandViktor Söderqvist cc84b38f55 Avoid .c, .d and .o files from being copied to the binary tar.gz releases (#1106)
As discussed here:
https://github.com/orgs/valkey-io/discussions/1103#discussioncomment-10814006

`cp` can't be used anymore, `rsync` is more powerful and allow to
exclude files.

Alternatively:

1. Remove the c, d and o files. Which isn't ideal either.
2. Improve the build. Eg. by building inside a `build` directory instead
of in the src folder.

Ps. I know these workflows aren't trigger in this PR. Only via "Build
Release Packages" workflow action:
https://github.com/valkey-io/valkey/actions/workflows/build-release-packages.yml..
So I can't fully test in this PR. But it should work ^^

Ps. ps. I did test `rsync -av --exclude='*.c' --exclude='*.d'
--exclude='*.o' src/valkey-*` command in isolation and that works as
expected!

---------

Signed-off-by: Melroy van den Berg <melroy@melroy.org>
2024-10-02 20:10:55 +02:00
BinbinandMadelyn Olson 6e3a0482f2 Fix timing issue in the new tot-net-out replica test (#1060)
Apparently there is a timing issue when using wait_for_ofs_sync:
```
[exception]: Executing test client: can't read "out_before": no such variable.
can't read "out_before": no such variable
```

The reason is that if the connection between the primary
and the replica is not established yet, the master_repl_offset
of the primary and replica in wait_for_ofs_sync is 0, and
the check fails, resulting in no replica client in the
client list below.

In this case, we need to make sure the replica is online
before proceeding.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-30 17:11:51 -07:00
Madelyn Olson 5fbdfb0898 Initial staging for 8.0.1 security release
Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
2024-09-30 17:06:18 -07:00
zhenwei piandMadelyn Olson a206ec10f4 Fix RDMA build dependence (#1074)
RDMA module has dependence on '$(SERVER_NAME)' rather than the old style
'$(REDIS_SERVER_NAME)'.

Signed-off-by: zhenwei pi <pizhenwei@bytedance.com>
2024-09-30 16:32:12 -07:00
Viktor SöderqvistandMadelyn Olson 9b45ca22a1 Fix bug for CLUSTER SLOTS from EVAL over TLS (#1072)
For fake clients like the ones used for Lua and modules, we don't
determine TLS in the right way, causing CLUSTER SLOTS from EVAL over TLS
to fail a debug-assert.

This error was introduced when the caching of CLUSTER SLOTS was
introduced, i.e. in 8.0.0.

Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2024-09-30 16:31:54 -07:00
8e975e2251 Print an empty primary log when primary lost its last slot (#1064)
The one in CLUSTER SETSLOT help us keep track of state better,
of course it also can make the test case happy.

The one in gossip process fixes a problem that a replica can
print a log saying it is an empty primary.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Ping Xie <pingxie@outlook.com>
2024-09-30 16:31:39 -07:00
BinbinandMadelyn Olson d5b4425f74 Fix module / script call CLUSTER SLOTS / SHARDS fake client check crash (#1063)
The reason is VM_Call will use a fake client without connection,
so we also need to check if c->conn is NULL.

This also affects scripts. If they are called in the script, the
server will crash. Injecting commands into AOF will also cause
startup failure.

Fixes #1054.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-30 16:31:16 -07:00
BinbinandMadelyn Olson e337cdd763 Use _Thread_local to solve threads.h build issue (#1053)
Apparently this will fail to compile in some masOS version.
And internet claims _Thread_local is portable.

Fixes #1051.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-30 16:28:47 -07:00
ranshidandMadelyn Olson 61243b9b22 Fix memory allocation for server databases (#1046)
Fix a bug in the way we allocate memory for the server databases
Introduced in #156.

Signed-off-by: Ran Shidlansik <ranshid@amazon.com>
2024-09-30 16:28:32 -07:00
BinbinandMadelyn Olson 8eefb05609 Fix default value of primary-reboot-down-after-period in sentinel.conf (#1040)
Since in here the monitor value is mymaster, we need to make sure the
primary name is the same, otherwise the default configuration cannot start
sentinel.
```
sentinel monitor mymaster 127.0.0.1 6379 2
```

The following error occurs when the default configuration is started:
```
*** FATAL CONFIG FILE ERROR (Version 255.255.255) ***
Reading the configuration file, at line 358
>>> 'SENTINEL primary-reboot-down-after-period myprimary 0'
No such master with specified name.
```

Introduced in #647.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-30 16:28:15 -07:00
Ping XieandGitHub 1549a4bc27 Fix a typo in the 8.0 release notes (#1036)
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 13:08:04 -07:00
dbcf7394b3 Add Valkey 8.0 GA Release Notes (#1034)
Based on #1031

---------

Signed-off-by: Ping Xie <pingxie@google.com>
Signed-off-by: Ping Xie <pingxie@outlook.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
2024-09-15 12:53:12 -07:00
Ping XieandPing Xie 27ffb82e3b Revert "Update version number to 8.0"
This reverts commit 3179f2528db86582fb7fbf26d6d0e59555cd6b18.

Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
Ping XieandPing Xie 28245745f7 Update version number to 8.0
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
ShivshankarandPing Xie 4522953882 Update valkey-benchmark log output to reference 'server' instead of 'Redis' (#1029)
Replaced "Could not connect to Redis" with "Could not connect to server" in the log
output for connection errors in `getRedisContext` and `createClient`.

Signed-off-by: Shivshankar-Reddy <shiva.sheri.github@gmail.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
BinbinandPing Xie 0204089bf5 Replica flush the old data after RDB file is ok in disk-based replication (#926)
Call emptyData right before rdbLoad to prevent errors in the middle
and we drop the replication stream and leaving an empty database.
The real changes is in disk-based part, the rest is just code movement.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
Ping XieandPing Xie bfd615d485 Improve code readability in dict.c (#943)
This pull request improves code readability, as a follow up of #749.

- Internal Naming Conventions: Removed the use of underscores (_) for
internal static structures/functions.

- Descriptive Function Names: Updated function names to be more
descriptive, making their purpose clearer. For instance, `_dictExpand`
is renamed to `dictExpandIfAutoResizeAllowed`.

---------

Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
BinbinandPing Xie c4c71b0ecc Fix replica unable trigger migration when it received CLUSTER SETSLOT in advance (#981)
Fix timing issue in evaluating `cluster-allow-replica-migration` for replicas

There is a timing bug where the primary and replica have different
`cluster-allow-replica-migration` settings. In issue #970, we found that if
the replica receives `CLUSTER SETSLOT` before the gossip update, it remains
in the original shard. This happens because we only process the
`cluster-allow-replica-migration` flag for primaries during `CLUSTER SETSLOT`.

This commit fixes the issue by also evaluating this flag for replicas in the
`CLUSTER SETSLOT` path, ensuring correct replica migration behavior.

Closes #970
---------

Signed-off-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Ping Xie <pingxie@outlook.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
BinbinandPing Xie 3c226143f2 Avoid false positive in election tests (#984)
The node may not be able to initiate an election in time due to
problems with cluster communication. If an election is initiated,
make sure its offset is 0.

Closes #967.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
BinbinandPing Xie be65f2e57d Trigger a save of the cluster configuration file before shutting down (#822)
The cluster configuration file is the metadata "database" for the
cluster. It is best to trigger a save when shutdown the server, to
avoid inconsistent content that is not refreshed.

We save the nodes.conf whenever something that affects the nodes.conf
has changed. But we are saving nodes.conf in clusterBeforeSleep, and
some events may save it without a fsync, there is a time gap.

And shutdown has its own save seems good to me, it doesn't need to
care about the others.

At the same time, a comment is added in unlock nodes.conf to explain
why we actively unlock when shutdown.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
Ping XieandPing Xie 1810c6f2df Re-enable empty-shard slot migration tests (#1024)
Related to #734 and #858

Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
xu0o0andPing Xie 6d683fca9f Make clang-format insert a newline at end of file if missing (#1023)
clang generates warning if there is no newline at the end of the source
file.

Update .clang-format to handle the missing newline at eof.

Signed-off-by: haoqixu <hq.xu0o0@gmail.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
04e00f7db5 Fix wrong count for replica's tot-net-out (#1013)
Fix duplicate calculation of replica's `net_output_bytes`

- Remove redundant calculation leftover from previous refactor
- Add test to prevent regression

Signed-off-by: Uri Yagelnik <uriy@amazon.com>
Signed-off-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
Madelyn OlsonandPing Xie 359425d0bb Optimize the per slot dictionary by checking for cluster mode earlier (#995)
While doing some profiling, I noticed that getKeySlot() was a fairly
large part (~0.7%) of samples doing perf with high pipeline during
standalone. I think this is because we do a very late check for
server.cluster_mode, we first call getKeySlot() and then call
calculateKeySlot(). (calculateKeySlot was surprisingly not automatically
inlined, we were doing a jump into it and then immediately returning
zero). We then also do useless work in the form of caching zero in
client->slot, which will further mess with cache lines.

So, this PR tries to accomplish a few things things.
1) The usage of the `slot` name made a lot more sense before the
introduction of the kvstore. Now with kvstore, we call this the database
index, so all the references to slot in standalone are no longer really
accurate.
2) Pull the cluster mode check all the way out of getKeySlot(), so
hopefully a bit more performant.
3) Remove calculateKeySlot() as independent from getKeySlot().
calculateKeySlot used to have 3 call sites outside of db.c, which
warranted it's own function. It's now only called in two places,
pubsub.c and networking.c.

I ran some profiling, and saw about ~0.3% improvement, but don't really
trust it because you'll see a much higher (~2%) variance in test runs
just by how the branch predictions will get changed with a new memory
layout. Running perf again showed no samples in getKeySlot() and a
reduction in samples in lookupKey(), so maybe this will help a little
bit.

---------

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
Madelyn OlsonandPing Xie 586af0fe01 Improve stability of hostnames test (#1016)
Maybe partially resolves https://github.com/valkey-io/valkey/issues/952.

The hostnames test relies on an assumption that node zero and node six
don't communicate with each other to test a bunch of behavior in the
handshake stake. This was done by previously dropping all meet packets,
however it seems like there was some case where node zero was sending a
single pong message to node 6, which was partially initializing the
state.

I couldn't track down why this happened, but I adjusted the test to
simply pause node zero which also correctly emulates the state we want
to be in since we're just testing state on node 6, and removes the
chance of errant messages. The test was failing about 5% of the time
locally, and I wasn't able to reproduce a failure with this new
configuration.

---------

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
Mikhail KoviazinandPing Xie 4119e3b359 Added .git-blame-ignore-revs (#1010)
This file enables developers to ignore the certain revisions in
git-blame. This is quite handy considering there was a commit that
reformatted the large amount of code in valkey.

As a downside, one has to do a manual step for each clone of valkey to
enable this feature. The instructions are available in the file itself.

---------

Signed-off-by: Mikhail Koviazin <mikhail.koviazin@aiven.io>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
BinbinandPing Xie b88e03bbf7 Fix module RdbLoad wrongly disable the AOF (#1001)
In RdbLoad, we disable AOF before emptyData and rdbLoad to prevent copy-on-write issues. After rdbLoad completes, AOF should be re-enabled, but the code incorrectly checks server.aof_state, which has been reset to AOF_OFF in stopAppendOnly. This leads to AOF not being re-enabled after being disabled.
---------

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
Amit NaglerandPing Xie d2e98b14dc Dual Channel Replication - Verify Replica Local Buffer Limit Configuration (#989)
Prior to comparing the replica buffer against the configured limit, we
need to ensure that the limit configuration is enabled. If the limit is
set to zero, it indicates that there is no limit, and we should skip the
buffer limit check.

---------

Signed-off-by: naglera <anagler123@gmail.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
90a22d63fd Use hashtable as the default type of temp set object during sunion/sdiff (#996)
This patch try to set the temp set object as default hash table type.
And did a simple predication of the temp set object encoding when
initialize `dstset` to reduce the unnecessary conversation.

According to existing code logic, when did operation like `sunion` and
`sdiff` , the temp set object could be `intset`, `listpack` and
`hashtable`, for the `listpack`, the efficiency is low when did
operation like `find` and `compare` , need to traverse all elements.
When we exploring the hotspots, found the `lpFind` and `memcmp` has been
the bottleneck when running workloads like below:

-
[memtier_benchmark-2keys-set-10-100-elements-sunion.yml](https://github.com/redis/redis-benchmarks-specification/blob/main/redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-set-10-100-elements-sunion.yml)
-
[memtier_benchmark-2keys-set-10-100-elements-sdiff.yml](https://github.com/redis/redis-benchmarks-specification/blob/main/redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-set-10-100-elements-sdiff.yml)

![image](https://github.com/user-attachments/assets/71dfc70b-2ad5-4832-a338-712deefca20e)

This patch try to set the temp set object as default hash table type.
And did a simple predication of the temp set object encoding when
initialize `dstset` to reduce the unnecessary conversation.

- OPERATING SYSTEM: Ubuntu 22.04.4 LTS
- Kernel: 5.15.0-116-generic
- PROCESSOR: Intel Xeon Platinum 8380
- Server and Client in same socket.

```
taskset -c 0-3 ~/valkey/src/valkey-server /tmp/valkey.conf

port 9001
bind * -::*
daemonize no
protected-mode no
save ""
```

| Test Name| Perf Boost|
|-|-|

|[memtier_benchmark-2keys-set-10-100-elements-sunion.yml](https://github.com/redis/redis-benchmarks-specification/blob/main/redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-set-10-100-elements-sunion.yml)
|41%|

|[memtier_benchmark-2keys-set-10-100-elements-sdiff.yml](https://github.com/redis/redis-benchmarks-specification/blob/main/redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-set-10-100-elements-sdiff.yml)
|27%|

With above test set which have total 110 elements in the 2 given sets.
We also did some benchmarking by adjusting the total number of elements
in all given sets. We can still observe the performance boost.

![image](https://github.com/user-attachments/assets/b2ab420c-43e5-45de-9715-7d943df229cb)

---------

Signed-off-by: Lipeng Zhu <lipeng.zhu@intel.com>
Co-authored-by: Wangyang Guo <wangyang.guo@intel.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
uriyageandPing Xie 47b14ef377 Fix crash in async IO threads with TLS (#1011)
Fix for https://github.com/valkey-io/valkey/issues/997

Root Cause Analysis:
1. Two different jobs (READ and WRITE) may be sent to the same IO
thread.
2. When processing the read job in `processIOThreadsReadDone`, the IO
thread may find that the write job has also been completed.
3. In this case, the IO thread calls `processClientIOWriteDone` to first
process the completed write job and free the COBs
https://github.com/valkey-io/valkey/blob/89a4fa7695108cd2a1ac584f84e820325e5a3ad9/src/networking.c#L4666
4. If there are pending writes (resulting from pipeline commands), a new
async IO write job is sent before processing the completed read job
https://github.com/valkey-io/valkey/blob/89a4fa7695108cd2a1ac584f84e820325e5a3ad9/src/networking.c#L2417
When sending the write job, the `TLS_CONN_FLAG_POSTPONE_UPDATE_STATE`
flag is set to prevent the IO thread from updating the event loop, which
is not thread-safe.
5. Upon resuming the read job processing, the flag is cleared,
https://github.com/valkey-io/valkey/blob/89a4fa7695108cd2a1ac584f84e820325e5a3ad9/src/networking.c#L4685
causing the IO thread to update the event loop.

Fix:
Prevent sending async write job for pending writes when a read job is
about to be processed.

Testing:
The issue could not be reproduced due to its rare occurrence, which
requires multiple specific conditions to align simultaneously.

Signed-off-by: Uri Yagelnik <uriy@amazon.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
cd79981faf For MEETs, save the extensions support flag immediately during MEET processing (#778)
For backwards compatibility reasons, a node will wait until it receives
a cluster message with the extensions flag before sending its own
extensions. This leads to a delay in shard ID propagation that can
corrupt nodes.conf with inaccurate shard IDs if a node is restarted
before this can stabilize.

This fixes much of that delay by immediately triggering the
extensions-supported flag during the MEET processing and attaching the
node to the link, allowing the PONG reply to contain OSS extensions.

Partially fixes #774

---------

Signed-off-by: Ben Totten <btotten@amazon.com>
Co-authored-by: Ben Totten <btotten@amazon.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
BinbinandPing Xie 77decba670 Add missing moduleapi getchannels test and fix tests (#1002)
Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
zhaozhao.zzandPing Xie 6413dade34 add assertion for kvstore's dictType (#1004)
Signed-off-by: zhaozhao.zz <zhaozhao.zz@alibaba-inc.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
373f51e9fe Migrate dict.c unit tests to new framework (#946)
This PR migrates the tests related to dict into new test framework as
part of #428.

Signed-off-by: haoqixu <hq.xu0o0@gmail.com>
Signed-off-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
a888fabc81 Migrate listpack.c unit tests to new framework (#949)
This PR migrates the tests related to listpack into new test framework
as part of #428.

Signed-off-by: haoqixu <hq.xu0o0@gmail.com>
Signed-off-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
BinbinandPing Xie 933b03140b Add client info to SHUTDOWN / CLUSTER FAILOVER logs (#875)
Print the full client info by using catClientInfoString, the
info is useful when we want to identify the source of request.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
BinbinandPing Xie 31473a4d58 Fix aof base suffix when modifying aof-use-rdb-preamble during rewrite (#886)
If we modify aof-use-rdb-preamble in the middle of rewrite,
we may get a wrong aof base suffix. This is because the suffix
is concatenated by the main process afterwards, and it may be
different from the beginning.

We cache this value when we start the rewrite.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
BinbinandPing Xie a35225204b Fix missing replication link re-connection when primary's IP/port is updated in clusterProcessGossipSection (#965)
`clusterProcessGossipSection` currently doesn't trigger a check and call `replicationSetPrimary` when `myself`'s primary node’s IP/port is updated. This fix ensures that after every node address update, `replicationSetPrimary` is called if the updated node is `myself`'s primary. This prevents missed updates and ensures that replicas reconnect properly to maintain their replication link with the primary.

Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
BinbinandPing Xie d752c6fa20 Add newline to argv in crash report when doing redact (#993)
Minor cleanup, introduced in #877.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
Kyle Kim (kimkyle@)andPing Xie 945673489f Add SLOT-STATS under CLUSTER HELP string. (#988)
Add help wording for cluster SLOT-STATS.

Signed-off-by: Kyle Kim <kimkyle@amazon.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
Viktor SöderqvistandPing Xie 692e12479a Rewrite lazyfree docs in valkey.conf to reflect that lazy is now default (#983)
Before this doc update, the comments in valkey.conf said that DEL is a
blocking command, and even refered to other synchronous freeing as "in a
blocking way, like if DEL was called". This has now become confusing and
incorrect, since DEL is now non-blocking by default.

The comments also mentioned too much about the "old default" and only
later explain that the "new default" is non-blocking.

This doc update focuses on the current default and expresses it like
"Starting from Valkey 8.0, lazy freeing is enabled by default", rather
than using words like old and new.

This is a follow-up to #913.

---------

Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
NAM UK KIMandPing Xie ae911e898d Fix typo in valkey-cli.c (#979)
Change from replicsa to replicas in valkey-cli.c

Signed-off-by: NAM UK KIM <namuk2004@naver.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
a3e1acd944 Improve type safety and refactor dict entry handling (#749)
This pull request introduces several changes to improve the type safety
of Valkey's dictionary implementation:

- Getter/Setter Macros: Implemented macros `DICT_SET_VALUE` and
`DICT_GET_VALUE` to centralize type casting within these macros. This
change emulates the behavior of C++ templates in C, limiting type
casting to specific low-level operations and preventing it from being
spread across the codebase.

- Reduced Assert Overhead: Removed unnecessary asserts from critical hot
paths in the dictionary implementation.

- Consistent Naming: Standardized the naming of dictionary entry types.
For example, all dictionary entry types start their names with
`dictEntry`.

Fix #737

---------

Signed-off-by: Ping Xie <pingxie@google.com>
Signed-off-by: Ping Xie <pingxie@outlook.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-15 11:49:49 -07:00
Madelyn Olson d8c86482e4 Added release notes for RC2
Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
2024-09-03 09:00:45 -07:00
Madelyn Olson 6fb955d941 Initialize all the fields for the test kvstore (#982)
Follow up to https://github.com/valkey-io/valkey/pull/966, which didn't
update the kvstore tests. I'm not actually entirely clear why it fixes
it, but the consistency prevents the crash very reliably so will merge
it now and maybe see if Zhao has a better explanation.

---------

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
2024-09-03 09:00:45 -07:00
Amit NaglerandMadelyn Olson f55e7c5ffa Add configuration hide-user-data-from-log to hide user data from server logs (#877)
Implement data masking for user data in server logs and diagnostic output. This change prevents potential exposure of confidential information, such as PII, and enhances privacy protection. It masks all command arguments, client names, and client usernames.

Added a new hide-user-data-from-log configuration item, default yes.

---------

Signed-off-by: Amit Nagler <anagler123@gmail.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 88e3151743 Fix set expire test due to the new lazyfree configs changes (#980)
Test failed because these two PRs #865 and #913.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
zhaozhao.zzandMadelyn Olson e9adffc3dd Use metadata to handle the reference relationship between kvstore and dict (#966)
Feature `one-dict-per-slot` refactors the database, and part of it
involved splitting the rehashing list from the global level back to the
database level, or more specifically, the kvstore level. This change is
fine, and it also simplifies the process of swapping databases, which is
good. And it should not have a major impact on the efficiency of
incremental rehashing.

To implement the kvstore-level rehashing list, each `dict` under the
`kvstore` needs to know which `kvstore` it belongs. However, kvstore did
not insert the reference relationship into the `dict` itself, instead,
it placed it in the `dictType`. In my view, this is a somewhat odd way.
Theoretically, `dictType` is just a collection of function handles, a
kind of virtual type that can be referenced globally, not an entity. But
now the `dictType` is instantiated, with each `kvstore` owning an actual
`dictType`, which in turn holds a reverse reference to the `kvstore`'s
resource pointer. This design is somewhat uncomfortable for me.

I think the `dictType` should not be instantiated. The references
between actual resources (`kvstore` and `dict`) should occur between
specific objects, rather than force materializing the `dictType`, which
is supposed to be virtual.

---------

Signed-off-by: zhaozhao.zz <zhaozhao.zz@alibaba-inc.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 8ab95da167 Change all the lazyfree configurations to yes by default (#913)
## Set replica-lazy-flush and lazyfree-lazy-user-flush to yes by
default.
There are many problems with running flush synchronously. Even in
single CPU environments, the thread managers should balance between
the freeing and serving incoming requests.

## Set lazy eviction, expire, server-del, user-del to yes by default
We now have a del and a lazyfree del, we also have these configuration
items to control: lazyfree-lazy-eviction, lazyfree-lazy-expire,
lazyfree-lazy-server-del, lazyfree-lazy-user-del. In most cases lazyfree
is better since it reduces the risk of blocking the main thread, and
because we have lazyfreeGetFreeEffort, on those with high effor
(currently
64) will use lazyfree.

Part of #653.

---------

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
Madelyn Olson f3134538f5 Fix zipmap test null pointer (#975)
The previous test does a strncmp on a NULL, which is not valid. It
should be using an empty length string instead. Addresses
https://github.com/valkey-io/valkey/actions/runs/10649272046/job/29519233939.

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 8be025b198 Fast path in SET if the expiration time is expired (#865)
If the expiration time passed in SET is expired, for example, it
has expired due to the machine time (DTS) or the expiration time
passed in (wrong arg). In this case, we don't need to set the key
and wait for the active expire scan before deleting the key.

Compared with previous changes:
1. If the key does not exist, previously we would set the key and wait
for the active expire to delete it, so it is a set + del from the
perspective
of propaganda. Now we will no set the key and return, so it a NOP.

2. If the key exists, previously we woule set the key and wait
for the active expire to delete it, so it is a set + del From the
perspective
of propaganda. Now we will delete it and return, so it is a del.

Adding a new deleteExpiredKeyFromOverwriteAndPropagate function
to reduce the duplicate code.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
2024-09-03 09:00:45 -07:00
Viktor SöderqvistandMadelyn Olson b5b2b7e187 Delete unused parts of zipmap (#973)
Deletes zipmapSet, zipmapGet, etc. Only keep iterator and validate
integrity, what we use when loading an old RDB file.

Adjust unit tests to not use zipmapSet, etc.

Solves a build failure where when compiling with fortify source.

---------

Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 57a86925f9 Fix timing issue in replica migration test (#968)
The reason is the server 3 still have the server 7 as its replica
due to a short wait, the wait is not enough, we should wait for
server loss its replica.
```
*** [err]: valkey-cli make source node ignores NOREPLICAS error when doing the last CLUSTER SETSLOT
Expected '{127.0.0.1 21497 267}' to be equal to '' (context: type eval line 34 cmd {assert_equal [lindex [R 3 role] 2] {}} proc ::test)
```

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
zhaozhao.zzandMadelyn Olson 98035ded96 standalone -REDIRECT handles special case of MULTI context (#895)
In standalone mode, when a `-REDIRECT` error occurs, special handling is
required if the client is in the `MULTI` context.

We have adopted the same handling method as the cluster mode:

1. If a command in the transaction encounters a `REDIRECT` at the time
of queuing, the execution of `EXEC` will return an `EXECABORT` error (we
expect the client to redirect and discard the transaction upon receiving
a `REDIRECT`). That is:

    ```
    MULTI    ==>  +OK
    SET x y  ==>  -REDIRECT
    EXEC     ==>  -EXECABORT
    ```
2. If all commands are successfully queued (i.e., `QUEUED` results are
received) but a redirect is detected during `EXEC` execution (such as a
primary-replica switch), a `REDIRECT` is returned to instruct the client
to perform a redirect. That is:

    ```
    MULTI    ==>  +OK
    SET x y  ==>  +QUEUED
    failover
    EXEC     ==>  -REDIRECT
    ```

---------

Signed-off-by: zhaozhao.zz <zhaozhao.zz@alibaba-inc.com>
2024-09-03 09:00:45 -07:00
5629882f32 Migrate zipmap unit test to new framework (#474)
Migrate zipmap unit test to new unit test framework, parent ticket #428
.

---------

Signed-off-by: Shivshankar-Reddy <shiva.sheri.github@gmail.com>
Signed-off-by: hwware <wen.hui.ware@gmail.com>
Co-authored-by: hwware <wen.hui.ware@gmail.com>
2024-09-03 09:00:45 -07:00
cb09d23fc6 Fix reconfiguring sub-replica causing data loss when myself change shard_id (#944)
When reconfiguring sub-replica, there may a case that the sub-replica will
use the old offset and win the election and cause the data loss if the old
primary went down.

In this case, sender is myself's primary, when executing updateShardId,
not only the sender's shard_id is updated, but also the shard_id of
myself is updated, casuing the subsequent areInSameShard check, that is,
the full_sync_required check to fail.

As part of the recent fix of #885, the sub-replica needs to decide whether
a full sync is required or not when switching shards. This shard membership
check is supposed to be done against sub-replica's current shard_id, which
however was lost in this code path. This then leads to sub-replica joining
the other shard with a completely different and incorrect replication history.

This is the only place where replicaof state can be updated on this path
so the most natural fix would be to pull the chain replication reduction
logic into this code block and before the updateShardId call.

This one follow #885 and closes #942.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Ping Xie <pingxie@outlook.com>
2024-09-03 09:00:45 -07:00
zhaozhao.zzandMadelyn Olson e1c78a4b71 free client's multi state when it becomes dirty (#961)
Release the client's MULTI state when the transaction becomes dirty to
save memory.

---------

Signed-off-by: zhaozhao.zz <zhaozhao.zz@alibaba-inc.com>
2024-09-03 09:00:45 -07:00
Ping XieandMadelyn Olson 9028417de2 Exclude '.' and ':' from isValidAuxChar's banned charset (#963)
Fix a bug in isValidAuxChar where valid characters '.' and ':' were
incorrectly included in the banned charset. This issue affected the
validation of auxiliary fields in the nodes.conf file used by Valkey in
cluster mode, particularly when handling IPv4 and IPv6 addresses. The
code now correctly allows '.' and ':' as valid characters, ensuring
proper handling of these fields. Comments were added to clarify the use
of the banned charset.
 
Related to #736

---------

Signed-off-by: Ping Xie <pingxie@google.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 4795df6189 Revert make KEYS to be an exact match if there is no pattern (#964)
In #792, the time complexity became ambiguous, fluctuating between
O(1) and O(n), which is a significant difference. And we agree uncertainty
can potentially bring disaster to the business, the right thing to do is
to persuade users to use EXISTS instead of KEYS in this case, to do the
right thing the right way, rather than accommodating this incorrect usage.

This reverts commit fcea7be0bd.
This reverts #792.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
Viktor SöderqvistandMadelyn Olson 9a1723d665 Delete TLS.md and update README.md about tests (#960)
Most of the content of TLS.md has already been copied to README.md in
#927.

The description of how to run tests with TLS is moved to
tests/README.md.

Descriptions of the additional scripts runtest-cluster, runtest-sentinel
and runtest-module are added in tests/README.md.

Links to tests/README.md and src/unit/README.md are added in the
top-level README.md along with a brief overview of the `make test-*`
commands.

Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2024-09-03 09:00:45 -07:00
Viktor SöderqvistandMadelyn Olson 97fea764ff Delete files MANIFESTO, BUGS and INSTALL (#958)
The MANIFESTO is not Valkey's manifesto and it doesn't even mention open
source. Let's write another one later, or some other document about our
project principles.

The other two files are one-line files with no relevant info. They're
polluting the file listing at root level. It's the first thing you see
when you start exploring the repo for the first time.

Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2024-09-03 09:00:45 -07:00
326b5ed0ce Migrate the contents of TLS.md into README.md (#927)
Migrate the contents in TLS.md into TLS sections including building,
running and detail supports. TODO list in the TLS.md is almost done
except the implementation of benchmark support is still not the best
approach which should migrate to hiredis async mode.

Closes #888

---------

Signed-off-by: I Hsin Cheng <richard120310@gmail.com>
Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2024-09-03 09:00:45 -07:00
Ping XieandMadelyn Olson 96cecf709a Add comment explaining log file reopening for rotation support (#956) 2024-09-03 09:00:45 -07:00
d49faad4f7 Using intrinsics to optimize counting HyperLogLog trailing bits (#846)
Godbolt link: https://godbolt.org/z/3YPvxsr5s

__builtin_ctz would generate shorter code than hand-written loop.

---------

Signed-off-by: mwish <maplewish117@gmail.com>
Signed-off-by: Binbin <binloveplay1314@qq.com>
Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 3ccbc84f09 Add pause path coverage to replica migration tests (#937)
In #885, we only add a shutdown path, there is another path
is that the server might got hang by slowlog. This PR added
the pause path coverage to cover it.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
Lipeng ZhuandMadelyn Olson 9872dcaa6b Move prepareClientToWrite out of loop for lrange command to reduce the redundant call. (#860)
## Description 
When I explore the cycles distributions for `lrange` test (
`valkey-benchmark -p 9001 -t lrange -d 100 -r 1000000 -n 1000000 -c 50
--threads 4`). I found the `prepareClientToWrite` and
`clientHasPendingReplies` could be reduced to single call outside
instead of called in a loop, ideally we can gain 3% performance. The
corresponding `LRANG_100`, `LRANG_300`, `LRANGE_500`, `LRANGE_600` have
~2% - 3% performance boost, the benchmark test prove it helps.

This patch try to move the `prepareClientToWrite` and its child
`clientHasPendingReplies` out of the loop to reduce the function
overhead.

---------

Signed-off-by: Lipeng Zhu <lipeng.zhu@intel.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 6838d170e2 Wait for the role change and fix the timing issue in the new test (#947)
The test might be fast enough and then there is no change in the role
causing the test to fail. Adding a wait to avoid the timing issue:
```
*** [err]: valkey-cli make source node ignores NOREPLICAS error when doing the last CLUSTER SETSLOT
Expected '{127.0.0.1 23154 267}' to be equal to '' (context: type eval line 24 cmd {assert_equal [lindex [R 3 role] 2] {}} proc ::test)
```

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
Vadym KhoptynetsandMadelyn Olson 0b4a0fb95f Use sdsAllocSize instead of sdsZmallocSize (#923)
sdsAllocSize returns the correct size without consulting the
allocator. Which is much faster than consulting the allocator.
The only exception is SDS_TYPE_5, for which it has to
consult the allocator.

This PR also sets alloc field correctly for embedded string objects.
It assumes that no allocator would allocate a buffer larger
than `259 + sizeof(robj)` for embedded string. We use embedded strings
for strings up to 44 bytes. If this assumption is wrong, the whole
function would require a rewrite. In general case sds type adjustment
might be needed. Such logic should go to sds.c.

---------

Signed-off-by: Vadym Khoptynets <vadymkh@amazon.com>
2024-09-03 09:00:45 -07:00
Amit NaglerandMadelyn Olson 4e1d985fe4 Remove dual-channel-replication Feature Flag's Protection (#908)
Currently, the `dual-channel-replication` feature flag is immutable if
`enable-protected-configs` is enabled, which is the default behavior.
This PR proposes to make the `dual-channel-replication` flag mutable,
allowing it to be changed dynamically without restarting the cluster.

**Motivation:**
The ability to change the `dual-channel-replication` flag dynamically is
essential for testing and validating the feature on real clusters
running in production environments. By making the flag mutable, we can
enable or disable the feature without disrupting the cluster's
operations, facilitating easier testing and experimentation.
Additionally, this change would provide more flexibility for users to
enable or disable the feature based on their specific requirements or
operational needs without requiring a cluster restart.

---------

Signed-off-by: naglera <anagler123@gmail.com>
2024-09-03 09:00:45 -07:00
Viktor SöderqvistandMadelyn Olson f0171def80 Connection minor fixes (#953)
1. Remove redundant connIncrRefs/connDecrRefs

    In socket.c, the reference counter is incremented before calling
callHandler, but the same reference counter is also incremented inside
callHandler before calling the actual callback.

        static inline int callHandler(connection *conn, ConnectionCallbackFunc handler) {
            connIncrRefs(conn);
            if (handler) handler(conn);
            connDecrRefs(conn);
            ...
        }

    This commit removes the redundant incr/decr calls in socket.c

2. Correct return value of connRead for TLS when peer closed

    According to comments in connection.h, connRead returns 0 when the peer
has closed the connection. This patch corrects the return value for TLS
connections. (Without this patch, it returns -1 which means error.)

    There is an observable difference in what is logged in the verbose
level: "Client closed connection" vs "Reading from client: (null)".

---------

Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
2024-09-03 09:00:45 -07:00
uriyageandMadelyn Olson 7c927ceb85 Improve multithreaded performance with memory prefetching (#861)
This PR utilizes the IO threads to execute commands in batches, allowing
us to prefetch the dictionary data in advance.

After making the IO threads asynchronous and offloading more work to
them in the first 2 PRs, the `lookupKey` function becomes a main
bottle-neck and it takes about 50% of the main-thread time (Tested with
SET command). This is because the Valkey dictionary is a straightforward
but inefficient chained hash implementation. While traversing the hash
linked lists, every access to either a dictEntry structure, pointer to
key, or a value object requires, with high probability, an expensive
external memory access.

### Memory Access Amortization

Memory Access Amortization (MAA) is a technique designed to optimize the
performance of dynamic data structures by reducing the impact of memory
access latency. It is applicable when multiple operations need to be
executed concurrently. The principle behind it is that for certain
dynamic data structures, executing operations in a batch is more
efficient than executing each one separately.

Rather than executing operations sequentially, this approach interleaves
the execution of all operations. This is done in such a way that
whenever a memory access is required during an operation, the program
prefetches the necessary memory and transitions to another operation.
This ensures that when one operation is blocked awaiting memory access,
other memory accesses are executed in parallel, thereby reducing the
average access latency.

We applied this method in the development of `dictPrefetch`, which takes
as parameters a vector of keys and dictionaries. It ensures that all
memory addresses required to execute dictionary operations for these
keys are loaded into the L1-L3 caches when executing commands.
Essentially, `dictPrefetch` is an interleaved execution of dictFind for
all the keys.


**Implementation details**

When the main thread iterates over the `clients-pending-io-read`, for
clients with ready-to-execute commands (i.e., clients for which the IO
thread has parsed the commands), a batch of up to 16 commands is
created. Initially, the command's argv, which were allocated by the IO
thread, is prefetched to the main thread's L1 cache. Subsequently, all
the dict entries and values required for the commands are prefetched
from the dictionary before the command execution. Only then will the
commands be executed.

---------

Signed-off-by: Uri Yagelnik <uriy@amazon.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 17c906109a Drop the outdated script replication example comments (#951)
This example was for script replication which we have
completely removed in 7.0, so this example is outdated now.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson d203361731 Make KEYS to be an exact match if there is no pattern (#792)
Although KEYS is a dangerous command and we recommend people
to avoid using it, some people who are not familiar with it
still using it, and even use KEYS with no pattern at all.

Once KEYS is using with no pattern, we can convert it to an
exact match to avoid iterating over all data.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
xu0o0andMadelyn Olson f736254b72 Fix invalid escape sequence in utils, minor cleanup in python script (#948)
According to the Python document[1], any invalid escape sequences in
string literals now generate a DeprecationWarning (SyntaxWarning as of
3.12) and in the future this will become a SyntaxError.

This Change uses Python’s raw string notation for regular expression
patterns to avoid it.

[1]: https://docs.python.org/3.10/library/re.html

Signed-off-by: haoqixu <hq.xu0o0@gmail.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson e97a2d2e3d Add explicit assert to ensure thread_shared_qb won't expand (#938)
Although this won't happen now, adding this statement explicitly.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 84e0d835ad Add epoch information to failover auth denied logs (#816)
When failover deny to vote, sometimes due to network or
some blocking operations, the time of FAILOVER_AUTH_REQUEST
packet arrival is very uncertain. Since there is no epoch
information in these logs, it is hard to associate the log
with other logs.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
NAM UK KIMandMadelyn Olson d21b292d6c Update "Total" message and used_memory_human log information in serverCron() function (#594)
At the VERBOSE/DEBUG log level, which is output once every 5 seconds,
added to show the "Total" message of all clients and to show memory
usage (used_memory) with used_memory_human.
Also, it seems clearer to show "total" number of keys and the number of
volatile in entire keys.

---------

Signed-off-by: NAM UK KIM <namuk2004@naver.com>
2024-09-03 09:00:45 -07:00
Ayush SharmaandMadelyn Olson a59a0b3265 Add support for setting the group on a unix domain socket (#901)
Add new optional, immutable string config called `unixsocketgroup`. 
Change the group of the unix socket to `unixsocketgroup` after `bind()`
if specified.

Adds tests to validate the behavior.

Fixes #873.

Signed-off-by: Ayush Sharma <mrayushs933@gmail.com>
2024-09-03 09:00:45 -07:00
Madelyn Olson 0e64577ae8 Remove accurate from extra test tag (#935)
Today if we attached the "run-extra-tests" tag it adds at least 20
minutes because the dump-fuzzer test runs with full accuracy. This
fuzzer is useful, but probably only really needed for the daily, so
removing it from the PRs. We still run the fuzzers, just not for as
long.

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson e8774bf911 valkey-cli make source node ignores NOREPLICAS when doing the last CLUSTER SETSLOT (#928)
This fixes #899. In that issue, the primary is cluster-allow-replica-migration no
and its replica is cluster-allow-replica-migration yes.

And during the slot migration:
1. Primary calling blockClientForReplicaAck, waiting its replica.
2. Its replica reconfiguring itself as a replica of other shards due to
replica migration and disconnect from the old primary.
3. The old primary never got the chance to receive the ack, so it got a
timeout and got a NOREPLICAS error.

In this case, the replicas might automatically migrate to another primary,
resulting in the client being unblocked with the NOREPLICAS error. In this
case, since the configuration will eventually propagate itself, we can safely
ignore this error on the source node.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
37548047de Fix CLUSTER SETSLOT block and unblock error when all replicas are down (#879)
In CLUSTER SETSLOT propagation logic, if the replicas are down, the
client will get block during command processing and then unblock
with `NOREPLICAS Not enough good replicas to write`.

The reason is that all replicas are down (or some are down), but
myself->num_replicas is including all replicas, so the client will
get block and always get timeout.

We should only wait for those online replicas, otherwise the waiting
propagation will always timeout since there are not enough replicas.
The admin can easily check if there are replicas that are down for an
extended period of time. If they decide to move forward anyways, we
should not block it. If a replica  failed right before the replication and
was not included in the replication, it would also unlikely win the election.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Ping Xie <pingxie@google.com>
2024-09-03 09:00:45 -07:00
Yunxiao DuandMadelyn Olson b1d3b057f9 Delete redundant declaration clusterNodeCoversSlot and countKeysInSlot (#930)
Delete redundant declaration, clusterNodeCoversSlot and countKeysInSlot
has been declared in cluster.h

Signed-off-by: Yunxiao Du <me@jackdu.cn>
2024-09-03 09:00:45 -07:00
Madelyn Olson cd898efeae Revert repl backlog size back to 1mb for dual channel tests (#934)
There is a test that assumes that the backlog will get overrun, but
because of the recent changes to the default it no longer fails. It
seems like it is a bit flakey now though, so resetting the value in the
test back to 1mb. (This relates to the CoB of 1100k. So it should
consistently work with a 1mb limit).

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
2024-09-03 09:00:45 -07:00
Wen HuiandMadelyn Olson 11c301d02e Decline unsubscribe related command in non-subscribed mode (#759)
Now, when clients run the unsubscribe, sunsubscribe and punsubscribe
commands in the non-subscribed mode, it returns 0.
Indeed this is a bug, we should not allow client run these kind of
commands here.

Thus, this PR fixes this bug, but it is a break change for existing
clients

---------

Signed-off-by: hwware <wen.hui.ware@gmail.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson fe7716366e Make runtest-cluster support --io-threads option (#933)
In #764, we add a --io-threads mode in test, but forgot
to handle runtest-cluster, they are different framework.

Currently runtest-cluster does not support tags, and we
don't have plan to support it. And currently cluster tests
does not have any io-threads tests, so this PR just align
--io-threads option with #764.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
ce0d3812f9 Avoid to re-establish replication if node is already myself primary in CLUSTER REPLICATE (#884)
If n is already myself primary, there is no need to re-establish the
replication connection.

In the past we allow a replica node to reconnect with its primary via
this CLUSTER REPLICATE command, it will use psync. But since #885, we
will assume that a full sync is needed in this case, so if we don't do
this, the replica will always use full sync.

Signed-off-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Ping Xie <pingxie@google.com>
2024-09-03 09:00:45 -07:00
uriyageandMadelyn Olson 410df42ca6 Skip tracking clients OOM test when I/O threads are enabled (#764)
Fix feedback loop in key eviction with tracking clients when using I/O
threads.

Current issue:
Evicting keys while tracking clients or key space-notification exist
creates a feedback loop when using I/O threads:

While evicting keys we send tracking async writes to I/O threads,
preventing immediate release of tracking clients' COB memory
consumption.

Before the I/O thread finishes its write, we recheck used_memory, which
now includes the tracking clients' COB and thus continue to evict more
keys.

**Fix:**
We will skip the test for now while IO threads are active. We may
consider avoiding sending writes in `processPendingWrites` to I/O
threads for tracking clients when we are out of memory.

---------

Signed-off-by: Uri Yagelnik <uriy@amazon.com>
Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
2024-09-03 09:00:45 -07:00
Harkrishn PatroandMadelyn Olson 62a6989c4f Update README.md file to reference valkey.io (#931)
Update README.md since the project is no longer under construction, and
can reference the main website.

---------

Signed-off-by: Harkrishn Patro <harkrisp@amazon.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 5616624244 Set repl-backlog-size from 1mb to 10mb by default (#911)
The repl-backlog-size 1mb is too small in most cases, now network
transmission and bandwidth performance have improved rapidly in more
than ten years.

The bigger the replication backlog, the longer the replica can endure
the disconnect and later be able to perform a partial resynchronization.

Part of #653.

---------

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
Wen HuiandMadelyn Olson 6d235257f6 Fix Error in Daily CI -- reply-schemas-validator (#922)
Just add one more test for command "sentinel IS-PRIMARY-DOWN-BY-ADDR" to
make the reply-schemas-validator
run successfully.

Note: test result here
https://github.com/hwware/valkey/actions/runs/10457516111

Signed-off-by: hwware <wen.hui.ware@gmail.com>
2024-09-03 09:00:45 -07:00
zhenwei piandMadelyn Olson c179b82482 RDMA: Support user keepalive command (#916)
If the client side crashes by any issue or exits normally, the kernel
will try to disconnect RDMA QPs. Then the kernel of server side receives
CM packets, valkey-server handles CM disconnected event and close
connection.

However, there is a lack of keepalive mechanism from RDMA transport
layer. Once the kernel of client side crashes, the server side will not
be notified. To avoid this issue, valkey server sents Keepaliv command
periodically to detect any dead QPs.

An example of mlx-cx5:

```
 # RDMA: CQ handle error status: transport retry counter exceeded[0xc], opcode : 0x0
 # RDMA: CQ handle error status: transport retry counter exceeded[0xc], opcode : 0x0
 # RDMA: CQ handle error status: Work Request Flushed Error[0x5], opcode : 0x0
 # RDMA: CQ handle error status: Work Request Flushed Error[0x5], opcode : 0x0
 # RDMA: CQ handle error status: Work Request Flushed Error[0x5], opcode : 0x0
 # RDMA: CQ handle error status: Work Request Flushed Error[0x5], opcode : 0x0
```

Signed-off-by: zhenwei pi <pizhenwei@bytedance.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 764e74273a Fix data loss when replica do a failover with a old history repl offset (#885)
Our current replica can initiate a failover without restriction when
it detects that the primary node is offline. This is generally not a
problem. However, consider the following scenarios:

1. In slot migration, a primary loses its last slot and then becomes
a replica. When it is fully synchronized with the new primary, the new
primary downs.

2. In CLUSTER REPLICATE command, a replica becomes a replica of another
primary. When it is fully synchronized with the new primary, the new
primary downs.

In the above scenario, case 1 may cause the empty primary to be elected
as the new primary, resulting in primary data loss. Case 2 may cause the
non-empty replica to be elected as the new primary, resulting in data
loss and confusion.

The reason is that we have cached primary logic, which is used for psync.
In the above scenario, when clusterSetPrimary is called, myself will cache
server.primary in server.cached_primary for psync. In replicationGetReplicaOffset,
we get server.cached_primary->reploff for offset, gossip it and rank it,
which causes the replica to use the old historical offset to initiate
failover, and it get a good rank, initiates election first, and then is
elected as the new primary.

The main problem here is that when the replica has not completed full
sync, it may get the historical offset in replicationGetReplicaOffset.

The fix is to clear cached_primary in these places where full sync is
obviously needed, and let the replica use offset == 0 to participate
in the election. In this way, this unhealthy replica has a worse rank
and is not easy to be elected.

Of course, it is possible that it will be elected with offset == 0.
In the future, we may need to prohibit the replica with offset == 0
from having the right to initiate elections.

Another point worth mentioning, in above cases:
1. In the ROLE command, the replica status will be handshake, and the
offset will be -1.
2. Before this PR, in the CLUSTER SHARD command, the replica status will
be online, and the offset will be the old cached value (which is wrong).
3. After this PR, in the CLUSTER SHARD, the replica status will be loading,
and the offset will be 0.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 92ba3d3443 Correct RDB_EOF_MARK_SIZE usage where EOF mark is relevant (#925)
In these places we should use RDB_EOF_MARK_SIZE, but we mixed
it with CONFIG_RUN_ID_SIZE. This is not an issue since they are
all 40, just a cleanup.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
9b12bdc374 Make use of a single listNode pointer for blocking utility lists (#919)
Saves some memory (one pointer) in the client struct.

Since a client cannot be blocked multiple times, we can assume
it will be held in only one extra utility list, so it is ok to maintain
a union of these listNode references. 

Signed-off-by: Ran Shidlansik <ranshid@amazon.com>
Signed-off-by: ranshid <88133677+ranshid@users.noreply.github.com>
Signed-off-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Co-authored-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
gmbnomisandMadelyn Olson 6fa83d8905 Fix valgrind timing issue failure in replica-redirect test (#917)
Wait for the replica to become online before starting the actual test.

Signed-off-by: Simon Baatz <gmbnomis@gmail.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 4ab8949b26 Optimize linear search of WAIT and WAITAOF when unblocking the client (#787)
Currently, if the client enters a blocked state, it will be
added to the server.clients_waiting_acks list. When the client
is unblocked, that is, when unblockClient is called, we will
need to linearly traverse server.clients_waiting_acks to delete
the client, and this search is O(N).

When WAIT (or WAITAOF) is used extensively in some cases, this
O(N) search may be time-consuming. We can remember the list node
and store it in the blockingState struct and it can avoid the
linear search in unblockClientWaitingReplicas.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
Wen HuiandMadelyn Olson 5a2ebc9112 Add 4 commands for sentinel and update most test cases and json files (#789)
Add 4 new commands for Sentinel (reference
https://github.com/valkey-io/valkey/issues/36)

Sentinel GET-PRIMARY-ADDR-BY-NAME
Sentinel PRIMARY
Sentinel PRIMARIES
Sentinel IS-PRIMARY-DOWN-BY-ADDR

and deprecate 4 old commands:

Sentinel GET-MASTER-ADDR-BY-NAME
Sentinel MASTER
Sentinel MASTERS
Sentinel IS-MASTER-DOWN-BY-ADDR

and all sentinel tests pass here
https://github.com/hwware/valkey/actions/runs/9962102363/job/27525124583

Note: 

1. runtest-sentinel pass all test cases
2. I finished a sentinel rolling upgrade test: 1 primary 2 replicas 3
sentinel
   there are 4 steps in this test scenario: 
step 1: all 3 sentinel nodes run old sentinel, shutdown primary, and
then new primary can be voted successfully.
step 2: replace sentinel 1 with new sentinel bin file, and then shutdown
primary, and then another new primary can be voted successfully
step 3: replace sentinel 2 with new sentinel bin file, and then shutdown
primary, and then another new primary can be voted successfully
step 4: replace sentinel 3 with new sentinel bin file, and then shutdown
primary, and then another new primary can be voted successfully
   
We can see, even mixed version sentinel running, whole system still
works.

---------

Signed-off-by: hwware <wen.hui.ware@gmail.com>
2024-09-03 09:00:45 -07:00
f4a72e7482 Add lfu support for DEBUG OBJECT command, added lfu_freq and lfu_access_time_minutes fields (#479)
For `debug object` command, we use `val->lru` but ignore the `lfu` mode.  
So in `lfu` mode, `debug object` would return meaningless `lru` descriptions. 

Added two new fields lfu_freq and lfu_access_time_minutes.

Signed-off-by: jiangyujie.jyj <yjjiang1996@163.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 4288a08782 Make a light weight version of DEBUG OBJECT, add FAST option (#881)
Adding FAST option to DEBUG OBJECT command.

The light version only shows the light weight infomation,
which mostly O(1). The pre-existing version that show more
stats such as serializedlength sometimes is time consuming.

This should allow looking into debug stats (the key expired
but not deleted), even on huge object, on which we're afraid
to run the command for fear of causing a server freeze.

Somehow like 98ee400b98.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 608dc468de Skip IPv6 tests when TCLSH version is < 8.6 (#910)
In #786, we did skip it in the daily, but not for the others.
When running ./runtest on MacOS, we will get the failure.
```
couldn't open socket: host is unreachable (nodename nor servname provided, or not known)
```

The reason is that TCL 8.5 doesn't support ipv6, so we skip tests
tagged with ipv6. This also revert #786.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
secwallandMadelyn Olson d4d3ac84d1 Log unexpected $ENDOFF responses in dual channel replication (#839)
I've tried to test a dual channel replication but forgot to add +sync
for my replication user. As a result replica entered silent cycle like
this:
```
* Connecting to PRIMARY 127.0.0.1:6379
* PRIMARY <-> REPLICA sync started
* Non blocking connect for SYNC fired the event.
* Primary replied to PING, replication can continue...
* Trying a partial resynchronization (request ...)
* PSYNC is not possible, initialize RDB channel.
* Aborting dual channel sync
```

And primary got endless cycle like this:
```
* Replica 127.0.0.1:6380 asks for synchronization
* Partial resynchronization not accepted: Replication ID mismatch (Replica asked for '...', my replication IDs are '...' and '...')
* Replica 127.0.0.1:6380 is capable of dual channel synchronization, and partial sync isn't possible. Full sync will continue with dedicated RDB channel.
```

There was no way to understand that replication user is missing +sync
acl on notice log level. With this one-line change we get a warning
message in our replica log.

---------

Signed-off-by: secwall <secwall@yandex-team.ru>
2024-09-03 09:00:45 -07:00
Pieter CailliauandMadelyn Olson fd10fdc8f5 Copyright update to reflect IP transfer from salvatore to Redis (#740)
Update references of copyright being assigned to Salvatore when it was
transferred to Redis Ltd. as per
https://github.com/valkey-io/valkey/issues/544.

---------

Signed-off-by: Pieter Cailliau <pieter@redis.com>
2024-09-03 09:00:45 -07:00
Salvatore MesoracaandMadelyn Olson 080069dabd Prevent later accesses to unallocated memory (#907)
A pointer to dtype is stored in the dict forever.
dtype is stack-allocated while the dict created is global.
The dict (and the pointer to dtype in it) will live past the lifetime of
dtype.
clusterManagerLinkDictType is a global object that has the same values
as dtype.

Signed-off-by: Salvatore Mesoraca <salvatore.mesoraca@aiven.io>
2024-09-03 09:00:45 -07:00
zhaozhao.zzandMadelyn Olson 0ef32c8c40 To avoid bouncing -REDIRECT during FAILOVER (#871)
Fix #821

During the `FAILOVER` process, when conditions are met (such as when the
force time is reached or the primary and replica offsets are
consistent), the primary actively becomes the replica and transitions to
the `FAILOVER_IN_PROGRESS` state. After the primary becomes the replica,
and after handshaking and other operations, it will eventually send the
`PSYNC FAILOVER` command to the replica, after which the replica will
become the primary. This means that the upgrade of the replica to the
primary is an asynchronous operation, which implies that during the
`FAILOVER_IN_PROGRESS` state, there may be a period of time where both
nodes are replicas. In this scenario, if a `-REDIRECT` is returned, the
request will be redirected to the replica and then redirected back,
causing back and forth redirection. To avoid this situation, during the
`FAILOVER_IN_PROGRESS state`, we temporarily suspend the clients that
need to be redirected until the replica truly becomes the primary, and
then resume the execution.

---------

Signed-off-by: zhaozhao.zz <zhaozhao.zz@alibaba-inc.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 2123819a71 Change server.daylight_active to an atomic variable (#876)
We are updating this variable in the main thread, and the
child threads can printing the logs at the same time. This
generating a warning in SANITIZER=thread:
```
WARNING: ThreadSanitizer: data race (pid=74208)
  Read of size 4 at 0x000102875c10 by thread T3:
    #0 serverLogRaw <null>:52173615 (valkey-server:x86_64+0x10003c556)
    #1 _serverLog <null>:52173615 (valkey-server:x86_64+0x10003ca89)
    #2 bioProcessBackgroundJobs <null>:52173615 (valkey-server:x86_64+0x1001402c9)

  Previous write of size 4 at 0x000102875c10 by main thread (mutexes: write M0):
    #0 afterSleep <null>:52173615 (valkey-server:x86_64+0x10004989b)
    #1 aeProcessEvents <null>:52173615 (valkey-server:x86_64+0x100031e52)
    #2 main <null>:52173615 (valkey-server:x86_64+0x100064a3c)
    #3 start <null>:52173615 (dyld:x86_64+0xfffffffffff5c365)
    #4 start <null>:52173615 (dyld:x86_64+0xfffffffffff5c365)
```

The refresh of daylight_active is not real time, we update
it in aftersleep, so we don't need a strong synchronization,
so using memory_order_relaxed. But also noted we are doing
load/store operations only for daylight_active, which is an
aligned 32-bit integer, so using memory_order_relaxed will
not provide more consistency than what we have today.

So this is just a cleanup that to clear the warning.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
Amit NaglerandMadelyn Olson ebee48c4a2 Fix dual-channel replication test under valgrind (#904)
Test dual-channel-replication primary gets cob overrun during replica
rdb load` fails during the Valgrind run. This is due to the load
handlers disconnecting before the tests complete, resulting in a low
primary COB. Increasing the handlers' timeout should resolve this issue.

Failure:
https://github.com/valkey-io/valkey/actions/runs/10361286333/job/28681321393

Server logs reveals that the load handler clients were disconnected
before the test started

Also the two previus test took about 20 seconds which is the handler
timeout.

---------

Signed-off-by: naglera <anagler123@gmail.com>
Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson c1da6e3993 Better messages when valkey-cli cluster --fix meet value check failed (#867)
The clusterManagerCompareKeysValues is introduced in
8f01adfb15, which calls
DEBUG DIGEST-VALUE to check whether the value of the
source node and the target node are consistent.

However, the DEBUG DIGEST-VALUE command is not supported
in older version, such as version 4, and the node will return
unknown subcommand. Or the DEBUG command can be disabled
in version 7, and the node will return DEBUG not allowed.

In these cases, we need to output friendly message to
allow users to proceed to the next step, instead of just
outputing `Value check failed!`.

Unknown subcommand example:
```
*** Target key exists
*** Checking key values on both nodes...
Node 127.0.0.1:30001 replied with error:
ERR unknown subcommand or wrong number of arguments for 'DIGEST-VALUE'. Try DEBUG HELP.
Node 127.0.0.1:30003 replied with error:
ERR unknown subcommand or wrong number of arguments for 'DIGEST-VALUE'. Try DEBUG HELP.
*** Value check failed!
DEBUG DIGEST-VALUE command is not supported.
You can relaunch the command with --cluster-replace option to force key overriding.
```

DEBUG not allowed example:
```
*** Target key exists
*** Checking key values on both nodes...
Node 127.0.0.1:30001 replied with error:
ERR DEBUG command not allowed. If the enable-debug-command option is ...
Node 127.0.0.1:30003 replied with error:
ERR DEBUG command not allowed. If the enable-debug-command option is ...
*** Value check failed!
DEBUG command is not allowed.
You can turn on the enable-debug-command option.
Or you can relaunch the command with --cluster-replace option to force key overriding.
```

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
RayacooandMadelyn Olson a651e8e683 Optimize ZUNION[STORE] command by removing unnecessary accumulator dict (#829)
In the past implementation of `ZUNION` and `ZUNIONSTORE` commands, we
first create a temporary dict called `accumulator`. After adding all
member-score mappings to `accumulator`, we still need to convert
`accumulator` back to the final dict `dstzset->dict`. However, we can
directly use `dstzset->dict` to avoid the additional copy operation.

This PR removes the `accumulator` dict and directly uses` dstzset->dict
`to store the member-score mappings.

- **Test**
First, I added 300 unique elements to two sorted sets called
'zunion_test1' and 'zunion_test2'. Then, I tested `zunion` and
`zunionstore` on these two sorted sets. The test results shown below
indicate that the performance of both zunion and zunionstore improved
about 31%.

### ZUNION
#### unstable
```
./valkey-benchmark -P 10 -n 100000 zunion 2 zunion_test1 zunion_test2

Summary:
  throughput summary: 2713.41 requests per second
  latency summary (msec):
          avg       min       p50       p95       p99       max
      146.252     3.464   153.343   182.015   184.959   192.895
```
#### This PR
```
./valkey-benchmark -P 10 -n 100000 zunion 2 zunion_test1 zunion_test2

Summary:
  throughput summary: 3543.84 requests per second
  latency summary (msec):
          avg       min       p50       p95       p99       max
      108.259     2.984   114.239   141.695   145.151   160.255
```
### ZUNIONSTORE
#### unstable
```
./valkey-benchmark -P 10 -n 100000 zunionstore out 2 zunion_test1 zunion_test2

Summary:
  throughput summary: 3168.07 requests per second
  latency summary (msec):
          avg       min       p50       p95       p99       max
      157.511     3.368   183.167   189.311   193.535   231.679
```
#### This PR
```
./valkey-benchmark -P 10 -n 100000 zunionstore out 2 zunion_test1 zunion_test2

Summary:
  throughput summary: 4144.73 requests per second
  latency summary (msec):
          avg       min       p50       p95       p99       max
      120.374     2.648   141.823   149.119   153.855   183.167
```

---------

Signed-off-by: RayCao <zisong.cw@alibaba-inc.com>
Signed-off-by: zisong.cw <zisong.cw@alibaba-inc.com>
2024-09-03 09:00:45 -07:00
511868bb07 Add debug-context config (#874)
A configuration option with zero impact on server operation but is
printed out on server crash and can be accessed by gdb for debugging. It
can be used by the user/operator to store any free-form string. This
string will persist as long as the server is running and will be
accessible in the following ways:

And printed in crash reports:
```
------ CONFIG DEBUG OUTPUT ------
lazyfree-lazy-eviction no
...
io-threads-do-reads yes
debug-context "test2"
proto-max-bulk-len 512mb
```

---------

Signed-off-by: Eran Liberty <eranl@amazon.com>
Co-authored-by: Eran Liberty <eranl@amazon.com>
2024-09-03 09:00:45 -07:00
2b33b547d3 Fix dual-channel-replication related issues (#837)
- Fix TLS bug where connection were shutdown by primary's main process
while the child process was still writing- causing main process to be
blocked.
- TLS connection fix -file descriptors are set to blocking mode in the
main thread, followed by a blocking write. This sets the file
descriptors to non-blocking if TLS is used (see `connTLSSyncWrite()`)
(@xbasel).
- Improve the reliability of dual-channel tests. Modify the pause
mechanism to verify process status directly, rather than relying on log.
- Ensure that `server.repl_offset` and `server.replid` are updated
correctly when dual channel synchronization completes successfully.
Thist led to failures in replication tests that validate replication IDs
or compare replication offsets.

---------

Signed-off-by: naglera <anagler123@gmail.com>
Signed-off-by: naglera <58042354+naglera@users.noreply.github.com>
Signed-off-by: xbasel <103044017+xbasel@users.noreply.github.com>
Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Signed-off-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: ranshid <88133677+ranshid@users.noreply.github.com>
Co-authored-by: xbasel <103044017+xbasel@users.noreply.github.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
nagleraandMadelyn Olson 143a32d683 Add debug assert on duplicate freeClientAsync (#896)
When debug assert mode enabled, verify that we don't insert the same
client twice into server.clients_to_close.

Signed-off-by: naglera <anagler123@gmail.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson f5577e7419 Dual channel replication should not update lastbgsave_status when transfer error (#811)
Currently lastbgsave_status is used in bgsave or disk-replication,
and the target is the disk. In #60, we update it when transfer error,
i think it is mainly used in tests, so we can use log to replace it.

It changes lastbgsave_status to err in this case, but it is strange
that it does not set ok or err in the above if and the following else.
Also noted this will affect stop-writes-on-bgsave-error.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson 585e60edd9 Correctly recode client infomation to the slowlog when runing script (#805)
Currently when we are runing a script, we will passing a fake client.
So if the command executed in the script is recoded in the slowlog,
the client's ip:port and name information will be empty.

before:
```
127.0.0.1:6379> client setname myclient
OK
127.0.0.1:6379> config set slowlog-log-slower-than 0
OK
127.0.0.1:6379> eval "redis.call('ping')" 0
(nil)
127.0.0.1:6379> slowlog get 2
1) 1) (integer) 2
   2) (integer) 1721314289
   3) (integer) 96
   4) 1) "eval"
      2) "redis.call('ping')"
      3) "0"
   5) "127.0.0.1:61106"
   6) "myclient"
2) 1) (integer) 1
   2) (integer) 1721314289
   3) (integer) 4
   4) 1) "ping"
   5) ""
   6) ""
```

after:
```
127.0.0.1:6379> client setname myclient
OK
127.0.0.1:6379> config set slowlog-log-slower-than 0
OK
127.0.0.1:6379> eval "redis.call('ping')" 0
(nil)
127.0.0.1:6379> slowlog get 2
1) 1) (integer) 2
   2) (integer) 1721314371
   3) (integer) 53
   4) 1) "eval"
      2) "redis.call('ping')"
      3) "0"
   5) "127.0.0.1:51983"
   6) "myclient"
2) 1) (integer) 1
   2) (integer) 1721314371
   3) (integer) 1
   4) 1) "ping"
   5) "127.0.0.1:51983"
   6) "myclient"
```

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
Harkrishn PatroandMadelyn Olson 81965923be Check if the server is currently running the feature before cron run (#838)
I think we should first check if the server is currently enabled in
cluster mode or if it has modules loaded prior to the throttled cron run
(`run_with_period`) condition.

Signed-off-by: Harkrishn Patro <harkrisp@amazon.com>
2024-09-03 09:00:45 -07:00
Harkrishn PatroandMadelyn Olson af3eb77b74 Assert network bytes out for replication slot stat computation is only allowed on primary (#847)
Added an assertion to avoid incorrect usage of the network bytes out for
replication code flow in slot stats computation.

Signed-off-by: Harkrishn Patro <harkrisp@amazon.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson f0754d70f1 Improve cluster cant failover log conditions (#780)
This PR adjusts the logging conditions of clusterLogCantFailover
in this two ways.

1. For the same cant_failover_reason, we will print the log once
in CLUSTER_CANT_FAILOVER_RELOG_PERIOD, but its value is 10s, which
is a bit long, shorten it to 1s, so we can better track its state.
We get to see the system making progress by watching the message.
Using 1s also covers pretty much all cases as i don't see a reason
for using a <1s node timeout, test or prod.

2. We will not print logs before the nolog_fail_time, its value
is cluster-node-timeout+5000. This may casue us to lose some logs,
for example, if cluster-node-timeout is small, auth_timeout will
be 2000, and auth_retry_time will be 4000. In this case, we will
lose all the reasons during the election if the failover is timedout.
So remove the nolog_fail_time logic, since we still do have the
CLUSTER_CANT_FAILOVER_RELOG_PERIOD logic, we won't print too many
logs.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
Yury-FridlyandandMadelyn Olson c4d6b45f50 Fix CI concurrency (#849)
Few CI improvements witch will reduce occupation CI queue and eliminate
stale runs.

1. Kill CI jobs on PRs once PR branch gets a new push. This will prevent
situation happened today - a huge job triggered twice in less than an
hour and occupied all **org** (for all repositories) runners queue for
the rest of the day (see pic). This completely blocked valkey-glide
team.
2. Distribute nightly croned jobs on time to prevent them running
together. Keep in mind, cron's TZ is UTC, so midnight tasks incur
developers located in other timezones.

This must be backported to all release branches (`valkey-x.y` and `x.y`)

![image](https://github.com/user-attachments/assets/923d8237-3cb7-42f5-80c8-5322b3f5187d)

---------

Signed-off-by: Yury-Fridlyand <yury.fridlyand@improving.com>
2024-09-03 09:00:45 -07:00
Harkrishn PatroandMadelyn Olson 8130caf97f Update sentinel conf access string to allow hello channel access (#854)
This example of a minimal user account in your Valkey server
for Sentinel is incorrect. If you add this ACL as-is to your
valkey users.acl, valkey will add resetchannels -@all before
the +client which prevents sentinel from publishing messages
to the __sentinel__:hello pubsub for sentinel discovery.

Fix #744.

Signed-off-by: Harkrishn Patro <harkrisp@amazon.com>
2024-09-03 09:00:45 -07:00
Wen HuiandMadelyn Olson 08ce534557 Update redis.conf to valkey.conf in log message (#855)
Update redis.conf to valkey.conf

Signed-off-by: hwware <wen.hui.ware@gmail.com>
2024-09-03 09:00:45 -07:00
BinbinandMadelyn Olson cb7edc4abc Fix outdated comment of migrate in valkey-cli --cluster (#864)
After 906dac85c2 the comment is outdated.

Signed-off-by: Binbin <binloveplay1314@qq.com>
2024-09-03 09:00:45 -07:00
Madelyn Olson 5d9ac5948f Disable empty shard slot migration test until test is de-flaked (#859)
We have a number of test failures in the empty shard migration which
seem to be related to race conditions in the failover, but could be more
pervasive. For now disable the tests to prevent so many false negative
test failures.

Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
2024-09-03 09:00:45 -07:00
52940f0899 Valkey 8.0 RC1 release notes (#850)
Signed-off-by: Ping Xie <pingxie@google.com>
Signed-off-by: Ping Xie <pingxie@outlook.com>
Signed-off-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
2024-08-01 08:59:43 -07:00
1215 changed files with 52275 additions and 133949 deletions
-76
View File
@@ -1,76 +0,0 @@
format:
_help_line_width:
- How wide to allow formatted cmake files
line_width: 120
_help_tab_size:
- How many spaces to tab for indent
tab_size: 4
_help_use_tabchars:
- If true, lines are indented using tab characters (utf-8
- 0x09) instead of <tab_size> space characters (utf-8 0x20).
- In cases where the layout would require a fractional tab
- character, the behavior of the fractional indentation is
- governed by <fractional_tab_policy>
use_tabchars: false
_help_separate_ctrl_name_with_space:
- If true, separate flow control names from their parentheses
- with a space
separate_ctrl_name_with_space: true
_help_min_prefix_chars:
- If the statement spelling length (including space and
- parenthesis) is smaller than this amount, then force reject
- nested layouts.
min_prefix_chars: 4
_help_max_prefix_chars:
- If the statement spelling length (including space and
- parenthesis) is larger than the tab width by more than this
- amount, then force reject un-nested layouts.
max_prefix_chars: 10
_help_max_lines_hwrap:
- If a candidate layout is wrapped horizontally but it exceeds
- this many lines, then reject the layout.
max_lines_hwrap: 2
_help_line_ending:
- What style line endings to use in the output.
line_ending: unix
_help_command_case:
- Format command names consistently as 'lower' or 'upper' case
command_case: lower
_help_keyword_case:
- Format keywords consistently as 'lower' or 'upper' case
keyword_case: unchanged
_help_always_wrap:
- A list of command names which should always be wrapped
always_wrap: []
_help_enable_sort:
- If true, the argument lists which are known to be sortable
- will be sorted lexicographicall
enable_sort: true
_help_autosort:
- If true, the parsers may infer whether or not an argument
- list is sortable (without annotation).
autosort: false
_help_require_valid_layout:
- By default, if cmake-format cannot successfully fit
- everything into the desired linewidth it will apply the
- last, most aggressive attempt that it made. If this flag is
- True, however, cmake-format will print error, exit with non-
- zero status code, and write-out nothing
require_valid_layout: false
_help_layout_passes:
- A dictionary mapping layout nodes to a list of wrap
- decisions. See the documentation for more information.
layout_passes: {}
encode:
_help_emit_byteorder_mark:
- If true, emit the unicode byte-order mark (BOM) at the start
- of the file
emit_byteorder_mark: false
_help_input_encoding:
- Specify the encoding of the input file. Defaults to utf-8
input_encoding: utf-8
_help_output_encoding:
- Specify the encoding of the output file. Defaults to utf-8.
- Note that cmake only claims to support utf-8 so be careful
- when using anything else
output_encoding: utf-8
+16 -10
View File
@@ -2,19 +2,19 @@
[files]
extend-exclude = [
".git/",
"deps/",
# crc16_slottable is primarily pre-generated random strings.
"src/crc16_slottable.h",
# 00-RELEASENOTES includes non-English names that might be flagged as typos in other contexts.
"00-RELEASENOTES",
]
ignore-hidden = false
[default.extend-words]
advices = "advices"
exat = "exat"
optin = "optin"
ro = "ro"
smove = "smove"
Parth = "Parth" # seems like the spellchecker does not like it is similar to "Path"
nd = "nd"
[default]
extend-ignore-re = [
@@ -28,7 +28,6 @@ extend-ignore-re = [
"D4C4DAA4", # sha1.c
"Georg Nees",
"\\[l\\]ist", # eval.c
'"LKE"', # test_rax.c
]
[type.tcl]
@@ -36,22 +35,26 @@ extend-ignore-re = [
"DUMPed",
]
[type.sv.extend-identifiers]
# sv = .h
module_gil_acquring = "module_gil_acquring"
[type.c.extend-identifiers]
advices = "advices"
ang = "ang"
clen = "clen"
fle = "fle"
module_gil_acquring = "module_gil_acquring"
nd = "nd"
ot = "ot"
[type.tcl.extend-identifiers]
fo = "fo"
oll = "oll"
stressers = "stressers"
[type.sv.extend-identifiers]
# sv = .h
fo = "fo"
[type.sv.extend-words]
# sv = .h
fo = "fo"
seeked = "seeked"
[type.c.extend-words]
@@ -59,10 +62,12 @@ arange = "arange"
fo = "fo"
frst = "frst"
limite = "limite"
lke = "lke"
pathc = "pathc"
pn = "pn"
seeked = "seeked"
tre = "tre"
ws = "ws"
[type.systemd.extend-words]
# systemd = .conf
@@ -70,4 +75,5 @@ ake = "ake"
[type.tcl.extend-words]
fo = "fo"
lst = "lst"
tre = "tre"
-3
View File
@@ -12,6 +12,3 @@ c41dd77a3e93e02be3c4bc75d8c76b7b4169a4ce
# Removed terms `master` and `slave` from the source code (#591)
54c97479356ecf41b4b63733494a1be2ab919e17
# Set ColumnLimit to 0 and reformat (#1045)
af811748e7819a5ac31a6df4b21622aa58c64ae4
+1 -1
View File
@@ -2,7 +2,7 @@
name: Bug report
about: Help us improve by reporting a bug
title: '[BUG]'
labels: ['bug']
labels: ''
assignees: ''
---
+4 -7
View File
@@ -1,17 +1,14 @@
blank_issues_enabled: true
contact_links:
- name: Questions?
url: https://github.com/kv-io/kv/discussions
url: https://github.com/valkey-io/valkey/discussions
about: Ask and answer questions on GitHub Discussions.
- name: Chat with us on Discord?
url: https://discord.gg/zbcPa5umUB
about: We are on Discord!
- name: Chat with us on Matrix?
url: https://matrix.to/#/#kv:matrix.org
url: https://matrix.to/#/#valkey:matrix.org
about: We are on Matrix too!
- name: Chat with us on Slack?
url: https://join.slack.com/t/kv-oss-developer/shared_invite/zt-2nxs51chx-EB9hu9Qdch3GMfRcztTSkQ
about: We are on Slack too!
- name: Documentation issue?
url: https://github.com/kv-io/kv-doc/issues
about: Report it on the kv-doc repo.
url: https://github.com/valkey-io/valkey-doc/issues
about: Report it on the valkey-doc repo.
-1
View File
@@ -1,7 +1,6 @@
name: Crash report
description: Submit a crash report
title: '[CRASH] <short description>'
labels: ['bug']
body:
- type: markdown
attributes:
+1 -1
View File
@@ -2,7 +2,7 @@
name: Feature request
about: Suggest a feature
title: '[NEW]'
labels: ['enhancement']
labels: ''
assignees: ''
---
-24
View File
@@ -1,24 +0,0 @@
---
name: Test failure
about: Report a failing or a flaky test
title: '[TEST-FAILURE] <short description>'
labels: ['test-failure']
assignees: ''
---
<!--Before submitting a test failure report, please check open issues to ensure this failure has not already been reported. -->
**Summary**
A short description of the failure.
**Failing test(s)**
- Test name:
- CI link(s):
**Error stack trace**
A relevant stack trace for the test failure.
> **Tip:** Copy and paste the full stack trace from the CI output into your issue, as CI links may expire over time.
@@ -1,20 +0,0 @@
[
{
"duration": 180,
"keyspacelen": [3000000],
"data_sizes": [16,96],
"pipelines": [1, 10],
"clients": [1600],
"commands": [
"SET",
"GET"
],
"cluster_mode": false,
"tls_mode": false,
"warmup": 30,
"io-threads": [1,9],
"benchmark-threads": 90,
"server_cpu_range": "0-8",
"client_cpu_range": "96-191"
}
]
@@ -1,20 +0,0 @@
[
{
"duration": 180,
"keyspacelen": [3000000],
"data_sizes": [16,96],
"pipelines": [1, 10],
"clients": [1600],
"commands": [
"SET",
"GET"
],
"cluster_mode": false,
"tls_mode": false,
"warmup": 30,
"io-threads": [1,9],
"benchmark-threads": 90,
"server_cpu_range": "0-8",
"client_cpu_range": "144-191,48-95"
}
]
-30
View File
@@ -1,30 +0,0 @@
# KV Project Instructions
You are an expert code reviewer for the KV project. Provide helpful, constructive feedback on code quality, safety, and adherence to project standards.
## 1. Review Tone & Focus
- **Tone:** Be professional, direct, constructive, and empathetic.
- **Focus:** Critique the *code*, never the *person*.
- **Constructive:** Suggest improvements, explain *why*, provide examples.
## 2. Critical Checks
- **DCO:** **Flag missing** `Signed-off-by: Name <email>` in commits. Every commit needs it.
- **Security:** If PR fixes a security vulnerability, flag it: "Security fixes should be reported privately to security@hanzo.ai, not via public PRs."
## 3. Major Decision Detection
Flag PRs that appear to be "Technical Major Decisions" requiring TSC consensus:
- Fundamental changes to core datastructures
- New data structures or APIs
- Backward compatibility breaks
- New user-visible fields requiring long-term maintenance
- New external libraries affecting runtime behavior
**Action:** Comment mentioning **@core-team** that this appears to require TSC review and ask if consensus was reached in a linked Issue.
## 4. Documentation Reminder
If PR changes user-facing behavior (new commands, changed semantics, new config):
- **Remind** author that docs at [kv-doc](https://github.com/hanzoai/kv-doc) may need updating.
- **Suggest** linking PR to related Issue with "Fixes #xyz" pattern if applicable.
## 5. Governance Changes
**ANY change to `GOVERNANCE.md`** requires special attention - comment mentioning **@core-team** for review.
@@ -1,48 +0,0 @@
---
applyTo:
- "src/**/*.{c,h}"
- "kv.conf"
---
# KV Core Engine Review Standards
Apply these standards to core engine C code. Do NOT apply to `deps/` (vendored dependencies).
## 1. Code Style (from DEVELOPMENT_GUIDE.md)
- **Formatting:** Follow clang-format (4-space indent, no tabs, braces attached).
- **Comments:**
- C-style `/* ... */` for single or multi-line.
- C++ `//` only for single-line.
- Multi-line: align leading `*`, final `*/` on last text line.
- Document *why* code exists, not just *what*. Document all functions.
- **Line Length:** Keep below 90 characters when reasonable.
- **Types:** Use the boolean type for true/false values.
- **Static:** Use `static` for file-local functions.
## 2. Naming Conventions
- **Variables:** `snake_case` or lowercase (e.g. `cached_reply`, `keylen`).
- **Functions:** `camelCase` or `namespace_camelCase` (e.g. `createStringObject`, `IOJobQueue_isFull`).
- **Macros:** `UPPER_CASE` (e.g. `MAKE_CMD`).
- **Structures:** `camelCase` (e.g. `user`).
## 3. Safety & Correctness
- **Memory:** Strict check for buffer overflows and leaks.
- **Strings:** Validate `sds` string handling.
- **Concurrency:** Verify thread safety in threaded I/O paths.
## 4. Design Guidelines
- **Configuration:** Avoid new configs if heuristics suffice. Only add for explicit trade-offs (CPU vs memory).
- **Metrics:** No new metrics on hot paths without zero-overhead proof.
- **PR Scope:** Separate refactoring from functional changes for easier backporting.
## 5. Testing & Documentation
- **Unit Tests:** Required for data structures in `src/unit/`. Test files should follow `test_*.c` naming.
- **Integration Tests:** Required for commands in `tests/`.
- **Command Changes:** New/modified commands need corresponding updates in `src/commands/*.json`.
- **New C Files:** Remind to update `CMakeLists.txt` when adding new `.c` source files.
- **License:** New files need BSD-3-Clause header. Material changes (>100 lines) also require it.
- **Documentation:** User-facing changes need docs at [kv-doc](https://github.com/hanzoai/kv-doc).
## 6. Critical Escalation
- **Trigger:** Changes to `cluster*.c`, `replication.c`, `rdb.c`, `aof.c`.
- **Action:** Comment mentioning **@core-team** to request architectural review.
@@ -1,23 +0,0 @@
---
applyTo:
- "tests/**/*.tcl"
---
# KV Integration Test Review Standards
Apply these standards to Tcl-based integration tests (from DEVELOPMENT_GUIDE.md).
## 1. Test Organization
- **Cluster:** Use `tests/unit/cluster/` (NOT legacy `tests/cluster/` which is deprecated).
- **Coverage:** All contributions should include tests. New commands require integration tests.
- **Naming:** Use descriptive test names that explain what is being tested.
## 2. Test Quality
- **Isolation:** Tests should not depend on execution order.
- **Cleanup:** Ensure proper cleanup of resources and temporary files.
- **Assertions:** Use clear assertions with meaningful error messages.
## 3. Best Practices
- **Readability:** Keep tests simple and focused on one scenario.
- **Reliability:** Avoid timing-dependent tests; use proper synchronization.
- **Documentation:** Comment complex test scenarios to explain intent.
@@ -1,24 +0,0 @@
---
applyTo:
- "utils/**/*"
---
# KV Utilities Review Standards
Apply these standards to utility scripts and tools.
## 1. Script Quality
- **Portability:** Scripts should work across common platforms (Linux, macOS).
- **Error Handling:** Check for errors and provide clear error messages.
- **Documentation:** Include usage instructions in comments or help text.
## 2. Code Standards
- **Python:** Follow PEP 8 style guidelines.
- **Ruby:** Follow standard Ruby conventions.
- **Shell:** Use shellcheck-compatible patterns.
- **C Tools:** Follow LLVM style (4-space indent, no tabs).
## 3. Best Practices
- **Dependencies:** Minimize external dependencies.
- **Safety:** Validate inputs and avoid destructive operations without confirmation.
- **Maintainability:** Keep utilities simple and well-commented.
-14
View File
@@ -1,14 +0,0 @@
name: Auto Author Assign
on:
pull_request_target:
types: [opened, reopened]
permissions:
pull-requests: write
jobs:
assign-author:
runs-on: ubuntu-latest
steps:
- uses: toshimaru/auto-author-assign@4d585cc37690897bd9015942ed6e766aa7cdb97f # v3.0.1
-173
View File
@@ -1,173 +0,0 @@
name: On-Demand Labeled Benchmark
on:
pull_request_target:
types: [labeled]
concurrency:
group: ec2-al-2023-pr-benchmarking-arm64
cancel-in-progress: false
defaults:
run:
shell: "bash -Eeuo pipefail -x {0}"
permissions:
contents: read
pull-requests: write
issues: write
jobs:
benchmark:
if: |
github.event.action == 'labeled' && github.event.label.name == 'run-benchmark' &&
github.repository == 'kv-io/kv'
runs-on: ["self-hosted", "ec2-al-2023-pr-benchmarking-arm64"]
timeout-minutes: 7200
steps:
- name: Checkout kv
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
path: kv
fetch-depth: 0
ref: ${{ github.event.pull_request.merge_commit_sha }}
persist-credentials: false
- name: Checkout kv-perf-benchmark
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ${{ github.repository_owner }}/kv-perf-benchmark
path: kv-perf-benchmark
fetch-depth: 1
persist-credentials: false
- name: Checkout kv for latest benchmark.
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: kv-io/kv
ref: "unstable"
path: kv_latest
fetch-depth: 0
persist-credentials: false
- name: Set up Python
uses: kishaningithub/setup-python-amazon-linux@a326cdc792983fe0fbd04c81d3d62b59b6123a6c # v1.1.0
with:
python-version: "3.10"
cache: "pip"
- name: Install dependencies
working-directory: kv-perf-benchmark
run: |
sudo dnf groupinstall "Development Tools" -y
sudo dnf install -y gcc gcc-c++ make \
python3-devel \
openssl-devel \
bzip2-devel \
libffi-devel
pip install -r requirements.txt
- name: Build latest kv_latest
working-directory: kv_latest
run: |
echo "Building latest kv-benchmark for latest benchmark executable..."
make distclean || true
make -j
if [[ -f "src/kv-benchmark" ]]; then
echo "Successfully built latest kv-benchmark"
ls -la src/kv-benchmark
./src/kv-benchmark --version || echo "Version check completed"
else
echo "Failed to build kv-benchmark"
exit 1
fi
KV_BENCHMARK_PATH="$(pwd)/src/kv-benchmark"
echo "KV_BENCHMARK_PATH=$KV_BENCHMARK_PATH" >> $GITHUB_ENV
echo "Latest kv-benchmark path: $KV_BENCHMARK_PATH"
- name: Run benchmarks
working-directory: kv-perf-benchmark
run: |
CONFIG_FILE="../kv/.github/benchmark_configs/benchmark-config-arm.json"
# Base benchmark arguments
BENCHMARK_ARGS=(
--config "$CONFIG_FILE"
--commits "${{ github.event.pull_request.merge_commit_sha }}"
--baseline "${{ github.event.pull_request.base.ref }}"
--kv-benchmark-path "$KV_BENCHMARK_PATH"
--target-ip ${{ secrets.EC2_ARM64_IP }}
--kv-path "../kv"
--results-dir "results"
--runs 5
)
# Run benchmark
python ./benchmark.py "${BENCHMARK_ARGS[@]}"
- name: Compare results
working-directory: kv-perf-benchmark
run: |
python ./utils/compare_benchmark_results.py \
--baseline ./results/${{ github.event.pull_request.base.ref }}/metrics.json \
--new ./results/${{ github.event.pull_request.merge_commit_sha }}/metrics.json \
--output ../comparison.md \
--metrics rps
- name: Upload artifacts
if: always()
continue-on-error: true
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: benchmark-results
path: |
./kv-perf-benchmark/results/${{ github.event.pull_request.merge_commit_sha }}/metrics.json
./kv-perf-benchmark/results/${{ github.event.pull_request.base.ref }}/metrics.json
comparison.md
- name: Comment PR with results
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const fs = require('fs');
const body = fs.readFileSync('comparison.md', 'utf8');
const {owner, repo} = context.repo;
const sha = '${{ github.event.pull_request.head.sha }}';
const short = sha.slice(0,7);
const link = `[\`${short}\`](https://github.com/${owner}/${repo}/commit/${sha})`
await github.rest.issues.createComment({
issue_number: context.issue.number,
owner,
repo,
body: `**Benchmark ran on this commit:** ${link}\n\n${body}`
});
- name: Cleanup any running kv processes
if: always()
continue-on-error: true
run: |
rm -rf comparison.md kv*
pkill -f kv
exit_code=$?
if [ $exit_code -eq 0 ]; then
echo "Killed running kv processes"
elif [ $exit_code -eq 1 ]; then
echo "No kv processes found to kill"
else
echo "Warning: pkill failed with exit code $exit_code"
fi
- name: Remove ${{ github.event.label.name }} label
if: always() && github.event.label.name == 'run-benchmark'
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
await github.rest.issues.removeLabel({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
name: '${{ github.event.label.name }}'
}).catch(err => console.log('label not present', err.message));
-288
View File
@@ -1,288 +0,0 @@
name: Compare KV Versions
on:
workflow_dispatch:
inputs:
version1:
description: "First version to compare (commit SHA, branch, or tag)"
required: true
type: string
version2:
description: "Second version to compare (commit SHA, branch, or tag)"
required: true
type: string
issue_id:
description: "Issue ID to comment results on"
required: true
type: string
runs:
description: "Number of benchmark runs per configuration"
required: false
type: number
default: 1
defaults:
run:
shell: "bash -Eeuo pipefail -x {0}"
permissions:
contents: read
pull-requests: write
issues: write
jobs:
benchmark:
if: github.repository == 'kv-io/kv'
strategy:
matrix:
include:
- arch: x86
machine: ec2-al-2023-pr-benchmarking-x86
config: benchmark-config-x86.json
- arch: arm64
machine: ec2-al-2023-pr-benchmarking-arm64
config: benchmark-config-arm.json
concurrency:
group: ${{ matrix.machine }}
cancel-in-progress: false
runs-on: ["self-hosted", "${{ matrix.machine }}"]
timeout-minutes: 7200
steps:
- name: Validate inputs
run: |
echo "Version 1: ${{ github.event.inputs.version1 }}"
echo "Version 2: ${{ github.event.inputs.version2 }}"
echo "Issue ID: ${{ github.event.inputs.issue_id }}"
echo "Architecture: ${{ matrix.arch }}"
echo "Config: ${{ matrix.config }}"
echo "Runs: ${{ github.event.inputs.runs }}"
# Validate issue ID is numeric
if ! [[ "${{ github.event.inputs.issue_id }}" =~ ^[0-9]+$ ]]; then
echo "Error: Issue ID must be a number"
exit 1
fi
# Validate runs is a positive number
if ! [[ "${{ github.event.inputs.runs }}" =~ ^[1-9][0-9]*$ ]]; then
echo "Error: Runs must be a positive number"
exit 1
fi
- name: Checkout kv for latest benchmark.
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: kv-io/kv
ref: "unstable"
path: kv_latest
fetch-depth: 0
persist-credentials: false
- name: Checkout kv
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
path: kv
- name: Checkout kv-perf-benchmark
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ${{ github.repository_owner }}/kv-perf-benchmark
path: kv-perf-benchmark
fetch-depth: 1
persist-credentials: false
- name: Set up Python
uses: kishaningithub/setup-python-amazon-linux@a326cdc792983fe0fbd04c81d3d62b59b6123a6c # v1.1.0
with:
python-version: "3.10"
cache: "pip"
- name: Install dependencies
working-directory: kv-perf-benchmark
run: |
sudo dnf groupinstall "Development Tools" -y
sudo dnf install -y gcc gcc-c++ make \
python3-devel \
openssl-devel \
bzip2-devel \
libffi-devel
pip install -r requirements.txt
- name: Build latest kv_latest
working-directory: kv_latest
run: |
echo "Building latest kv-benchmark for latest benchmark executable..."
# Clean any previous builds
make distclean || true
# Build kv-benchmark with latest code
make -j$(nproc)
# Verify the binary was created
if [[ -f "src/kv-benchmark" ]]; then
echo "✓ Successfully built latest kv-benchmark"
ls -la src/kv-benchmark
# Test the binary
echo "Testing kv-benchmark binary..."
./src/kv-benchmark --version || echo "Version check completed"
else
echo "Failed to build kv-benchmark"
exit 1
fi
# Store the absolute path for later use
KV_BENCHMARK_PATH="$(pwd)/src/kv-benchmark"
echo "KV_BENCHMARK_PATH=$KV_BENCHMARK_PATH" >> $GITHUB_ENV
echo "Latest kv-benchmark path: $KV_BENCHMARK_PATH"
- name: Run benchmarks
working-directory: kv-perf-benchmark
env:
EC2_X86_IP: ${{ secrets.EC2_X86_IP }}
EC2_ARM64_IP: ${{ secrets.EC2_ARM64_IP }}
run: |
# Set the target IP based on the matrix architecture
if [[ "${{ matrix.arch }}" == "x86" ]]; then
TARGET_IP=$EC2_X86_IP
echo "Using x86 machine IP"
elif [[ "${{ matrix.arch }}" == "arm64" ]]; then
TARGET_IP=$EC2_ARM64_IP
echo "Using ARM64 machine IP"
else
echo "Error: Unknown architecture: ${{ matrix.arch }}"
exit 1
fi
CONFIG_FILE="../kv/.github/benchmark_configs/${{ matrix.config }}"
# Verify our custom kv-benchmark exists
if [[ ! -f "$KV_BENCHMARK_PATH" ]]; then
echo "Custom kv-benchmark not found at: $KV_BENCHMARK_PATH"
exit 1
fi
echo "Using custom kv-benchmark from: $KV_BENCHMARK_PATH"
# Base benchmark arguments with custom kv-benchmark path
BENCHMARK_ARGS=(
--config "$CONFIG_FILE"
--commits ${{ github.event.inputs.version1 }} ${{ github.event.inputs.version2 }}
--kv-benchmark-path "$KV_BENCHMARK_PATH"
--target-ip $TARGET_IP
--results-dir "results"
--runs ${{ github.event.inputs.runs }}
)
echo "Running benchmarks with the following setup:"
echo "- Version 1: ${{ github.event.inputs.version1 }}"
echo "- Version 2: ${{ github.event.inputs.version2 }}"
echo "- Architecture: ${{ matrix.arch }}"
echo "- Config: ${{ matrix.config }}"
echo "- Using latest kv-benchmark: $KV_BENCHMARK_PATH"
echo "- This ensures both versions use the same (latest) benchmark tool for consistent results"
# Run benchmark with custom kv-benchmark executable
python ./benchmark.py "${BENCHMARK_ARGS[@]}"
- name: Compare results
working-directory: kv-perf-benchmark
run: |
# Find the actual result directories (they might have different names than input)
VERSION1_DIR=$(find ./results -maxdepth 1 -type d -name "*${{ github.event.inputs.version1 }}*" | head -1)
VERSION2_DIR=$(find ./results -maxdepth 1 -type d -name "*${{ github.event.inputs.version2 }}*" | head -1)
if [[ -z "$VERSION1_DIR" ]]; then
echo "Could not find results for version1: ${{ github.event.inputs.version1 }}"
echo "Available result directories:"
ls -la ./results/
exit 1
fi
if [[ -z "$VERSION2_DIR" ]]; then
echo "Could not find results for version2: ${{ github.event.inputs.version2 }}"
echo "Available result directories:"
ls -la ./results/
exit 1
fi
echo "Comparing results:"
echo "Version 1 (${{ github.event.inputs.version1 }}): $VERSION1_DIR"
echo "Version 2 (${{ github.event.inputs.version2 }}): $VERSION2_DIR"
# Generate RPS-focused comparison and graphs for GitHub comments
python utils/compare_benchmark_results.py \
--baseline "$VERSION1_DIR/metrics.json" \
--new "$VERSION2_DIR/metrics.json" \
--output ../comparison.md \
--metrics rps
- name: Upload artifacts
if: always()
continue-on-error: true
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: benchmark-results-${{ matrix.arch }}-${{ github.event.inputs.issue_id }}
path: |
./kv-perf-benchmark/results/*
comparison.md
- name: Cleanup any running kv processes and files
if: always()
continue-on-error: true
run: |
pkill -f kv || echo "No kv processes found to kill"
rm -rf *
combine-results:
needs: benchmark
runs-on: ubuntu-latest
steps:
- name: Download all artifacts
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
with:
path: artifacts
- name: Combine results and create comprehensive report
run: |
echo "# Multi-Architecture Benchmark Comparison: ${{ github.event.inputs.version1 }} vs ${{ github.event.inputs.version2 }}" > combined_report.md
echo "" >> combined_report.md
echo "**Versions Compared:**" >> combined_report.md
echo "- Version 1: \`${{ github.event.inputs.version1 }}\`" >> combined_report.md
echo "- Version 2: \`${{ github.event.inputs.version2 }}\`" >> combined_report.md
echo "" >> combined_report.md
echo "**Runs:** ${{ github.event.inputs.runs }} per configuration" >> combined_report.md
echo "**Workflow Run:** [View Details](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> combined_report.md
echo "" >> combined_report.md
echo "---" >> combined_report.md
echo "" >> combined_report.md
# Process each architecture's results
for arch in x86 arm64; do
artifact_dir="artifacts/benchmark-results-${arch}-${{ github.event.inputs.issue_id }}"
if [[ -d "$artifact_dir" && -f "$artifact_dir/comparison.md" ]]; then
echo "## ${arch^^} Architecture Results" >> combined_report.md
echo "" >> combined_report.md
cat "$artifact_dir/comparison.md" >> combined_report.md
echo "" >> combined_report.md
echo "---" >> combined_report.md
echo "" >> combined_report.md
fi
done
- name: Comment on issue with combined results
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const fs = require('fs');
// Read the combined report
const body = fs.readFileSync('combined_report.md', 'utf8');
await github.rest.issues.createComment({
issue_number: ${{ github.event.inputs.issue_id }},
owner: context.repo.owner,
repo: context.repo.repo,
body: body
});
+27 -200
View File
@@ -1,16 +1,6 @@
name: CI
on:
push:
paths-ignore:
- '**/*.md'
- '**/00-RELEASENOTES'
- '**/COPYING'
pull_request:
paths-ignore:
- '**/*.md'
- '**/00-RELEASENOTES'
- '**/COPYING'
on: [push, pull_request]
concurrency:
group: ci-${{ github.head_ref || github.ref }}
@@ -23,19 +13,11 @@ jobs:
test-ubuntu-latest:
runs-on: ubuntu-latest
steps:
- name: Install libbacktrace
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ianlancetaylor/libbacktrace
ref: b9e40069c0b47a722286b94eb5231f7f05c08713
path: libbacktrace
- run: cd libbacktrace && ./configure && make && sudo make install
- name: Checkout KV
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: make
# Fail build if there are warnings
# build with TLS just for compilation coverage
run: make -j4 all-with-unit-tests SERVER_CFLAGS='-Werror' BUILD_TLS=yes USE_FAST_FLOAT=yes USE_LIBBACKTRACE=yes
run: make all-with-unit-tests SERVER_CFLAGS='-Werror' BUILD_TLS=yes
- name: test
run: |
sudo apt-get install tcl8.6 tclx
@@ -46,118 +28,34 @@ jobs:
run: |
touch src/commands/ping.json
make commands.def
dirty="$(git diff)"
if [[ ! -z "$dirty" ]]; then echo "$dirty"; exit 1; fi
dirty=$(git diff)
if [[ ! -z $dirty ]]; then echo $dirty; exit 1; fi
- name: unit tests
run: |
./src/kv-unit-tests
test-ubuntu-latest-compatibility:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
server:
- {version: "7.2.11", file: "kv-7.2.11-noble-x86_64.tar.gz"}
- {version: "8.0.6", file: "kv-8.0.6-noble-x86_64.tar.gz"}
- {version: "8.1.4", file: "kv-8.1.4-noble-x86_64.tar.gz"}
steps:
- name: Install libbacktrace
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ianlancetaylor/libbacktrace
ref: b9e40069c0b47a722286b94eb5231f7f05c08713
path: libbacktrace
- run: cd libbacktrace && ./configure && make && sudo make install
- name: Checkout KV
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: make
# Fail build if there are warnings
# build with TLS just for compilation coverage
run: make -j4 all-with-unit-tests SERVER_CFLAGS='-Werror' BUILD_TLS=yes USE_FAST_FLOAT=yes USE_LIBBACKTRACE=yes
- name: Install old server (${{ matrix.server.version }}) for compatibility testing
run: |
mkdir -p tests/tmp
cd tests/tmp
wget https://download.kv.io/releases/${{ matrix.server.file }}
tar -xvf ${{ matrix.server.file }}
- name: Run compatibility tests against ${{ matrix.server.version }}
run: |
sudo apt-get install -y tcl8.6 tclx
./runtest --verbose --tags "-slow needs:other-server" --dump-logs \
--other-server-path tests/tmp/kv-${{ matrix.server.version }}-noble-x86_64/bin/kv-server
- name: Module API tests against ${{ matrix.server.version }}
run: |
CFLAGS='-Werror' ./runtest-moduleapi --tags needs:other-server --verbose --dump-logs \
--other-server-path tests/tmp/kv-${{ matrix.server.version }}-noble-x86_64/bin/kv-server
test-ubuntu-latest-cmake-tls:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: cmake and make
run: |
sudo apt-get install -y cmake libssl-dev
mkdir -p build-release
cd build-release
cmake -DCMAKE_BUILD_TYPE=Release .. -DBUILD_TLS=yes -DBUILD_UNIT_TESTS=yes
make -j$(nproc)
- name: test
run: |
sudo apt-get install -y tcl8.6 tclx tcl-tls
./utils/gen-test-certs.sh
./build-release/runtest --verbose --tags -slow --dump-logs --tls
- name: unit tests
run: |
./build-release/bin/kv-unit-tests
./src/valkey-unit-tests
test-sanitizer-address:
runs-on: ubuntu-latest
steps:
- name: Install libbacktrace
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ianlancetaylor/libbacktrace
ref: b9e40069c0b47a722286b94eb5231f7f05c08713
path: libbacktrace
- run: cd libbacktrace && ./configure && make && sudo make install
- name: Checkout KV
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: make
# build with TLS module just for compilation coverage
run: make -j4 all-with-unit-tests SANITIZER=address SERVER_CFLAGS='-Werror' BUILD_TLS=module USE_LIBBACKTRACE=yes
run: make SANITIZER=address SERVER_CFLAGS='-Werror' BUILD_TLS=module
- name: testprep
run: sudo apt-get install tcl8.6 tclx -y
- name: test
run: ./runtest --verbose --tags -slow --dump-logs
- name: module api test
run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs
- name: unit tests
run: ./src/kv-unit-tests
test-rdma:
runs-on: ubuntu-latest
steps:
- name: Install libbacktrace
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ianlancetaylor/libbacktrace
ref: b9e40069c0b47a722286b94eb5231f7f05c08713
path: libbacktrace
- run: cd libbacktrace && ./configure && make && sudo make install
- name: Checkout KV
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: prepare-development-libraries
run: sudo apt-get install librdmacm-dev libibverbs-dev
- name: make-rdma-module
run: make -j4 BUILD_RDMA=module USE_LIBBACKTRACE=yes
- name: make-rdma-builtin
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: make
run: |
make distclean
make -j4 BUILD_RDMA=yes USE_LIBBACKTRACE=yes
sudo apt-get install librdmacm-dev libibverbs-dev
make BUILD_RDMA=module
- name: clone-rxe-kmod
run: |
mkdir -p tests/rdma/rxe
@@ -170,120 +68,49 @@ jobs:
- name: show-kernel-log
run: sudo dmesg -c
test-tls-only:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: generate-test-certificates
run: ./utils/gen-test-certs.sh
- name: install-test-dependencies
run: sudo apt-get install -y tcl8.6 tclx tcl-tls
- name: make-tls-module
run: make -j4 BUILD_TLS=module SERVER_CFLAGS='-Werror'
- name: test-tls-module
run: ./runtest --verbose --single unit/tls --dump-logs --tls-module
- name: make-tls-builtin
run: |
make distclean
make -j4 BUILD_TLS=yes SERVER_CFLAGS='-Werror'
- name: test-tls-builtin
run: ./runtest --verbose --single unit/tls --dump-logs --tls
build-debian-old:
runs-on: ubuntu-latest
container: debian:bullseye
steps:
- name: Install libbacktrace
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ianlancetaylor/libbacktrace
ref: b9e40069c0b47a722286b94eb5231f7f05c08713
path: libbacktrace
- name: Build libbacktrace
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: make
run: |
apt-get update && apt-get install -y build-essential
cd libbacktrace && ./configure && make && make install
- name: Checkout KV
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: make
run: make -j4 SERVER_CFLAGS='-Werror' USE_LIBBACKTRACE=yes
make SERVER_CFLAGS='-Werror'
build-macos-latest:
runs-on: macos-latest
steps:
- name: Install libbacktrace
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ianlancetaylor/libbacktrace
ref: b9e40069c0b47a722286b94eb5231f7f05c08713
path: libbacktrace
- run: cd libbacktrace && ./configure && make && sudo make install
- name: Checkout KV
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: make
# Build with additional upcoming features
run: make -j3 all-with-unit-tests SERVER_CFLAGS='-Werror' USE_FAST_FLOAT=yes USE_LIBBACKTRACE=yes
run: make SERVER_CFLAGS='-Werror'
build-32bit:
runs-on: ubuntu-latest
steps:
- name: Install libbacktrace
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ianlancetaylor/libbacktrace
ref: b9e40069c0b47a722286b94eb5231f7f05c08713
path: libbacktrace
- name: Build libbacktrace (32-bit)
run: |
sudo apt-get update
sudo apt-get install libc6-dev-i386 libstdc++-11-dev-i386-cross gcc-multilib g++-multilib
cd libbacktrace && ./configure CFLAGS="-m32" --prefix=/usr/local/libbacktrace32 && make && sudo make install
- name: Checkout KV
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: make
# Fast float requires C++ 32-bit libraries to compile on 64-bit ubuntu
# machine i.e. "-cross" suffixed version. Cross-compiling c++ to 32-bit
# also requires multilib support for g++ compiler i.e. "-multilib"
# suffixed version of g++. g++-multilib generally includes libstdc++.
# *cross version as well, but it is also added explicitly just in case.
run: make -j4 SERVER_CFLAGS='-Werror' 32bit USE_FAST_FLOAT=yes USE_LIBBACKTRACE=yes LIBBACKTRACE_PREFIX=/usr/local/libbacktrace32
- name: unit tests
run: |
./src/kv-unit-tests
sudo apt-get update && sudo apt-get install libc6-dev-i386
make SERVER_CFLAGS='-Werror' 32bit
build-libc-malloc:
runs-on: ubuntu-latest
steps:
- name: Install libbacktrace
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ianlancetaylor/libbacktrace
ref: b9e40069c0b47a722286b94eb5231f7f05c08713
path: libbacktrace
- run: cd libbacktrace && ./configure && make && sudo make install
- name: Checkout KV
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: make
run: make -j4 SERVER_CFLAGS='-Werror' MALLOC=libc USE_FAST_FLOAT=yes USE_LIBBACKTRACE=yes
run: make SERVER_CFLAGS='-Werror' MALLOC=libc
build-almalinux8-jemalloc:
runs-on: ubuntu-latest
container: almalinux:8
steps:
- name: Install libbacktrace
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ianlancetaylor/libbacktrace
ref: b9e40069c0b47a722286b94eb5231f7f05c08713
path: libbacktrace
- name: Build libbacktrace
run: |
dnf -y install epel-release gcc gcc-c++ make procps-ng which
cd libbacktrace && ./configure && make && make install
- name: Checkout KV
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: make
run: make -j4 SERVER_CFLAGS='-Werror' USE_FAST_FLOAT=yes USE_LIBBACKTRACE=yes
run: |
dnf -y install epel-release gcc make procps-ng which
make -j SERVER_CFLAGS='-Werror'
format-yaml:
runs-on: ubuntu-latest
-59
View File
@@ -1,59 +0,0 @@
name: Clang Format Check
on:
push:
paths-ignore:
- '**/*.md'
- '**/00-RELEASENOTES'
- '**/COPYING'
pull_request:
paths-ignore:
- '**/*.md'
- '**/00-RELEASENOTES'
- '**/COPYING'
concurrency:
group: clang-${{ github.head_ref || github.ref }}
cancel-in-progress: true
jobs:
clang-format-check:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Clang
run: |
sudo apt-get update -y
sudo apt-get upgrade -y
sudo apt-get install software-properties-common -y
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor | sudo tee /usr/share/keyrings/llvm-toolchain.gpg > /dev/null
echo "deb [signed-by=/usr/share/keyrings/llvm-toolchain.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-18 main" | sudo tee /etc/apt/sources.list.d/llvm.list
sudo apt-get update -y
sudo apt-get install clang-format-18 -y
- name: Run clang-format
id: clang-format
run: |
# Run clang-format and capture the diff
cd src
shopt -s globstar
clang-format-18 -i **/*.c **/*.h
# Capture the diff output
DIFF=$(git diff)
if [ ! -z "$DIFF" ]; then
# Encode the diff in Base64 to ensure it's handled as a single line
ENCODED_DIFF=$(echo "$DIFF" | base64 -w 0)
echo "diff=$ENCODED_DIFF" >> $GITHUB_OUTPUT
fi
shell: bash
- name: Check for formatting changes
if: ${{ steps.clang-format.outputs.diff }}
run: |
echo "ERROR: Code is not formatted correctly. Here is the diff:"
# Decode the Base64 diff to display it
echo "${{ steps.clang-format.outputs.diff }}" | base64 --decode
exit 1
shell: bash
+7 -23
View File
@@ -2,17 +2,7 @@ name: "Codecov"
# Enabling on each push is to display the coverage changes in every PR,
# where each PR needs to be compared against the coverage of the head commit
on:
push:
paths-ignore:
- '**/*.md'
- '**/00-RELEASENOTES'
- '**/COPYING'
pull_request:
paths-ignore:
- '**/*.md'
- '**/00-RELEASENOTES'
- '**/COPYING'
on: [push, pull_request]
concurrency:
group: codecov-${{ github.head_ref || github.ref }}
@@ -20,25 +10,19 @@ concurrency:
jobs:
code-coverage:
runs-on: ubuntu-22.04
runs-on: ubuntu-latest
steps:
- name: Install libbacktrace
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ianlancetaylor/libbacktrace
ref: b9e40069c0b47a722286b94eb5231f7f05c08713
path: libbacktrace
- run: cd libbacktrace && ./configure && make && sudo make install
- name: Checkout KV
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Install lcov and run test
run: |
sudo apt-get install lcov tclx
make lcov USE_LIBBACKTRACE=yes
sudo apt-get install lcov
make lcov
- name: Upload code coverage
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: ./src/kv.info
file: ./src/valkey.info
+1 -10
View File
@@ -1,16 +1,7 @@
name: "CodeQL"
on:
push:
paths-ignore:
- '**/*.md'
- '**/00-RELEASENOTES'
- '**/COPYING'
pull_request:
paths-ignore:
- '**/*.md'
- '**/00-RELEASENOTES'
- '**/COPYING'
schedule:
# run weekly new vulnerability was added to the database
- cron: '0 3 * * 0'
@@ -26,7 +17,7 @@ jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
if: github.event_name != 'schedule' || github.repository == 'kv-io/kv'
if: github.event_name != 'schedule' || github.repository == 'valkey-io/valkey'
permissions:
security-events: write
+6 -14
View File
@@ -16,27 +16,19 @@ permissions:
jobs:
coverity:
if: github.repository == 'kv-io/kv'
if: github.repository == 'valkey-io/valkey'
runs-on: ubuntu-latest
steps:
- name: Install libbacktrace
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ianlancetaylor/libbacktrace
ref: b9e40069c0b47a722286b94eb5231f7f05c08713
path: libbacktrace
- run: cd libbacktrace && ./configure && make && sudo make install
- name: Checkout KV
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Download and extract the Coverity Build Tool
run: |
wget -q https://scan.coverity.com/download/cxx/linux64 --post-data "token=${{ secrets.COVERITY_SCAN_TOKEN }}&project=kv-io%2Fkv" -O cov-analysis-linux64.tar.gz
wget -q https://scan.coverity.com/download/cxx/linux64 --post-data "token=${{ secrets.COVERITY_SCAN_TOKEN }}&project=valkey-io%2Fvalkey" -O cov-analysis-linux64.tar.gz
mkdir cov-analysis-linux64
tar xzf cov-analysis-linux64.tar.gz --strip 1 -C cov-analysis-linux64
- name: Install KV dependencies
- name: Install Valkey dependencies
run: sudo apt install -y gcc procps libssl-dev
- name: Build with cov-build
run: cov-analysis-linux64/bin/cov-build --dir cov-int make USE_LIBBACKTRACE=yes
run: cov-analysis-linux64/bin/cov-build --dir cov-int make
- name: Upload the result
run: |
tar czvf cov-int.tgz cov-int
@@ -44,4 +36,4 @@ jobs:
--form email=${{ secrets.COVERITY_SCAN_EMAIL }} \
--form token=${{ secrets.COVERITY_SCAN_TOKEN }} \
--form file=@cov-int.tgz \
https://scan.coverity.com/builds?project=kv-io%2Fkv
https://scan.coverity.com/builds?project=valkey-io%2Fvalkey
File diff suppressed because it is too large Load Diff
-58
View File
@@ -1,58 +0,0 @@
name: Build and Deploy Hanzo KV
on:
push:
branches: [main]
tags: ['v*']
paths:
- 'src/**'
- 'deps/**'
- 'Dockerfile'
- '.github/workflows/deploy.yml'
workflow_dispatch:
jobs:
build:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Image metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ghcr.io/hanzoai/kv
tags: |
type=raw,value=latest,enable={{is_default_branch}}
type=raw,value=9,enable={{is_default_branch}}
type=sha,prefix=
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
- name: Build and push
uses: docker/build-push-action@v5
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha,scope=kv
cache-to: type=gha,mode=max,scope=kv
+20 -53
View File
@@ -1,16 +1,8 @@
name: External Server Tests
on:
push:
paths-ignore:
- '**/*.md'
- '**/00-RELEASENOTES'
- '**/COPYING'
pull_request:
paths-ignore:
- '**/*.md'
- '**/00-RELEASENOTES'
- '**/COPYING'
push:
schedule:
- cron: '0 2 * * *'
@@ -24,23 +16,15 @@ permissions:
jobs:
test-external-standalone:
runs-on: ubuntu-latest
if: github.event_name != 'schedule' || github.repository == 'kv-io/kv'
timeout-minutes: 1440
if: github.event_name != 'schedule' || github.repository == 'valkey-io/valkey'
timeout-minutes: 14400
steps:
- name: Install libbacktrace
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ianlancetaylor/libbacktrace
ref: b9e40069c0b47a722286b94eb5231f7f05c08713
path: libbacktrace
- run: cd libbacktrace && ./configure && make && sudo make install
- name: Checkout KV
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Build
run: make SERVER_CFLAGS=-Werror USE_LIBBACKTRACE=yes
- name: Start kv-server
run: make SERVER_CFLAGS=-Werror
- name: Start valkey-server
run: |
./src/kv-server --daemonize yes --save "" --logfile external-server.log \
./src/valkey-server --daemonize yes --save "" --logfile external-server.log \
--enable-protected-configs yes --enable-debug-command yes --enable-module-command yes
- name: Run external test
run: |
@@ -57,27 +41,18 @@ jobs:
test-external-cluster:
runs-on: ubuntu-latest
if: github.event_name != 'schedule' || github.repository == 'kv-io/kv'
timeout-minutes: 1440
if: github.event_name != 'schedule' || github.repository == 'valkey-io/valkey'
timeout-minutes: 14400
steps:
- name: Install libbacktrace
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ianlancetaylor/libbacktrace
ref: b9e40069c0b47a722286b94eb5231f7f05c08713
path: libbacktrace
- run: cd libbacktrace && ./configure && make && sudo make install
- name: Checkout KV
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Build
run: make SERVER_CFLAGS=-Werror USE_LIBBACKTRACE=yes
- name: Start kv-server
run: make SERVER_CFLAGS=-Werror
- name: Start valkey-server
run: |
./src/kv-server --cluster-enabled yes --cluster-databases 16 --daemonize yes \
--save "" --logfile external-server.log \
./src/valkey-server --cluster-enabled yes --daemonize yes --save "" --logfile external-server.log \
--enable-protected-configs yes --enable-debug-command yes --enable-module-command yes
- name: Create a single node cluster
run: ./src/kv-cli cluster addslots $(for slot in {0..16383}; do echo $slot; done); sleep 5
run: ./src/valkey-cli cluster addslots $(for slot in {0..16383}; do echo $slot; done); sleep 5
- name: Run external test
run: |
./runtest \
@@ -94,23 +69,15 @@ jobs:
test-external-nodebug:
runs-on: ubuntu-latest
if: github.event_name != 'schedule' || github.repository == 'kv-io/kv'
timeout-minutes: 1440
if: github.event_name != 'schedule' || github.repository == 'valkey-io/valkey'
timeout-minutes: 14400
steps:
- name: Install libbacktrace
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ianlancetaylor/libbacktrace
ref: b9e40069c0b47a722286b94eb5231f7f05c08713
path: libbacktrace
- run: cd libbacktrace && ./configure && make && sudo make install
- name: Checkout KV
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Build
run: make SERVER_CFLAGS=-Werror USE_LIBBACKTRACE=yes
- name: Start kv-server
run: make SERVER_CFLAGS=-Werror
- name: Start valkey-server
run: |
./src/kv-server --daemonize yes --save "" --logfile external-server.log
./src/valkey-server --daemonize yes --save "" --logfile external-server.log
- name: Run external test
run: |
./runtest \
+2 -2
View File
@@ -6,7 +6,7 @@ on:
workflow_dispatch:
inputs:
version:
description: Version of KV to build
description: Version of Valkey to build
required: true
environment:
description: Environment to build
@@ -41,7 +41,7 @@ jobs:
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
with:
token: ${{ secrets.AUTOMATION_PAT }}
repository: ${{ github.repository_owner }}/kv-release-automation
repository: ${{ github.repository_owner }}/valkey-release-automation
event-type: build-release
client-payload: >
{
-68
View File
@@ -1,68 +0,0 @@
name: Weekly Test Workflow for Released Branches
on:
schedule:
- cron: '0 6 * * 0'
workflow_dispatch: {}
permissions:
actions: read
contents: read
pull-requests: read
concurrency:
group: weekly-release-tests
cancel-in-progress: false
jobs:
determine-release-branches:
if: github.repository == 'kv-io/kv'
runs-on: ubuntu-latest
outputs:
branches: ${{ steps.release-branches.outputs.branches }}
has-branches: ${{ steps.release-branches.outputs.has-branches }}
steps:
- name: Collect release branches
id: release-branches
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const { owner, repo } = context.repo;
const MIN_MAJOR = 7;
const MIN_MINOR = 2;
const branches = await github.paginate(github.rest.repos.listBranches, {
owner,
repo,
per_page: 100,
});
const releaseBranches = branches
.map(({ name }) => name)
.filter(name => /^\d+\.\d+$/.test(name))
.filter(name => {
const [major, minor] = name.split('.').map(Number);
return Number.isInteger(major) &&
Number.isInteger(minor) &&
(major > MIN_MAJOR || (major === MIN_MAJOR && minor >= MIN_MINOR));
})
.sort((a, b) => a.localeCompare(b, undefined, { numeric: true }));
core.info(`Weekly release branches: ${releaseBranches.join(', ') || '(none)'}`);
core.setOutput('branches', JSON.stringify(releaseBranches));
core.setOutput('has-branches', releaseBranches.length > 0 ? 'true' : 'false');
run-daily-for-release-branches:
needs: determine-release-branches
if: needs.determine-release-branches.outputs.has-branches == 'true' && github.repository == 'kv-io/kv'
permissions:
actions: read
contents: read
pull-requests: read
strategy:
fail-fast: false
matrix:
release-branch: ${{ fromJson(needs.determine-release-branches.outputs.branches) }}
max-parallel: 1
uses: ./.github/workflows/daily.yml
with:
use_repo: kv-io/kv
use_git_ref: ${{ matrix.release-branch }}
skipjobs: ""
skiptests: ""
secrets: inherit
-8
View File
@@ -4,8 +4,6 @@
*.xo
*.so
*.d
*.lo
*.la
*.log
dump*.rdb
*-benchmark
@@ -50,9 +48,3 @@ redis.code-workspace
nodes*.conf
tests/cluster/tmp/*
tests/rdma/rdma-test
tags
build/
build-debug/
build-release/
cmake-build-debug/
cmake-build-release/
+575 -11
View File
@@ -1,16 +1,580 @@
Hello! This file is just a placeholder, since this is the "unstable" branch
of KV, the place where all the development happens.
Valkey 8.0 release notes
========================
--------------------------------------------------------------------------------
Upgrade urgency levels:
LOW: No need to upgrade unless there are new features you want to use.
MODERATE: Program an upgrade of the server, but it's not urgent.
HIGH: There is a critical bug that may affect a subset of users. Upgrade!
CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP.
SECURITY: There are security fixes in the release.
--------------------------------------------------------------------------------
There is no release notes for this branch, it gets forked into another branch
every time there is a partial feature freeze in order to eventually create
a new stable release.
================================================================================
Valkey 8.0.7 - Released Mon 23 February 2026
================================================================================
Usually "unstable" is stable enough for you to use it in development environments
however you should never use it in production environments. It is possible
to download the latest stable release here:
Upgrade urgency SECURITY: This release includes security fixes we recommend you
apply as soon as possible.
https://kv.io/download/
Security fixes
==============
* (CVE-2026-21863) Remote DoS with malformed Valkey Cluster bus message
* (CVE-2025-67733) RESP Protocol Injection via Lua error_reply
More information is available at https://kv.io
Bug fixes
=========
* Fix ltrim should not call signalModifiedKey when no elements are removed (#2787)
* Fix chained replica crash when doing dual channel replication (#2983)
* Fix used_memory_dataset underflow due to miscalculated used_memory_overhead (#3005)
* Avoids crash during MODULE UNLOAD when ACL rules reference a module command and subcommand (#3160)
* Fix server assert on ACL LOAD and resetchannels (#3182)
* Fix bug causing no response flush sometimes when IO threads are busy (#3205)
Happy hacking!
================================================================================
Valkey 8.0.6 - Released Fri 03 October 2025
================================================================================
Upgrade urgency SECURITY: This release includes security fixes we recommend you
apply as soon as possible.
Security fixes
==============
* (CVE-2025-49844) A Lua script may lead to remote code execution
* (CVE-2025-46817) A Lua script may lead to integer overflow and potential RCE
* (CVE-2025-46818) A Lua script can be executed in the context of another user
* (CVE-2025-46819) LUA out-of-bound read
Bug fixes
=========
* Fix accounting for dual channel RDB bytes in replication stats (#2616)
* Minor fix for dual rdb channel connection conn error log (#2658)
* Fix unsigned difference expression compared to zero (#2101)
================================================================================
Valkey 8.0.5 - Released Thu 22 Aug 2025
================================================================================
Upgrade urgency SECURITY: This release includes security fixes we recommend you
apply as soon as possible.
Bug fixes
=========
* Fix clients remaining blocked when reprocessing commands after certain
blocking operations (#2109)
* Fix a memory corruption issue in the sharded pub/sub unsubscribe logic (#2137)
* Fix potential memory leak by ensuring module context is freed when `aux_save2`
callback writes no data (#2132)
* Fix `CLIENT UNBLOCK` triggering unexpected errors when used on paused clients
(#2117)
* Fix missing NULL check on `SSL_new()` when creating outgoing TLS connections
(#2140)
* Fix incorrect casting of ping extension lengths to prevent silent packet drops
(#2144)
* Fix replica failover stall due to outdated config epoch (#2178)
* Fix incorrect port/tls-port info in `CLUSTER SLOTS`/`CLUSTER NODES` after
dynamic config change (#2186)
* Ensure empty error tables in Lua scripts don't crash Valkey (#2229)
* Fix client tracking memory overhead calculation (#2360)
* Handle divergent shard-id from nodes.conf and reconcile to the primary node's
shard-id (#2174)
* Fix pre-size hashtables per slot when reading RDB files (#2466)
Behavior changes
================
* Trigger election immediately during a forced manual failover (`CLUSTER
FAILOVER FORCE`) to avoid delay (#1067)
* Reset ongoing election state when initiating a new manual failover (#1274)
Logging and Tooling Improvements
================================
* Add support to drop all cluster packets (#1252)
* Improve log clarity in failover auth denial message (#1341)
Security fixes
==============
* CVE-2025-27151: Check length of AOF file name in valkey-check-aof and reject
paths longer than `PATH_MAX` (#2146)
================================================================================
Valkey 8.0.4 - Released Mon 07 July 2025
================================================================================
Upgrade urgency SECURITY: This release includes security fixes we recommend you
apply as soon as possible.
Security fixes
==============
* CVE-2025-32023 prevent out-of-bounds write during hyperloglog operations (#2146)
* CVE-2025-48367 retry accept on transient errors (#2315)
================================================================================
Valkey 8.0.3 - Released Wed 23 Apr 2025
================================================================================
Upgrade urgency SECURITY: This release includes security fixes we recommend you
apply as soon as possible.
Bug fixes
=========
* Optimize RDB load performance and fix cluster mode resizing on replica side (#1199)
* Fix memory leak in forgotten node ping ext code path (#1574)
* Fix cluster info sent stats for message with light header (#1563)
* Fix module LatencyAddSample still work when latency-monitor-threshold is 0 (#1541)
* Fix potential crash in radix tree recompression of huge keys (#1722)
* Fix error "SSL routines::bad length" when connTLSWrite is called second time with smaller buffer (#1737)
* Fix temp file leak druing replication error handling (#1721)
* Fix ACL LOAD crash on replica since the primary client don't has a user (#1842)
* Fix RANDOMKEY infinite loop during CLIENT PAUSE (#1850)
* fix: add samples to stream object consumer trees (#1825)
* Fix cluster slot stats assertion during promotion of replica (#1950)
* Fix panic in primary when blocking shutdown after previous block with timeout (#1948)
* Ignore stale gossip packets that arrive out of order (#1777)
* Fix incorrect lag reported in XINFO GROUPS (#1952)
* Avoid shard id update of replica if not matching with primary shard id (#573)
Security fixes
==============
* CVE-2025-21605 Limit output buffer for unauthenticated clients (#1993)
================================================================================
Valkey 8.0.2 - Released Mon 6 Jan 2025
================================================================================
Upgrade urgency SECURITY: This release includes security fixes we recommend you
apply as soon as possible.
Bug fixes
=========
* Fix an uncommon crash when using TLS with dual channel replication. (#1173)
* Make sure `repl_down_since` is correctly reset when dual channel replication
fails. (#1149)
* Fix a performance regression where a replica does not properly initiaize
the database size when loading a snapshot during replication. (#1199)
* Make sure the last accessed time is correctly updated when using the `TOUCH`
command with the `CLIENT NO-TOUCH` option.
* Fix a bug where `total_net_repl_output_bytes` would report the wrong. (#1486)
* Fix a bug where `used_memory_scripts` may report the wrong value. (#1255)
* Fix a bug where server might crash when using active defrag when scripts
are evicted from the script cache. (#1310)
* Fix a bug where extra memory would be used when storing strings in the
inline protocol. (#1213)
* Fix a bug where the `SORT` command may throw a cross slot error. (#1182)
* Fix a bug where the `RANDOMKEY` command may omit returning keys in cluster
mode. (#1155)
* Send the correct error message when `FUNCTION KIlL` is used to kill an
ongoing script. (#1171)
* Fix a potential memory corruption when databases are emptied, such as
through `FLUSHDB`, when during active defrag is running. (#1512)
Behavior changes
================
* Revert an unintended breaking change when sending an unsubuscribe command
when a client is not subscribed to any channels. (#1265)
Security fixes
==============
* (CVE-2024-46981) Lua script commands may lead to remote code execution. (#1513)
* (CVE-2024-51741) Denial-of-service due to malformed ACL selectors. (#1514)
================================================================================
Valkey 8.0.1 - Released Tue 1 Oct 2024
================================================================================
Upgrade urgency SECURITY: This release includes security fixes we recommend you
apply as soon as possible.
Bug fixes
=========
* Fix a build issue with RDMA when using additional make parameters. (#1074)
* Fix an issue where `CLUSTER SLOTS` might return the wrong tcp or tls port when called
from inside a script or from a module. (#1072)
* Fix a crash when `CLUSTER SLOTS` or `CLUSTER SHARDS` is called from inside
a script or from a module. (#1063)
* Fix a build issue on systems where `<threads.h>` is unavailable. (#1053)
* Fix an issue with the default `sentinel.conf` being invalid. (#1040)
Security fixes
==============
* (CVE-2024-31449) Lua library commands may lead to stack overflow and potential RCE.
* (CVE-2024-31227) Potential Denial-of-service due to malformed ACL selectors.
* (CVE-2024-31228) Potential Denial-of-service due to unbounded pattern matching.
================================================================================
Valkey 8.0.0 GA - Released Sun 15 Sep 2024
================================================================================
Upgrade urgency LOW: This is the first release of Valkey 8.0, which
includes stability and performance improvements over the second release
candidate. This release is fully compatible with Redis OSS 7.2.4.
Logging and Tooling Improvements
================================
* Added full client info to SHUTDOWN and CLUSTER FAILOVER logs for better traceability
of requests. (#875)
Bug fixes
=========
* Resolved issues in replicationSetPrimary where the primary node's IP/port updates were
not correctly handled in the cluster gossip section. (#965)
* Fixed AOF base suffix during rewrites when modifying the aof-use-rdb-preamble setting,
ensuring correct suffix caching to prevent inconsistencies. (#886)
* Addressed rare crashes in async IO threads with TLS by preventing concurrent read and
write job overlap. (#1011)
* Prevented AOF from being incorrectly disabled after loading RDB data, ensuring proper
re-enabling of AOF. (#1001)
* Triggered a save of the cluster configuration file before shutdown to prevent
inconsistencies caused by unsaved node configuration changes. (#822)
* Fixed timing issue in CLUSTER SETSLOT to ensure replicas handle migration correctly
when receiving the command before the gossip update. (#981)
Performance
===========
* Optimized the handling of temporary set objects in SUNION and SDIFF commands, resulting
in a 41% performance improvement for SUNION and 27% for SDIFF. (#996)
Behavior Changes
================
* Replicas now flush old data after checking RDB file is valid during disk-based replication,
preventing partial data loss and ensuring a clean data load. (#926)
================================================================================
Valkey 8.0.0 RC2 - Released Tue 2 Sep 2024
================================================================================
Upgrade urgency LOW: This is the second release candidate of Valkey 8.0, which
generally includes stability and performance improvements over the first release
candidate. This release is fully compatible with Redis OSS 7.2.4.
New Features
============
* Add support for setting the group on a unix domain socket. (#901)
* Experimental - RDMA: Support user keepalive command. (#916)
Logging and Tooling Improvements
================================
* Improve logging when a cluster is unable to failover. (#780)
* Add epoch information to failover auth denied logs. (#816)
* Improve messaging when valkey-cli cluster --fix meet check failed. (#867)
* Log unexpected $ENDOFF responses in dual channel replication. (#839)
* Skip IPv6 tests when it is unsupported on TCLSH. (#910)
* Add a light weight version of DEBUG OBJECT. (#881)
* Add lfu support for DEBUG OBJECT command, added lfu_freq and
lfu_access_time_minutes fields. (#479)
Bug fixes
=========
* Correctly expose client infomation in the slowlog when running commands in scripts. (#805)
* Fix a bug where lastbgsave_status was unecessarily being updated when dual
channel replication failed. (#811)
* Fix a bug which caused dual channel replication to get stuck because of a TLS
issue. (#837)
* Fix a bug which caused bouncing `-REDIRECT` messages during a FAILOVER (#871)
* Avoid re-establishing replication to a node that is already the primary in `CLUSTER REPLICATE`. (#884)
* Fix `CLUSTER SETSLOT` with `BLOCK` to only wait for replicas that are online. (#879)
* Fix `valkey-cli` to make source node ignores NOREPLICAS when doing the last CLUSTER SETSLOT (#928)
* Exclude '.' and ':' from `isValidAuxChar`'s banned charset to better support
IPV6 addresses. (#963)
* Better handle `-REDIRECT` messages in the MULTI context. (#895)
Performance
===========
* Improve multithreaded performance with memory prefetching. (#861)
* Optimize ZUNION[STORE] command by removing unnecessary accumulator dict. (#829)
* Optimize linear search of WAIT and WAITAOF when unblocking the client. (#787)
* Move prepareClientToWrite out of loop for lrange command to remove redundant calls. (#860)
* Optimize various commands by using sdsAllocSize instead of sdsZmallocSize. (#923)
* Using intrinsics to optimize counting HyperLogLog trailing bits. (#846)
* Free client's MULTI state when it becomes dirty to reduce memory usage. (#961)
* Optimize the fast path of SET if the expiration time has already expired. (#865)
Compatibility Improvements
==========================
* Add 4 sentinel command `GET-PRIMARY-ADDR-BY-NAME`, `PRIMARY`, `PRIMARIES`, and
`IS-PRIMARY-DOWN-BY-ADDR` to allow clients to use inclusive language. (#789)
Behavior Changes
================
* Block unsubscribe related commands for clients that are not in subscribed mode. (#759)
Configuration Changes
=====================
* Change repl-backlog-size from 1MB to 10MB by default. (#911)
* Remove the protected flag from `dual-channel-replication`. (#908)
================================================================================
Valkey 8.0.0 RC1 - Released Thu 1 Aug 2024
================================================================================
Upgrade urgency LOW: This is the first release candidate of Valkey 8.0, with
performance, reliability, and observability improvements. It includes asynchronous
I/O threading, better cluster scaling reliability, dual primary-replica channel
for faster full synchronization, per-slot metrics for resource management, and
experimental RDMA support for increased throughput and reduced latency. This
release is fully compatible with Redis OSS 7.2.4.
Un-deprecated Commands - Cluster
================================
* Un-deprecate the `CLUSTER SLOTS` command. (#536)
New/Modified Commands - Core
============================
* Add `SCRIPT SHOW` sub-command to dump scripts via SHA1. (#617)
* Add `NOSCORES` option to `ZSCAN` command. (#324)
* Add `NOVALUES` option to `HSCAN` command. (Redis#12765)
* Expose Lua `os.clock()` API to allow scripts to determine how long the
script has been executing. (Redis#12971)
* Implement `CLIENT KILL MAXAGE <MAXAGE>`. (Redis#12299)
* Allow running `WAITAOF` in scripts, remove `NOSCRIPT` flag. (Redis#12977)
* Support `XREAD[GROUP]` with `BLOCK` option in scripts. (Redis#12596)
* Introduce `+` as a special ID for the last item in stream on `XREAD`
Command. (Redis#7388, Redis#13117)
New/Modified Commands - Cluster
===============================
* Introduce `CLUSTER SLOT-STATS` command which allows tracking of per slot
metrics for key count, CPU utilization, network bytes in, and network
bytes out. (#20, #351)
* Add `TIMEOUT` option to `CLUSTER SETSLOT` command. (#556, #445)
New/Modified Metrics - Core
===========================
* Introduce per-client metrics for network traffic and command execution
in `CLIENT LIST` and `CLIENT INFO`. (#327)
* Add metrics for DB memory overhead and rehashing status to `INFO MEMORY`
and `MEMORY STATS`. (Redis#12913)
* Add `pubsub_clients` metric to `INFO CLIENTS`. (Redis#12849)
* Add metrics for client buffer limit disconnections to `INFO`. (Redis#12476)
* Add metrics for monitoring clients using `WATCH` command and watched keys.
(Redis#12966)
* Added allocator muzzy memory metrics to `INFO MEMORY` and `MEMORY STATS`.
(Redis#12996)
New Features - Core
===================
* Support replica redirect for read/write operations to primary in standalone
mode. (#325)
* Add server config for cluster blacklist TTL. (#738)
* Add availability zone server config. (#700)
New Features - Cluster
======================
* Support IPv4 and IPv6 dual stack and client-specific IPs in clusters. (#736)
* Support `BY/GET` options for `SORT/SORT_RO` in cluster mode when pattern
implies a single slot. (Redis#12728)
Compatibility Improvements - Core
=================================
* Derive RDB and module child process names based on server start name for
compatibility. (#454)
* Update server identity in `serverPanic` output based on `extended-redis-compatibility`
config. (#415)
Compatibility Improvements - Sentinel
=====================================
* Accept `redis-sentinel` to start Valkey in sentinel mode. (#731)
Performance/Efficiency Improvements - Core
==========================================
* Introduce dual channel for more efficient full sync replication. (#60)
* Introduce async IO threading for improved multi-threaded performance.
(#763, #758)
* Embed key directly in main dictionary entry for improved memory efficiency.
(#541)
* Use thread-local storage to reduce atomic contention in updating memory
metrics. (#674)
* Reduce redundant calls to `prepareClientToWrite` for continuous `addReply*`.
(#670)
* Optimize the logic for checking conversion to skip list during `ZADD` operations.
(#806)
* Optimize `sdsfree` with `zfree_with_size` to avoid redundant size calculation.
(#453)
* Combine events to eliminate redundant `kevent(2)` calls. (#638)
* Introduce shared query buffer for client reads to reduce memory usage. (#258)
* Optimize CRC64 performance for large batches by processing bytes in parallel.
(#350)
* Use `SOCK_NONBLOCK` to reduce system calls for outgoing connections. (#293)
* Enable `accept4()` detection on specific versions of various platforms. (#294)
* Convert CRC16 slot table to fixed-size array for improved memory efficiency.
(Redis#13112)
* Run `SCRIPT FLUSH` truly asynchronously and close Lua interpreter in a
background thread. (Redis#13087)
* Optimize `DEL` command to avoid redundant deletions for expired keys. (Redis#13080)
* Improve defragmentation for large bins to enhance memory efficiency. (Redis#12996)
* Optimize hash table resizing to include empty dictionaries. (Redis#12819)
* Reduce performance impact of dictionary rehashing by optimizing bucket processing.
(Redis#12899)
* Optimize performance for simultaneous client `[P|S]UNSUBSCRIBE`. (Redis#12838)
* Optimize CPU cache efficiency during dictionary rehashing. (Redis#5692)
* Optimize `ZRANGE` offset location from linear search to skip list jump. (Redis#12450)
* Fix `aeSetDontWait` timing to avoid unnecessary waits in `aeProcessEvent`. (Redis#12068)
Performance/Efficiency Improvements - Cluster
=============================================
* Add lightweight cluster message header for Pub/Sub messages. (#654)
* Minor performance improvement in Valkey cluster by avoid initializing key
buffer in `getKeysResult`. (#631)
* Cache `CLUSTER SLOTS` response to improve throughput and reduce latency. (#53)
* Replace slots_to_channels radix tree with slot-specific dictionaries for
shard channels. (Redis#12804)
* Optimize `KEYS` command when pattern includes hashtag and implies a single
slot. (Redis#12754)
* Optimize `SCAN` command with `MATCH` when pattern implies a single slot.
(Redis#12536)
* Replace cluster metadata with slot specific dictionaries to reduce memory
usage when using Valkey cluster. (Redis#11695, Redis#12704)
Reliability Improvements - Core
===============================
* Limit tracking custom errors (e.g. from Lua) while allowing normal errors
to be tracked (#500, Redis#13141)
* Manage maximum number of new connections per cycle to prevent connection
storms. (Redis#12178)
Reliability Improvements - Cluster
==================================
* Reduce fail-over time in Valkey cluster when multiple sequential fail-overs
occurred by resetting `failover_auth_time` when the new primary node goes
down. (#782)
* Restrict node failure marking to primaries with assigned slots. (#634)
* Enhance cluster meet reliability under link failures. (#461)
* Improve reliability of slot migration in Valkey clusters. (#445)
Usability Improvements - Core
=============================
* Re-brand and refine latency report messages. (#644)
* Optimize `ACL LOAD` to avoid disconnecting clients whose users are unchanged.
(Redis#12171)
Usability Improvements - Cluster
================================
* Adjust log levels for various cluster-related logs to improve clarity. (#633)
* Maintain deterministic ordering of replica(s) in `CLUSTER SLOTS` response. (#265)
Usability Improvements - CLI
================================
* Add prompt message when Ctrl-C is pressed in `valkey-cli`. (#702)
* Keep an in-memory history of all commands in `valkey-cli` so that sensitive
commands can be shown within the same session. (Redis#12862)
Module Improvements - Core
==========================
* Add `ValkeyModule_TryCalloc()` and `ValkeyModule_TryRealloc()` to handle
allocation failures gracefully. (Redis#12985)
* Make `ValkeyModule_Yield` thread-safe by handling events in the main thread.
(Redis#12905)
* Allow modules to declare new ACL categories. (Redis#12486)
Module Improvements - Cluster
=============================
* Add API `ValkeyModule_ClusterKeySlot` and `ValkeyModule_ClusterCanonicalKeyNameInSlot`.
(Redis#13069)
Behavior Changes - Core
=======================
* Re-brand the Lua debugger. (#603)
* Change default pidfile from `redis.pid` to `valkey.pid`. (#378)
* Abort transactions on nested `MULTI` or `WATCH` commands. (#723)
* Ensure keys that match the `SCAN` filter are not lazily expired and return
an error for invalid types. (#501)
* Rename `redis` in AOF logs and proc title to `valkey-aof-rewrite`. (#393)
* Change default syslog-ident from `redis` to `valkey`. (#390)
* Update `Redis` to `Valkey` in `serverLog` messages in server.c file. (#231)
* Remove `Redis` from various error reply messages. See GitHub PR for more
details. (#206)
* Reject empty strings for configs `dir`, `dbfilename`, and `cluster-config-file`.
(#636)
* Change key-spec flag from `RW` to `OW` for `SINTERSTORE` command. (Redis#12917)
* Return more precise error messages for some cases verifying keys during script
execution. (Redis#12707)
* Return errors for `BITCOUNT` and `BITPOS` with non-existing keys or invalid
arguments instead of zero. (Redis#11734)
* Validate `BITCOUNT` arguments before key existence check. (Redis#12394)
* Redact ACL username information and mark `*-key-file-pass` configs as
sensitive. (Redis#12860)
* Allow `MULTI/EXEC` to use a small amount of additional memory beyond the
used-memory limit. (Redis#12961)
Behavior Changes - Cluster
==========================
* Allow `CLUSTER NODES/INFO/MYID/MYSHARDID` during loading state. (#596)
* Make cluster replicas return `ASK` and `TRYAGAIN` during slot migration. (#495)
Behavior Changes - Sentinel
===========================
* Replace `master-reboot-down-after-period` with `primary-reboot-down-after-period`
in `sentinel.conf`. (#647)
Bug Fixes - Core
================
* Fix a bug that caused LRU/LFU inconsistencies for some integer objects. (#250)
* Fix a bug where Valkey may use a sub-optimal encoding for some data types.
(Redis#13148)
* Fix propagation of `entries_read` by calling `streamPropagateGroupID`
unconditionally. (Redis#12898)
* Fix race condition issues between the main thread and module threads.
(Redis#12817)
* Wake blocked clients ASAP in next `beforeSleep` for `WAITAOF`. (Redis#12627)
* Fix crash in crash-report and improve thread management with RW locks.
(Redis#12623)
Bug Fixes - Cluster
===================
* Fix a bug where a shard returns the incorrect slot slot information in
`CLUSTER SHARDS` command on primary failure. (#790)
* Allow module authentication to succeed when the cluster is down. (#693)
* Fix `PONG` message processing for primary-ship tracking during fail-overs.
(Redis#13055)
* Prevent double freeing of cluster link with `DEBUG CLUSTERLINK KILL`.
(Redis#12930)
* Unsubscribe all clients from replica for shard channel if the primary
ownership changes. (Redis#12577)
Bug Fixes - Tooling
===============
* Fix `valkey-check-aof` misidentifying data in manifest format as MP-AOF.
(Redis#12958)
* Fix `valkey-cli` to respect the `--count` option without requiring
`--pattern`. (Redis#13092)
* Fix `valkey-benchmark` to distribute operations across all slots owned by
a node in cluster mode. (Redis#12986)
Internal Codebase Improvements
==============================
* Enable debug asserts for cluster and sentinel tests. (#588)
* Introduce a minimal debugger for Tcl integration test suite. (#683)
* Set up clang-format GitHub action for automated code formatting checks. (#538)
* Replace custom atomic logic with C11 _Atomics. (#490)
* Add fast fail option for Tcl test cases. (#482)
* Introduce a simple unit test framework. (#460)
* An initial simple unit test framework. (#344)
* Introduce Codecov for automated code coverage tracking. (#316)
* Remove deprecated `redis-trib` CLI program. (#281)
* Add `-fno-omit-frame-pointer` to default compilation flags to improve
debuggability. (Redis#12973)
* Refactor the per-slot dict-array db.c into a new kvstore data structure.
(Redis#12822)
* Unified database rehash method for both standalone and cluster modes.
(Redis#12848)
* Clarify and decouple the sampling logic in eviction to improve readability.
(Redis#12781)
* Rewrite large printf calls to smaller ones for readability. (Redis#12257)
Experimental
============
* Introduce Valkey Over RDMA transport (experimental). (#477)
We appreciate the efforts of all who contributed code to this release!
lan Slang, Binbin, Brennan, Chen Tianjie, Cui Fliter, Daniel House, Darren Jiang,
David Carlier, Debing Sun, Dingrui, Dmitry Polyakovsky, Eran Liberty, Gabi Ganam,
George Guimares, Guillaume Koenig, Guybe, Harkrishn Patro, Hassaan Khan, Hwang Si Yeon,
ICHINOSE Shogo, icy17, Ikko Eltociear Ashimine, iKun, Itamar Haber, Jachin, Jacob Murphy,
Jason Elbaum, Jeff Liu, John Sully, John Vandenberg, Jonathan Wright, Jonghoonpark, Joe Hu,
Josiah Carlson, Juho Kim, judeng, Jun Luo, K.G. Wang, Karthik Subbarao, Karthick Ariyaratnam,
kell0gg, Kyle Kim, Leibale Eidelman, LiiNen, Lipeng Zhu, Lior Kogan, Lior Lahav, Madelyn Olson,
Makdon, Maria Markova, Mason Hall, Matthew Douglass, meiravgri, michalbiesek, Mike Dolan,
Mikel Olasagasti Uranga, Moshe Kaplan, mwish, naglera, NAM UK KIM, Neal Gompa, nitaicaro,
Nir Rattner, Oran Agra, Ouri Half, Ozan Tezcan, Parth, PatrickJS, Pengfei Han, Pierre, Ping Xie,
poiuj, pshankinclarke, ranshid, Ronen Kalish, Roshan Khatri, Samuel Adetunji, Sankar, secwall,
Sergey Fedorov, Sher_Sun, Shivshankar, skyfirelee, Slava Koyfman, Subhi Al Hasan, sundb,
Ted Lyngmo, Thomas Fline, tison, Tom Morris, Tyler Bream, uriyage, Viktor Söderqvist, Vitaly,
Vitah Lin, VoletiRam, w. ian douglas, WangYu, Wen Hui, Wenwen Chen, Yaacov Hazan, Yanqi Lv,
Yehoshua Hershberg, Yves LeBras, zalj, Zhao Zhao, zhenwei pi, zisong.cw
-1
View File
@@ -1 +0,0 @@
LLM.md
-1
View File
@@ -1 +0,0 @@
LLM.md
-78
View File
@@ -1,78 +0,0 @@
cmake_minimum_required(VERSION 3.10)
# Must be done first
if (APPLE)
# Force clang compiler on macOS
find_program(CLANGPP "clang++")
find_program(CLANG "clang")
if (CLANG AND CLANGPP)
message(STATUS "Found ${CLANGPP}, ${CLANG}")
set(CMAKE_CXX_COMPILER ${CLANGPP})
set(CMAKE_C_COMPILER ${CLANG})
endif ()
endif ()
# Options
option(BUILD_LUA "Build KV Lua scripting engine" ON)
option(BUILD_UNIT_TESTS "Build kv-unit-tests" OFF)
option(BUILD_TEST_MODULES "Build all test modules" OFF)
option(BUILD_EXAMPLE_MODULES "Build example modules" OFF)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
project("kv")
add_compile_options(-Wundef)
set(CMAKE_C_STANDARD 11)
set(CMAKE_C_STANDARD_REQUIRED ON)
set(CMAKE_C_EXTENSIONS ON)
include(KVSetup)
add_subdirectory(src)
add_subdirectory(tests)
# Include the packaging module
include(Packaging)
# Clear cached variables from the cache
unset(BUILD_LUA CACHE)
unset(BUILD_TESTS CACHE)
unset(CLANGPP CACHE)
unset(CLANG CACHE)
unset(BUILD_RDMA_MODULE CACHE)
unset(BUILD_TLS_MODULE CACHE)
unset(BUILD_UNIT_TESTS CACHE)
unset(BUILD_TEST_MODULES CACHE)
unset(BUILD_EXAMPLE_MODULES CACHE)
unset(USE_TLS CACHE)
unset(DEBUG_FORCE_DEFRAG CACHE)
# Helper to copy runtest scripts to allow running tests with CMake built binaries
function(copy_runtest_script script_name)
set(src "${CMAKE_SOURCE_DIR}/${script_name}")
set(dst "${CMAKE_BINARY_DIR}/${script_name}")
file(READ "${src}" contents)
# Split at the first newline (after shebang #!/bin/sh)
string(FIND "${contents}" "\n" index_of_first_newline_char)
math(EXPR first_index_script_body "${index_of_first_newline_char} + 1")
string(SUBSTRING "${contents}" 0 ${first_index_script_body} shebang_line)
string(SUBSTRING "${contents}" ${first_index_script_body} -1 script_body)
# Insert our environment variable lines
set(insert_content
"# Most tests assume running from the project root
cd ${CMAKE_SOURCE_DIR}
export KV_BIN_DIR=\"${CMAKE_BINARY_DIR}/bin\"
")
# Reconstruct the full script
set(new_contents "${shebang_line}${insert_content}${script_body}")
file(WRITE "${dst}" "${new_contents}")
file(CHMOD "${dst}" PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
endfunction()
copy_runtest_script("runtest")
copy_runtest_script("runtest-cluster")
copy_runtest_script("runtest-moduleapi")
copy_runtest_script("runtest-rdma")
copy_runtest_script("runtest-sentinel")
+1 -1
View File
@@ -49,7 +49,7 @@ representative at an online or offline event.
Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
this email address: maintainers@hanzo.ai.
this email address: placeholderkv@gmail.com.
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
reporter of any incident.
+18 -41
View File
@@ -1,23 +1,22 @@
Contributing to KV
Contributing to Valkey
======================
Welcome and thank you for wanting to contribute!
# Project governance
The KV project is led by a Technical Steering Committee, whose responsibilities are laid out in [GOVERNANCE.md](GOVERNANCE.md).
The Valkey project is led by a Technical Steering Committee, whose responsibilities are laid out in [GOVERNANCE.md](GOVERNANCE.md).
## Get started
* Have a question? Ask it on
[GitHub Discussions](https://github.com/hanzoai/kv/discussions)
or [KV's Discord](https://discord.gg/zbcPa5umUB)
or [KV's Matrix](https://matrix.to/#/#kv:matrix.org)
* Found a bug? [Report it here](https://github.com/hanzoai/kv/issues/new?template=bug_report.md&title=%5BBUG%5D)
* KV crashed? [Submit a crash report here](https://github.com/hanzoai/kv/issues/new?template=crash_report.md&title=%5BCRASH%5D+%3Cshort+description%3E)
* Suggest a new feature? [Post your detailed feature request here](https://github.com/hanzoai/kv/issues/new?template=feature_request.md&title=%5BNEW%5D)
* Report a test failure? [Report it here](https://github.com/hanzoai/kv/issues/new?template=test-failure.md)
* Want to help with documentation? [Move on to kv-doc](https://github.com/hanzoai/kv-doc)
[GitHub Discussions](https://github.com/valkey-io/valkey/discussions)
or [Valkey's Discord](https://discord.gg/zbcPa5umUB)
or [Valkey's Matrix](https://matrix.to/#/#valkey:matrix.org)
* Found a bug? [Report it here](https://github.com/valkey-io/valkey/issues/new?template=bug_report.md&title=%5BBUG%5D)
* Valkey crashed? [Submit a crash report here](https://github.com/valkey-io/valkey/issues/new?template=crash_report.md&title=%5BCRASH%5D+%3Cshort+description%3E)
* Suggest a new feature? [Post your detailed feature request here](https://github.com/valkey-io/valkey/issues/new?template=feature_request.md&title=%5BNEW%5D)
* Want to help with documentation? [Move on to valkey-doc](https://github.com/valkey-io/valkey-doc)
* Report a vulnerability? See [SECURITY.md](SECURITY.md)
## Developer Certificate of Origin
@@ -58,7 +57,7 @@ By making a contribution to this project, I certify that:
involved.
```
We require that every contribution to KV to be signed with a DCO. We require the
We require that every contribution to Valkey to be signed with a DCO. We require the
usage of known identity (such as a real or preferred name). We do not accept anonymous
contributors nor those utilizing pseudonyms. A DCO signed commit will contain a line like:
@@ -72,21 +71,22 @@ user.name and user.email are set in your git configs, you can use `git commit` w
or `--signoff` to add the `Signed-off-by` line to the end of the commit message. We also
require revert commits to include a DCO.
If you're contributing code to the KV project in any other form, including
If you're contributing code to the Valkey project in any other form, including
sending a code fragment or patch via private email or public discussion groups,
you need to ensure that the contribution is in accordance with the DCO.
## How to provide a patch or a new feature
# How to provide a patch or a new feature
1. If it is a major feature or a semantical change, please don't start coding
straight away: if your feature is not a conceptual fit you'll lose a lot of
time writing the code without any reason. Start by creating an issue at Github with the
description of, exactly, what you want to accomplish and why. Use cases are important for
features to be accepted. Here you can see if there is consensus about your idea.
time writing the code without any reason. Start by posting in the mailing list
and creating an issue at Github with the description of, exactly, what you want
to accomplish and why. Use cases are important for features to be accepted.
Here you can see if there is consensus about your idea.
2. If in step 1 you get an acknowledgment from the project leaders, use the following
procedure to submit a patch:
1. Fork KV on GitHub ([HOWTO](https://docs.github.com/en/github/getting-started-with-github/fork-a-repo))
1. Fork Valkey on GitHub ([HOWTO](https://docs.github.com/en/github/getting-started-with-github/fork-a-repo))
1. Create a topic branch (`git checkout -b my_branch`)
1. Make the needed changes and commit with a DCO. (`git commit -s`)
1. Push to your branch (`git push origin my_branch`)
@@ -100,32 +100,9 @@ certain issues/PRs over others. If you think your issue/PR is very important
try to popularize it, have other users commenting and sharing their point of
view, and so forth. This helps.
4. While developing code, make sure to refer to our [DEVELOPMENT_GUIDE.md](DEVELOPMENT_GUIDE.md),
which includes documentation about various best practices for writing KV code.
5. For minor fixes, open a pull request on GitHub.
4. For minor fixes, open a pull request on GitHub.
To link a pull request to an existing issue, please write "Fixes #xyz" somewhere
in the pull request description, where xyz is the issue number.
## Running the daily workflow on demand for your branch
Use [`.github/workflows/daily.yml`](.github/workflows/daily.yml) with
`workflow_dispatch` to run daily tests manually on any branch in your fork.
1. Open your fork on GitHub and go to **Actions** -> **Daily**.
2. Click **Run workflow**.
3. In the **Branch** dropdown, select the branch that contains the workflow file you want to use.
4. In the input fields, set:
* `use_repo` to your fork (for example, `your-user/kv`)
* `use_git_ref` to your branch name (or a specific commit SHA)
5. Optionally set `skipjobs`, `skiptests`, `test_args`, and `cluster_test_args`.
6. Click **Run workflow**.
Notes:
* To run the full matrix, set `skipjobs` and `skiptests` to `none`.
Do not leave them empty, since the workflow input defaults may be applied.
* The scheduled part of this workflow is gated to `hanzoai/kv`, but manual
`workflow_dispatch` runs work for forks.
Thanks!
+1 -3
View File
@@ -1,10 +1,8 @@
# License 1
SPDX-License-Identifier: BSD-3-Clause
BSD 3-Clause License
Copyright (c) 2024-present, KV contributors
Copyright (c) 2024-present, Valkey contributors
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
-75
View File
@@ -1,75 +0,0 @@
# KV development guidelines
This document provides a general overview for writing and designing code for KV.
During our long development history, we've made a lot of inconsistent decisions, but we strive to get incrementally better.
## General Best practices
1. Try to limit the number of lines changed in a PR when possible.
We do a lot of backporting as a project, and the more lines changed, the higher the chance of having to resolve merge conflicts.
Please separate refactoring and functional changes into separate PRs, to make it easier to handle backporting.
1. Avoid adding configuration when a feature can be fully controlled by heuristics.
We want KV to work correctly out of the box without much tuning.
Configurations can be added to provide additional tuning of features.
When the workload characteristics can't be inferred or imply a tradeoff (CPU vs memory), then provide a configuration.
## General style guidelines
Most of the style guidelines are enforced by clang format, but some additional comments are included here.
1. C style comments `/* comment */` can be used for both single and multi-line comments.
C++ comments `//` can only be used for single line comments.
Multi line comments should have the leading `*` align and the final `*/` should be on the same line as the last line of text.
e.g.
```c
/* Blah Blah
* Blah Blah. */
```
2. Comments should generally be used to describe behavior that is not obvious from reading the code itself.
This includes complex behavior, why code was written the way it was and describing non-obvious behavior.
Additionally, functions should be documented to explain all of the function's behavior without having to read the code.
1. Generally keep line lengths below 90 characters when reasonable, however there is no explicit line length enforcement.
Use your best judgement for readability.
1. Use static functions when a function is only intended to be accessed from the same file.
For historical reasons, some private functions are prefixed by `_`, and they are kept as is to make it easier to backport changes.
1. Use the boolean type for true/false values.
For historical reasons, some functions used the integer type, and they are kept as is to make it easier to backport changes.
## Naming conventions
KV has a long history of inconsistent naming conventions.
Generally follow the style of the surrounding code, but you can also always use the following conventions for variable and structure names:
- Variable names: `snake_case` or all lower case for short names (e.g. `cached_reply` or `keylen`).
- Function names: `camelCase` or `namespace_camelCase` (e.g. `createStringObject` or `IOJobQueue_isFull`).
- Macros: `UPPER_CASE` (e.g. `MAKE_CMD`).
- Structures: `camelCase` (e.g. `user`).
## Licensing information
When creating new source code files, use the following snippet to indicate the license:
```
/*
* Copyright (c) KV Contributors
* All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*/
```
If you are making material changes to a file that has a different license at the top, also add the above license snippet.
There isn't a well defined test for what is considered a material change, but a good rule of thumb is that material changes are more than 100 lines of code.
## Test coverage
KV uses two types of tests: unit and integration tests.
All contributions should include a test of some form.
Unit tests are present in the `src/unit` directory, and are intended to test individual structures or files.
For example, most changes to data structures should include corresponding unit tests.
Integration tests are located in the `tests/` directory, and are intended to test end-to-end functionality.
Adding new commands should come with corresponding integration tests.
When writing cluster mode tests, do not use the legacy `tests/cluster` framework, which has been deprecated, and instead write tests in `unit/cluster`.
## Documentation
KV keeps most of the user documentation in the [kv-doc](https://github.com/hanzoai/kv-doc) repository in a few areas:
1. Major functionality is documented in the [topics](https://github.com/hanzoai/kv-doc/tree/main/topics) section.
1. Specific command behavior is documented in the [commands](https://github.com/hanzoai/kv-doc/tree/main/commands) section.
Command history is also documented in the [command json file](https://github.com/hanzoai/kv/tree/unstable/src/commands).
1. Server info fields are documented in the [INFO](https://github.com/hanzoai/kv-doc/blob/main/commands/info.md) command.
When a PR is opened that requires documentation to be updated, the `needs-doc-pr` should be added until the corresponding documentation PR is open.
-31
View File
@@ -1,31 +0,0 @@
ARG KV_VERSION=9
# Hanzo KV: High-performance key-value store
FROM kv/kv:${KV_VERSION}-alpine AS base
FROM base
LABEL maintainer="dev@hanzo.ai"
LABEL org.opencontainers.image.source="https://github.com/hanzoai/kv"
LABEL org.opencontainers.image.description="Hanzo KV - High-performance key-value store"
LABEL org.opencontainers.image.vendor="Hanzo AI"
# Install Hanzo KV CLI tools
# Primary names are kv-* ; legacy kv-* names remain as symlinks
RUN cp /usr/local/bin/kv-server /usr/local/bin/kv-server \
&& cp /usr/local/bin/kv-cli /usr/local/bin/kv-cli \
&& ln -sf /usr/local/bin/kv-cli /usr/local/bin/kv \
&& cp /usr/local/bin/kv-sentinel /usr/local/bin/kv-sentinel 2>/dev/null; \
cp /usr/local/bin/kv-benchmark /usr/local/bin/kv-benchmark 2>/dev/null; \
cp /usr/local/bin/kv-check-aof /usr/local/bin/kv-check-aof 2>/dev/null; \
cp /usr/local/bin/kv-check-rdb /usr/local/bin/kv-check-rdb 2>/dev/null; \
true
EXPOSE 6379
HEALTHCHECK --interval=15s --timeout=3s --start-period=10s --retries=3 \
CMD kv ping | grep -q PONG || exit 1
ENTRYPOINT ["kv-server"]
CMD ["--bind", "0.0.0.0", "--dir", "/data", "--maxmemory-policy", "allkeys-lru", "--protected-mode", "no"]
+28 -53
View File
@@ -1,29 +1,22 @@
# Project Governance
The KV project is managed by a Technical Steering Committee (TSC) composed of the maintainers of the KV repository.
The KV project includes all of the current and future repositories under the hanzoai organization.
Committers are defined as individuals with write access to the code within a repository.
Maintainers are defined as individuals with full access to a repository and own its governance.
Both maintainers and committers shall be clearly listed in the MAINTAINERS.md file in a given project's repository.
Maintainers of other repositories within the KV project are not members of the TSC unless explicitly added.
The Valkey project is managed by a Technical Steering Committee (TSC) composed of the maintainers of the Valkey repository.
The Valkey project includes all of the current and future repositories under the Valkey-io organization.
Maintainers are defined as individuals with full commit access to a repository, which shall be in sync with the MAINTAINERS.md file in a given projects repository.
Maintainers of other repositories within the Valkey project are not members of the TSC unless explicitly added.
## Technical Steering Committee
The TSC is responsible for oversight of all technical, project, approval, and policy matters for KV.
The TSC members are listed in the [MAINTAINERS.md](MAINTAINERS.md) file in the KV repository.
At any time, no more than one third (1/3) of the TSC members may be employees, contractors, or representatives of the same organization or affiliated organizations.
For the purposes of this document, “organization” includes companies, corporations, universities, research institutes, non-profits, governmental institutions, and any of their subsidiaries or affiliates.
If, at any time, the 1/3 organization limit is exceeded (for example, due to changes in employment, company acquisitions, or organizational affiliations), the TSC shall be notified as soon as possible.
The TSC must promptly take action to restore compliance, which may include removing or reassigning members in accordance with the procedures outlined in the [Termination of Membership](#termination-of-membership) section.
The TSC shall strive to resolve the situation within 30 days of notification, and document the steps taken to restore compliance.
The TSC is responsible for oversight of all technical, project, approval, and policy matters for Valkey.
The TSC members are listed in the [MAINTAINERS.md](MAINTAINERS.md) file in the Valkey repository.
Maintainers (and accordingly, TSC members) may be added or removed by no less than 2/3 affirmative vote of the current TSC.
The TSC shall appoint a Chair responsible for organizing TSC meetings.
If the TSC Chair is removed from the TSC (or the Chair steps down from that role), it is the responsibility of the TSC to appoint a new Chair.
The TSC can amend this governance document by no less than a 2/3 affirmative vote.
The TSC may, at its discretion, add or remove members who are not maintainers of the main KV repository.
The TSC may, at its discretion, add or remove maintainers from other repositories within the KV project.
The TSC may, at its discretion, add or remove members who are not maintainers of the main Valkey repository.
The TSC may, at its discretion, add or remove maintainers from other repositories within the Valkey project.
## Voting
@@ -33,34 +26,17 @@ Rather, the TSC shall determine consensus based on their good faith consideratio
The TSC shall document evidence of consensus in accordance with these requirements.
If consensus cannot be reached, the TSC shall make the decision by a vote.
A vote shall also be called when an issue or pull request is marked as a major decision, which are decisions that have a significant impact on the KV architecture or design.
### Technical Major Decisions
Technical major decisions include:
* Fundamental changes to the KV core datastructures
* Adding a new data structure or API
* Changes that affect backward compatibility
* New user visible fields that need to be maintained
* Adding or removing a new external library such as a client or module to the project when it affects runtime behavior
Technical major decisions shall be approved by a simple majority vote whenever one can be obtained.
If a simple majority cannot be reached within a two-week voting period, and no TSC member has voted against, the decision may instead be approved through explicit “+2” support from at least two TSC members, recorded on the relevant issue or pull request.
If the pull request author or issue proposer is a TSC member, their +1 counts toward the +2.
If any TSC member casts a negative vote, the decision must follow the simple majority voting process and cannot be approved through +2.
Once a technical major decision has been approved through the +2 mechanism, any subsequent concerns shall be raised through a new major decision process; +2 approvals are not retracted directly.
### Governance Major Decisions
Governance major decisions include:
* Adding TSC members or involuntary removal of TSC members
* Modifying this governance document
* Delegation of maintainership for projects or governance authority
* Creating, modifying, or removing roles within the KV project
* Any change that alters voting rules, TSC responsibilities, or project oversight
* Structural changes to the TSC, including composition limits
Governance major decisions shall require approval by a super-majority vote of at least two thirds (2/3) of the entire TSC.
A vote shall also be called when an issue or pull request is marked as a major decision, which are decisions that have a significant impact on the Valkey architecture or design.
Examples of major decisions:
* Fundamental changes to the Valkey core datastructures
* Adding a new data structure or API
* Changes that affect backward compatibility
* New user visible fields that need to be maintained
* Modifications to the TSC or other governance documents
* Adding members to other roles within the Valkey project
* Delegation of maintainership for projects to other groups or individuals
* Adding or removing a new external library such as a client
or module to the project.
Any member of the TSC can call a vote with reasonable notice to the TSC, setting out a discussion period and a separate voting period.
Any discussion may be conducted in person or electronically by text, voice, or video.
@@ -68,24 +44,23 @@ The discussion shall be open to the public, with the notable exception of discus
In any vote, each voting TSC member will have one vote.
The TSC shall give at least two weeks for all members to submit their vote.
Except as specifically noted elsewhere in this document, decisions by vote require a simple majority vote of all voting members.
If a vote results in a tie, the status quo is preserved.
It is the responsibility of the TSC Chair to help facilitate the voting process as needed to make sure it completes within the voting period.
It is the responsibility of the TSC chair to help facilitate the voting process as needed to make sure it completes within the voting period.
## Termination of Membership
A maintainer's access (and accordingly, their position on the TSC) will be removed if any of the following occur:
* Involuntary Removal: Removal via the [Governance Major Decision](#governance-major-decisions) voting process.
* Resignation: Written notice of resignation to the TSC.
* TSC Vote: 2/3 affirmative vote of the TSC to remove a member
* Unreachable Member: If a member is unresponsive for more than six months, the remaining active members of the TSC may vote to remove the unreachable member by a simple majority.
## Technical direction for other KV projects
## Technical direction for other Valkey projects
The TSC may delegate decision making for other projects within the KV organization to the maintainers responsible for those projects.
Delegation of decision making for a project is considered a [Governance Major Decision](#governance-major-decisions).
Projects within the KV organization must indicate the individuals with commit permissions by updating the MAINTAINERS.md within their repositories.
The TSC may delegate decision making for other projects within the Valkey organization to the maintainers responsible for those projects.
Delegation of decision making for a project is considered a major decision, and shall happen with an explicit vote.
Projects within the Valkey organization must indicate the individuals with commit permissions by updating the MAINTAINERS.md within their repositories.
The TSC may, at its discretion, overrule the decisions made by other projects within the KV organization, although they shall show restraint in doing so.
The TSC may, at its discretion, overrule the decisions made by other projects within the Valkey organization, although they should show restraint in doing so.
## License of this document
-7
View File
@@ -1,7 +0,0 @@
# kv — AI Assistant Context
<p align="center">
<strong>Hanzo KV</strong>
</p>
<p align="center">
+11 -26
View File
@@ -3,36 +3,21 @@
This document contains a list of maintainers in this repo.
See [GOVERNANCE.md](GOVERNANCE.md) that explains the function of this file.
## Committee Chair
- **Madelyn Olson**
Term: March 28, 2024 Present
## Current Maintainers
Maintainers listed in alphabetical order by their github ID.
| Maintainer | GitHub ID | Affiliation |
| ------------------- | ------------- | ----------- |
| Binbin Zhu | @enjoy-binbin | Tencent |
| Harkrishn Patro | @hpatro | Amazon |
| Lucas Yang | @lucasyonge | - |
| Madelyn Olson | @madolson | Amazon |
| Jacob Murphy | @murphyjacob4 | Google |
| Ping Xie | @pingxie | Oracle |
| Ran Shidlansik | @ranshid | Amazon |
| Zhao Zhao | @soloestoy | Alibaba |
| Viktor Söderqvist | @zuiderkwast | Ericsson |
| Maintainer | GitHub ID | Affiliation |
| ------------------- | ----------------------------------------------- | ----------- |
| Zhu Binbin | [enjoy-binbin](https://github.com/enjoy-binbin) | Tencent |
| Wen Hui | [hwware](https://github.com/hwware) | Huawei |
| Madelyn Olson | [madolson](https://github.com/madolson) | Amazon |
| Ping Xie | [pingxie](https://github.com/pingxie) | Google |
| Zhao Zhao | [soloestoy](https://github.com/soloestoy) | Alibaba |
| Viktor Söderqvist | [zuiderkwast](https://github.com/zuiderkwast) | Ericsson |
## Current Committers
Committers listed in alphabetical order by their github ID.
### Former Maintainers
| Committer | GitHub ID | Affiliation |
| ------------------- | ------------- | ----------- |
| Ricardo Dias | @rjd15372 | Percona |
## Former Maintainers and Committers
| Name | GitHub ID | Role | Term | Affiliation |
| ------------------- | ------------- | ---- | ---- | ----------- |
| Maintainer | GitHub ID | Affiliation |
| ------------------- | ----------------------------------------------- | ----------- |
+265 -106
View File
@@ -1,145 +1,304 @@
<p align="center">
<strong>Hanzo KV</strong>
</p>
[![codecov](https://codecov.io/gh/valkey-io/valkey/graph/badge.svg?token=KYYSJAYC5F)](https://codecov.io/gh/valkey-io/valkey)
<p align="center">
High-performance key-value store for the Hanzo ecosystem.<br/>
In-memory data store used as database, cache, streaming engine, and message broker.
</p>
This project was forked from the open source Redis project right before the transition to their new source available licenses.
<p align="center">
<a href="https://github.com/hanzoai/kv/actions"><img src="https://github.com/hanzoai/kv/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
<a href="https://github.com/hanzoai/kv/releases"><img src="https://img.shields.io/github/v/release/hanzoai/kv" alt="Release"></a>
<a href="https://github.com/hanzoai/kv/blob/main/LICENSE"><img src="https://img.shields.io/github/license/hanzoai/kv" alt="License"></a>
</p>
This README is just a fast *quick start* document. More details can be found under [valkey.io](https://valkey.io/)
---
What is Valkey?
--------------
Valkey is a high-performance data structure server that primarily serves key/value workloads.
It supports a wide range of native structures and an extensible plugin system for adding new data structures and access patterns.
## Features
Building Valkey
--------------
- **In-memory key-value store** -- sub-millisecond reads and writes
- **Redis-compatible protocol** -- drop-in replacement for existing Redis clients
- **Persistence** -- RDB snapshots and AOF (append-only file) for durability
- **Replication** -- primary-replica with automatic failover via Sentinel
- **Lua scripting** -- server-side scripting for atomic operations
- **Pub/Sub** -- publish and subscribe messaging
- **Streams** -- append-only log data structure for event sourcing
- **Cluster mode** -- horizontal scaling with automatic sharding
- **Modules** -- extensible plugin system for custom data structures
- **ZAP native** -- built-in [ZAP binary protocol](https://github.com/luxfi/zap) on port 9653 (17x faster than JSON-RPC)
- **Multi-arch** -- linux/amd64 and linux/arm64
Valkey can be compiled and used on Linux, OSX, OpenBSD, NetBSD, FreeBSD.
We support big endian and little endian architectures, and both 32 bit
and 64 bit systems.
## Quick Start
It may compile on Solaris derived systems (for instance SmartOS) but our
support for this platform is *best effort* and Valkey is not guaranteed to
work as well as in Linux, OSX, and \*BSD.
### Docker
It is as simple as:
```bash
docker run -d --name hanzo-kv -p 6379:6379 ghcr.io/hanzoai/kv
% make
To build with TLS support, you'll need OpenSSL development libraries (e.g.
libssl-dev on Debian/Ubuntu).
To build TLS support as Valkey built-in:
% make BUILD_TLS=yes
TO build TLS as Valkey module:
% make BUILD_TLS=module
Note that sentinel mode does not support TLS module.
To build with experimental RDMA support you'll need RDMA development libraries
(e.g. librdmacm-dev and libibverbs-dev on Debian/Ubuntu). For now, Valkey only
supports RDMA as connection module mode. Run:
% make BUILD_RDMA=module
To build with systemd support, you'll need systemd development libraries (such
as libsystemd-dev on Debian/Ubuntu or systemd-devel on CentOS) and run:
% make USE_SYSTEMD=yes
To append a suffix to Valkey program names, use:
% make PROG_SUFFIX="-alt"
You can build a 32 bit Valkey binary using:
% make 32bit
After building Valkey, it is a good idea to test it using:
% make test
The above runs the main integration tests. Additional tests are started using:
% make test-unit # Unit tests
% make test-modules # Tests of the module API
% make test-sentinel # Valkey Sentinel integration tests
% make test-cluster # Valkey Cluster integration tests
More about running the integration tests can be found in
[tests/README.md](tests/README.md) and for unit tests, see
[src/unit/README.md](src/unit/README.md).
Fixing build problems with dependencies or cached build options
---------
Valkey has some dependencies which are included in the `deps` directory.
`make` does not automatically rebuild dependencies even if something in
the source code of dependencies changes.
When you update the source code with `git pull` or when code inside the
dependencies tree is modified in any other way, make sure to use the following
command in order to really clean everything and rebuild from scratch:
% make distclean
This will clean: jemalloc, lua, hiredis, linenoise and other dependencies.
Also if you force certain build options like 32bit target, no C compiler
optimizations (for debugging purposes), and other similar build time options,
those options are cached indefinitely until you issue a `make distclean`
command.
Fixing problems building 32 bit binaries
---------
If after building Valkey with a 32 bit target you need to rebuild it
with a 64 bit target, or the other way around, you need to perform a
`make distclean` in the root directory of the Valkey distribution.
In case of build errors when trying to build a 32 bit binary of Valkey, try
the following steps:
* Install the package libc6-dev-i386 (also try g++-multilib).
* Try using the following command line instead of `make 32bit`:
`make CFLAGS="-m32 -march=native" LDFLAGS="-m32"`
Allocator
---------
Selecting a non-default memory allocator when building Valkey is done by setting
the `MALLOC` environment variable. Valkey is compiled and linked against libc
malloc by default, with the exception of jemalloc being the default on Linux
systems. This default was picked because jemalloc has proven to have fewer
fragmentation problems than libc malloc.
To force compiling against libc malloc, use:
% make MALLOC=libc
To compile against jemalloc on Mac OS X systems, use:
% make MALLOC=jemalloc
Monotonic clock
---------------
By default, Valkey will build using the POSIX clock_gettime function as the
monotonic clock source. On most modern systems, the internal processor clock
can be used to improve performance. Cautions can be found here:
http://oliveryang.net/2015/09/pitfalls-of-TSC-usage/
To build with support for the processor's internal instruction clock, use:
% make CFLAGS="-DUSE_PROCESSOR_CLOCK"
Verbose build
-------------
Valkey will build with a user-friendly colorized output by default.
If you want to see a more verbose output, use the following:
% make V=1
Running Valkey
-------------
To run Valkey with the default configuration, just type:
% cd src
% ./valkey-server
If you want to provide your valkey.conf, you have to run it using an additional
parameter (the path of the configuration file):
% cd src
% ./valkey-server /path/to/valkey.conf
It is possible to alter the Valkey configuration by passing parameters directly
as options using the command line. Examples:
% ./valkey-server --port 9999 --replicaof 127.0.0.1 6379
% ./valkey-server /etc/valkey/6379.conf --loglevel debug
All the options in valkey.conf are also supported as options using the command
line, with exactly the same name.
Running Valkey with TLS:
------------------
### Running manually
To manually run a Valkey server with TLS mode (assuming `./gen-test-certs.sh` was invoked so sample certificates/keys are available):
* TLS built-in mode:
```
./src/valkey-server --tls-port 6379 --port 0 \
--tls-cert-file ./tests/tls/valkey.crt \
--tls-key-file ./tests/tls/valkey.key \
--tls-ca-cert-file ./tests/tls/ca.crt
```
* TLS module mode:
```
./src/valkey-server --tls-port 6379 --port 0 \
--tls-cert-file ./tests/tls/valkey.crt \
--tls-key-file ./tests/tls/valkey.key \
--tls-ca-cert-file ./tests/tls/ca.crt \
--loadmodule src/valkey-tls.so
```
Note that you can disable TCP by specifying `--port 0` explicitly.
It's also possible to have both TCP and TLS available at the same time,
but you'll have to assign different ports.
Use `valkey-cli` to connect to the Valkey server:
```
./src/valkey-cli --tls \
--cert ./tests/tls/valkey.crt \
--key ./tests/tls/valkey.key \
--cacert ./tests/tls/ca.crt
```
### Connect
Specifying `--tls-replication yes` makes a replica connect to the primary.
```bash
docker exec -it hanzo-kv kv
Using `--tls-cluster yes` makes Valkey Cluster use TLS across nodes.
127.0.0.1:6379> SET hello world
OK
127.0.0.1:6379> GET hello
"world"
```
Running Valkey with RDMA:
------------------
Any Redis-compatible CLI also works out of the box.
Note that Valkey Over RDMA is an experimental feature.
It may be changed or removed in any minor or major version.
Currently, it is only supported on Linux.
### Build from Source
To manually run a Valkey server with RDMA mode:
```bash
make
make test
make install
```
% ./src/valkey-server --protected-mode no \
--loadmodule src/valkey-rdma.so bind=192.168.122.100 port=6379
## CLI Tools
It's possible to change bind address/port of RDMA by runtime command:
| Command | Description |
|---------|-------------|
| `kv` | Interactive CLI (default) |
| `kv-server` | Start KV server |
| `kv-cli` | Command-line client |
| `kv-sentinel` | High-availability sentinel |
| `kv-benchmark` | Performance benchmarking |
| `kv-check-aof` | AOF file integrity check |
| `kv-check-rdb` | RDB file integrity check |
192.168.122.100:6379> CONFIG SET rdma.port 6380
## Configuration
It's also possible to have both RDMA and TCP available, and there is no
conflict of TCP(6379) and RDMA(6379), Ex:
Pass a config file at startup:
% ./src/valkey-server --protected-mode no \
--loadmodule src/valkey-rdma.so bind=192.168.122.100 port=6379 \
--port 6379
```bash
kv-server /etc/kv/kv.conf
```
Note that the network card (192.168.122.100 of this example) should support
RDMA. To test a server supports RDMA or not:
Or set options via command line:
% rdma res show (a new version iproute2 package)
Or:
```bash
kv-server --port 6379 --maxmemory 256mb --appendonly yes
```
% ibv_devices
## ZAP Binary Protocol
Hanzo KV speaks [ZAP](https://github.com/luxfi/zap) natively on port **9653** — no sidecar needed.
Playing with Valkey
------------------
ZAP is a zero-copy binary protocol that's 17x faster than JSON-RPC with 11x less memory usage.
You can use valkey-cli to play with Valkey. Start a valkey-server instance,
then in another terminal try the following:
### Enable ZAP
% cd src
% ./valkey-cli
valkey> ping
PONG
valkey> set foo bar
OK
valkey> get foo
"bar"
valkey> incr mycounter
(integer) 1
valkey> incr mycounter
(integer) 2
valkey>
ZAP is enabled by default. Load the module:
Installing Valkey
-----------------
```bash
kv-server --loadmodule /path/to/zap.so
# or with custom port:
kv-server --loadmodule /path/to/zap.so PORT 9653
```
In order to install Valkey binaries into /usr/local/bin, just use:
### ZAP Operations
% make install
| Path | Body | Description |
|------|------|-------------|
| `/get` | `{"key":"mykey"}` | GET a key |
| `/set` | `{"key":"mykey","value":"myval"}` | SET a key |
| `/del` | `{"key":"mykey"}` | DEL a key |
| `/cmd` | `{"cmd":"PING","args":[]}` | Execute any command |
You can use `make PREFIX=/some/other/directory install` if you wish to use a
different destination.
### Module API
_Note_: For compatibility with Redis, we create symlinks from the Redis names (`redis-server`, `redis-cli`, etc.) to the Valkey binaries installed by `make install`.
The symlinks are created in same directory as the Valkey binaries.
The symlinks are removed when using `make uninstall`.
The creation of the symlinks can be skipped by setting the makefile variable `USE_REDIS_SYMLINKS=no`.
Develop custom modules using the KV Module API:
`make install` will just install binaries in your system, but will not configure
init scripts and configuration files in the appropriate place. This is not
needed if you just want to play a bit with Valkey, but if you are installing
it the proper way for a production system, we have a script that does this
for Ubuntu and Debian systems:
```c
#include "kvmodule.h"
% cd utils
% ./install_server.sh
int KVModule_OnLoad(KVModuleCtx *ctx, KVModuleString **argv, int argc) {
if (KVModule_Init(ctx, "mymod", 1, KVMODULE_APIVER_1) == KVMODULE_ERR)
return KVMODULE_ERR;
// register commands...
return KVMODULE_OK;
}
```
_Note_: `install_server.sh` will not work on Mac OSX; it is built for Linux only.
## Client SDKs
The script will ask you a few questions and will setup everything you need
to run Valkey properly as a background daemon that will start again on
system reboots.
| Language | Package | Install |
|----------|---------|---------|
| Python | [hanzo-kv](https://pypi.org/project/hanzo-kv) | `pip install hanzo-kv` |
| Go | [hanzo/kv-go](https://github.com/hanzoai/kv-go) | `go get github.com/hanzoai/kv-go` |
| Node.js | [@hanzo/kv](https://github.com/hanzoai/kv-client) | `npm install @hanzo/kv` |
You'll be able to stop and start Valkey using the script named
`/etc/init.d/valkey_<portnumber>`, for instance `/etc/init.d/valkey_6379`.
Any Redis-compatible client library will also work.
Code contributions
-----------------
Please see the [CONTRIBUTING.md][2]. For security bugs and vulnerabilities, please see [SECURITY.md][3].
## Documentation
[1]: https://github.com/valkey-io/valkey/blob/unstable/COPYING
[2]: https://github.com/valkey-io/valkey/blob/unstable/CONTRIBUTING.md
[3]: https://github.com/valkey-io/valkey/blob/unstable/SECURITY.md
Full documentation is available at [docs.hanzo.ai](https://docs.hanzo.ai).
## License
BSD-3-Clause
Copyright (c) 2024-2026 Hanzo AI Inc. All rights reserved.
Valkey is an open community project under LF Projects
-----------------
Valkey a Series of LF Projects, LLC
2810 N Church St, PMB 57274
Wilmington, Delaware 19802-4447
+3 -3
View File
@@ -1,6 +1,6 @@
## Reporting a Vulnerability
If you believe you've discovered a security vulnerability, please contact the KV team at security@hanzo.ai.
If you believe you've discovered a security vulnerability, please contact the Valkey team at security@lists.valkey.io.
Please *DO NOT* create an issue.
We follow a responsible disclosure procedure, so depending on the severity of the issue we may notify KV vendors about the issue before releasing it publicly.
If you would like to be added to our list of vendors, please reach out to the KV team at maintainers@hanzo.ai.
We follow a responsible disclosure procedure, so depending on the severity of the issue we may notify Valkey vendors about the issue before releasing it publicly.
If you would like to be added to our list of vendors, please reach out to the Valkey team at maintainers@lists.valkey.io.
-379
View File
@@ -1,379 +0,0 @@
include(CheckIncludeFiles)
include(ProcessorCount)
include(Utils)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
# Generate compile_commands.json file for IDEs code completion support
set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
processorcount(KV_PROCESSOR_COUNT)
message(STATUS "Processor count: ${KV_PROCESSOR_COUNT}")
# Installed executables will have this permissions
set(KV_EXE_PERMISSIONS
OWNER_EXECUTE
OWNER_WRITE
OWNER_READ
GROUP_EXECUTE
GROUP_READ
WORLD_EXECUTE
WORLD_READ)
set(KV_SERVER_CFLAGS "")
set(KV_SERVER_LDFLAGS "")
# ----------------------------------------------------
# Helper functions & macros
# ----------------------------------------------------
macro (add_kv_server_compiler_options value)
set(KV_SERVER_CFLAGS "${KV_SERVER_CFLAGS} ${value}")
endmacro ()
macro (add_kv_server_linker_option value)
list(APPEND KV_SERVER_LDFLAGS ${value})
endmacro ()
macro (get_kv_server_linker_option return_value)
list(JOIN KV_SERVER_LDFLAGS " " ${value} ${return_value})
endmacro ()
set(IS_FREEBSD 0)
if (CMAKE_SYSTEM_NAME MATCHES "^.*BSD$|DragonFly")
message(STATUS "Building for FreeBSD compatible system")
set(IS_FREEBSD 1)
include_directories("/usr/local/include")
add_kv_server_compiler_options("-DUSE_BACKTRACE")
endif ()
# Helper function for creating symbolic link so that: link -> source
macro (kv_create_symlink source link)
add_custom_command(
TARGET ${source} POST_BUILD
COMMAND ${CMAKE_COMMAND} -E create_symlink
"$<TARGET_FILE_NAME:${source}>"
"$<TARGET_FILE_DIR:${source}>/${link}"
VERBATIM
)
endmacro ()
# Install a binary
macro (kv_install_bin target)
# Install cli tool and create a redis symbolic link
install(
TARGETS ${target}
DESTINATION ${CMAKE_INSTALL_BINDIR}
PERMISSIONS ${KV_EXE_PERMISSIONS}
COMPONENT "kv")
endmacro ()
# Helper function that defines, builds and installs `target` In addition, it creates a symbolic link between the target
# and `link_name`
macro (kv_build_and_install_bin target sources ld_flags libs link_name)
add_executable(${target} ${sources})
if (USE_JEMALLOC
OR USE_TCMALLOC
OR USE_TCMALLOC_MINIMAL)
# Using custom allocator
target_link_libraries(${target} ${ALLOCATOR_LIB})
endif ()
# Place this line last to ensure that ${ld_flags} is placed last on the linker line
target_link_libraries(${target} ${libs} ${ld_flags})
target_link_libraries(${target} kv::kv)
if (USE_TLS)
# Add required libraries needed for TLS
target_link_libraries(${target} OpenSSL::SSL kv::kv_tls)
endif ()
if (USE_RDMA)
# Add required libraries needed for RDMA
target_link_libraries(${target} kv::kv_rdma)
endif ()
if (IS_FREEBSD)
target_link_libraries(${target} execinfo)
endif ()
# Enable all warnings + fail on warning
target_compile_options(${target} PRIVATE -Werror -Wall)
# Install cli tool and create a redis symbolic link
kv_install_bin(${target})
kv_create_symlink(${target} ${link_name})
endmacro ()
# Determine if we are building in Release or Debug mode
if (CMAKE_BUILD_TYPE MATCHES Debug OR CMAKE_BUILD_TYPE MATCHES DebugFull)
set(KV_DEBUG_BUILD 1)
set(KV_RELEASE_BUILD 0)
message(STATUS "Building in debug mode")
else ()
set(KV_DEBUG_BUILD 0)
set(KV_RELEASE_BUILD 1)
message(STATUS "Building in release mode")
endif ()
# ----------------------------------------------------
# Helper functions - end
# ----------------------------------------------------
# ----------------------------------------------------
# Build options (allocator, tls, rdma et al)
# ----------------------------------------------------
if (NOT BUILD_MALLOC)
if (APPLE)
set(BUILD_MALLOC "libc")
elseif (UNIX)
set(BUILD_MALLOC "jemalloc")
endif ()
endif ()
# User may pass different allocator library. Using -DBUILD_MALLOC=<libname>, make sure it is a valid value
if (BUILD_MALLOC)
if ("${BUILD_MALLOC}" STREQUAL "jemalloc")
set(MALLOC_LIB "jemalloc")
set(ALLOCATOR_LIB "jemalloc")
add_kv_server_compiler_options("-DUSE_JEMALLOC")
set(USE_JEMALLOC 1)
elseif ("${BUILD_MALLOC}" STREQUAL "libc")
set(MALLOC_LIB "libc")
elseif ("${BUILD_MALLOC}" STREQUAL "tcmalloc")
set(MALLOC_LIB "tcmalloc")
kv_pkg_config(libtcmalloc ALLOCATOR_LIB)
add_kv_server_compiler_options("-DUSE_TCMALLOC")
set(USE_TCMALLOC 1)
elseif ("${BUILD_MALLOC}" STREQUAL "tcmalloc_minimal")
set(MALLOC_LIB "tcmalloc_minimal")
kv_pkg_config(libtcmalloc_minimal ALLOCATOR_LIB)
add_kv_server_compiler_options("-DUSE_TCMALLOC")
set(USE_TCMALLOC_MINIMAL 1)
else ()
message(FATAL_ERROR "BUILD_MALLOC can be one of: jemalloc, libc, tcmalloc or tcmalloc_minimal")
endif ()
endif ()
message(STATUS "Using ${MALLOC_LIB}")
# TLS support
if (BUILD_TLS)
kv_parse_build_option(${BUILD_TLS} USE_TLS)
if (USE_TLS EQUAL 1)
# Only search for OpenSSL if needed
find_package(OpenSSL REQUIRED)
message(STATUS "OpenSSL include dir: ${OPENSSL_INCLUDE_DIR}")
message(STATUS "OpenSSL libraries: ${OPENSSL_LIBRARIES}")
include_directories(${OPENSSL_INCLUDE_DIR})
endif ()
if (USE_TLS EQUAL 1)
add_kv_server_compiler_options("-DUSE_OPENSSL=1")
add_kv_server_compiler_options("-DBUILD_TLS_MODULE=0")
else ()
# Build TLS as a module RDMA can only be built as a module. So disable it
message(WARNING "BUILD_TLS can be one of: [ON | OFF | 1 | 0], but '${BUILD_TLS}' was provided")
message(STATUS "TLS support is disabled")
set(USE_TLS 0)
endif ()
else ()
# By default, TLS is disabled
message(STATUS "TLS is disabled")
set(USE_TLS 0)
endif ()
if (BUILD_RDMA)
set(BUILD_RDMA_MODULE 0)
# RDMA support (Linux only)
if (LINUX AND NOT APPLE)
kv_parse_build_option(${BUILD_RDMA} USE_RDMA)
find_package(PkgConfig REQUIRED)
# Locate librdmacm & libibverbs, fail if we can't find them
kv_pkg_config(librdmacm RDMACM_LIBS)
kv_pkg_config(libibverbs IBVERBS_LIBS)
message(STATUS "${RDMACM_LIBS};${IBVERBS_LIBS}")
list(APPEND RDMA_LIBS "${RDMACM_LIBS};${IBVERBS_LIBS}")
if (USE_RDMA EQUAL 2) # Module
message(STATUS "Building RDMA as module")
add_kv_server_compiler_options("-DUSE_RDMA=2")
set(BUILD_RDMA_MODULE 2)
elseif (USE_RDMA EQUAL 1) # Builtin
message(STATUS "Building RDMA as builtin")
add_kv_server_compiler_options("-DUSE_RDMA=1")
add_kv_server_compiler_options("-DBUILD_RDMA_MODULE=0")
list(APPEND SERVER_LIBS "${RDMA_LIBS}")
endif ()
else ()
message(WARNING "RDMA is only supported on Linux platforms")
endif ()
else ()
# By default, RDMA is disabled
message(STATUS "RDMA is disabled")
set(USE_RDMA 0)
endif ()
set(BUILDING_ARM64 0)
set(BUILDING_ARM32 0)
if ("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "arm64" OR "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64")
set(BUILDING_ARM64 1)
endif ()
if ("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "arm")
set(BUILDING_ARM32 1)
endif ()
message(STATUS "Building on ${CMAKE_HOST_SYSTEM_NAME}")
if (BUILDING_ARM64)
message(STATUS "Compiling kv for ARM64")
add_kv_server_linker_option("-funwind-tables")
endif ()
if (APPLE)
add_kv_server_linker_option("-rdynamic")
add_kv_server_linker_option("-ldl")
elseif (UNIX)
add_kv_server_linker_option("-rdynamic")
add_kv_server_linker_option("-pthread")
add_kv_server_linker_option("-ldl")
add_kv_server_linker_option("-lm")
endif ()
if (KV_DEBUG_BUILD)
# Debug build, use enable "-fno-omit-frame-pointer"
add_kv_server_compiler_options("-fno-omit-frame-pointer")
endif ()
# Check for Atomic
check_include_files(stdatomic.h HAVE_C11_ATOMIC)
if (HAVE_C11_ATOMIC)
add_kv_server_compiler_options("-std=gnu11")
else ()
add_kv_server_compiler_options("-std=c99")
endif ()
# Sanitizer
if (BUILD_SANITIZER)
# Common CFLAGS
list(APPEND KV_SANITAIZER_CFLAGS "-fno-sanitize-recover=all")
list(APPEND KV_SANITAIZER_CFLAGS "-fno-omit-frame-pointer")
if ("${BUILD_SANITIZER}" STREQUAL "address")
list(APPEND KV_SANITAIZER_CFLAGS "-fsanitize=address")
list(APPEND KV_SANITAIZER_LDFLAGS "-fsanitize=address")
elseif ("${BUILD_SANITIZER}" STREQUAL "thread")
list(APPEND KV_SANITAIZER_CFLAGS "-fsanitize=thread")
list(APPEND KV_SANITAIZER_LDFLAGS "-fsanitize=thread")
elseif ("${BUILD_SANITIZER}" STREQUAL "undefined")
list(APPEND KV_SANITAIZER_CFLAGS "-fsanitize=undefined")
list(APPEND KV_SANITAIZER_LDFLAGS "-fsanitize=undefined")
else ()
message(FATAL_ERROR "Unknown sanitizer: ${BUILD_SANITIZER}")
endif ()
endif ()
include_directories("${CMAKE_SOURCE_DIR}/deps/libkv/include")
include_directories("${CMAKE_SOURCE_DIR}/src/modules/lua")
include_directories("${CMAKE_SOURCE_DIR}/deps/linenoise")
include_directories("${CMAKE_SOURCE_DIR}/deps/hdr_histogram")
include_directories("${CMAKE_SOURCE_DIR}/deps/fpconv")
add_subdirectory("${CMAKE_SOURCE_DIR}/deps")
# Update linker flags for the allocator
if (USE_JEMALLOC)
include_directories("${CMAKE_SOURCE_DIR}/deps/jemalloc/include")
endif ()
# Common compiler flags
add_kv_server_compiler_options("-pedantic")
if (NOT BUILD_LUA)
message(STATUS "Lua scripting engine is disabled")
endif()
# ----------------------------------------------------
# Build options (allocator, tls, rdma et al) - end
# ----------------------------------------------------
# -------------------------------------------------
# Code Generation section
# -------------------------------------------------
find_program(PYTHON_EXE python3)
if (PYTHON_EXE)
# Python based code generation
message(STATUS "Found python3: ${PYTHON_EXE}")
# Rule for generating commands.def file from json files
message(STATUS "Adding target generate_commands_def")
file(GLOB COMMAND_FILES_JSON "${CMAKE_SOURCE_DIR}/src/commands/*.json")
add_custom_command(
OUTPUT ${CMAKE_BINARY_DIR}/commands_def_generated
DEPENDS ${COMMAND_FILES_JSON}
COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/utils/generate-command-code.py
COMMAND touch ${CMAKE_BINARY_DIR}/commands_def_generated
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/src")
add_custom_target(generate_commands_def DEPENDS ${CMAKE_BINARY_DIR}/commands_def_generated)
# Rule for generating fmtargs.h
message(STATUS "Adding target generate_fmtargs_h")
add_custom_command(
OUTPUT ${CMAKE_BINARY_DIR}/fmtargs_generated
DEPENDS ${CMAKE_SOURCE_DIR}/utils/generate-fmtargs.py
COMMAND sed '/Everything/,$$d' fmtargs.h > fmtargs.h.tmp
COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/utils/generate-fmtargs.py >> fmtargs.h.tmp
COMMAND mv fmtargs.h.tmp fmtargs.h
COMMAND touch ${CMAKE_BINARY_DIR}/fmtargs_generated
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/src")
add_custom_target(generate_fmtargs_h DEPENDS ${CMAKE_BINARY_DIR}/fmtargs_generated)
# Rule for generating test_files.h
message(STATUS "Adding target generate_test_files_h")
file(GLOB UNIT_TEST_SRCS "${CMAKE_SOURCE_DIR}/src/unit/*.c")
add_custom_command(
OUTPUT ${CMAKE_BINARY_DIR}/test_files_generated
DEPENDS "${UNIT_TEST_SRCS};${CMAKE_SOURCE_DIR}/utils/generate-unit-test-header.py"
COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/utils/generate-unit-test-header.py
COMMAND touch ${CMAKE_BINARY_DIR}/test_files_generated
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/src")
add_custom_target(generate_test_files_h DEPENDS ${CMAKE_BINARY_DIR}/test_files_generated)
else ()
# Fake targets
add_custom_target(generate_commands_def)
add_custom_target(generate_fmtargs_h)
add_custom_target(generate_test_files_h)
endif ()
# Generate release.h file (always)
add_custom_target(
release_header
COMMAND sh -c '${CMAKE_SOURCE_DIR}/src/mkreleasehdr.sh'
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/src")
# -------------------------------------------------
# Code Generation section - end
# -------------------------------------------------
# ----------------------------------------------------------
# All our source files are defined in SourceFiles.cmake file
# ----------------------------------------------------------
include(SourceFiles)
# Clear the below variables from the cache
unset(CMAKE_C_FLAGS CACHE)
unset(KV_SERVER_LDFLAGS CACHE)
unset(KV_SERVER_CFLAGS CACHE)
unset(PYTHON_EXE CACHE)
unset(HAVE_C11_ATOMIC CACHE)
unset(USE_TLS CACHE)
unset(USE_RDMA CACHE)
unset(BUILD_TLS CACHE)
unset(BUILD_RDMA CACHE)
unset(BUILD_MALLOC CACHE)
unset(USE_JEMALLOC CACHE)
unset(BUILD_TLS_MODULE CACHE)
unset(BUILD_TLS_BUILTIN CACHE)
-44
View File
@@ -1,44 +0,0 @@
set(CPACK_PACKAGE_NAME "kv")
kv_parse_version(CPACK_PACKAGE_VERSION_MAJOR CPACK_PACKAGE_VERSION_MINOR CPACK_PACKAGE_VERSION_PATCH)
set(CPACK_PACKAGE_CONTACT "maintainers@lists.kv.io")
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "KV is an open source (BSD) high-performance key/value datastore")
set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/COPYING")
set(CPACK_RESOURCE_FILE_README "${CMAKE_SOURCE_DIR}/README.md")
set(CPACK_STRIP_FILES TRUE)
kv_get_distro_name(DISTRO_NAME)
message(STATUS "Current host distro: ${DISTRO_NAME}")
if (DISTRO_NAME MATCHES ubuntu
OR DISTRO_NAME MATCHES debian
OR DISTRO_NAME MATCHES mint)
message(STATUS "Adding target package for ${DISTRO_NAME}")
set(CPACK_PACKAGING_INSTALL_PREFIX "/opt/kv")
# Debian related parameters
set(CPACK_DEBIAN_PACKAGE_MAINTAINER "KV contributors")
set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT)
set(CPACK_GENERATOR "DEB")
endif ()
include(CPack)
unset(DISTRO_NAME CACHE)
# ---------------------------------------------------
# Create a helper script for creating symbolic links
# ---------------------------------------------------
write_file(
${CMAKE_BINARY_DIR}/CreateSymlink.sh
"\
#!/bin/bash \n\
if [ -z \${DESTDIR} ]; then \n\
# Script is called during 'make install' \n\
PREFIX=${CMAKE_INSTALL_PREFIX}/bin \n\
else \n\
# Script is called during 'make package' \n\
PREFIX=\${DESTDIR}${CPACK_PACKAGING_INSTALL_PREFIX}/bin \n\
fi \n\
cd \$PREFIX \n\
ln -sf \$1 \$2")
-178
View File
@@ -1,178 +0,0 @@
# -------------------------------------------------
# Define the sources to be built
# -------------------------------------------------
# kv-server source files
set(KV_SERVER_SRCS
${CMAKE_SOURCE_DIR}/src/threads_mngr.c
${CMAKE_SOURCE_DIR}/src/adlist.c
${CMAKE_SOURCE_DIR}/src/vector.c
${CMAKE_SOURCE_DIR}/src/quicklist.c
${CMAKE_SOURCE_DIR}/src/ae.c
${CMAKE_SOURCE_DIR}/src/anet.c
${CMAKE_SOURCE_DIR}/src/dict.c
${CMAKE_SOURCE_DIR}/src/hashtable.c
${CMAKE_SOURCE_DIR}/src/kvstore.c
${CMAKE_SOURCE_DIR}/src/sds.c
${CMAKE_SOURCE_DIR}/src/zmalloc.c
${CMAKE_SOURCE_DIR}/src/lzf_c.c
${CMAKE_SOURCE_DIR}/src/lzf_d.c
${CMAKE_SOURCE_DIR}/src/pqsort.c
${CMAKE_SOURCE_DIR}/src/zipmap.c
${CMAKE_SOURCE_DIR}/src/sha1.c
${CMAKE_SOURCE_DIR}/src/ziplist.c
${CMAKE_SOURCE_DIR}/src/release.c
${CMAKE_SOURCE_DIR}/src/memory_prefetch.c
${CMAKE_SOURCE_DIR}/src/io_threads.c
${CMAKE_SOURCE_DIR}/src/networking.c
${CMAKE_SOURCE_DIR}/src/util.c
${CMAKE_SOURCE_DIR}/src/object.c
${CMAKE_SOURCE_DIR}/src/db.c
${CMAKE_SOURCE_DIR}/src/replication.c
${CMAKE_SOURCE_DIR}/src/rdb.c
${CMAKE_SOURCE_DIR}/src/t_string.c
${CMAKE_SOURCE_DIR}/src/t_list.c
${CMAKE_SOURCE_DIR}/src/t_set.c
${CMAKE_SOURCE_DIR}/src/t_zset.c
${CMAKE_SOURCE_DIR}/src/t_hash.c
${CMAKE_SOURCE_DIR}/src/config.c
${CMAKE_SOURCE_DIR}/src/aof.c
${CMAKE_SOURCE_DIR}/src/pubsub.c
${CMAKE_SOURCE_DIR}/src/multi.c
${CMAKE_SOURCE_DIR}/src/debug.c
${CMAKE_SOURCE_DIR}/src/sort.c
${CMAKE_SOURCE_DIR}/src/intset.c
${CMAKE_SOURCE_DIR}/src/syncio.c
${CMAKE_SOURCE_DIR}/src/cluster.c
${CMAKE_SOURCE_DIR}/src/cluster_migrateslots.c
${CMAKE_SOURCE_DIR}/src/cluster_legacy.c
${CMAKE_SOURCE_DIR}/src/cluster_slot_stats.c
${CMAKE_SOURCE_DIR}/src/crc16.c
${CMAKE_SOURCE_DIR}/src/commandlog.c
${CMAKE_SOURCE_DIR}/src/eval.c
${CMAKE_SOURCE_DIR}/src/bio.c
${CMAKE_SOURCE_DIR}/src/rio.c
${CMAKE_SOURCE_DIR}/src/rand.c
${CMAKE_SOURCE_DIR}/src/memtest.c
${CMAKE_SOURCE_DIR}/src/syscheck.c
${CMAKE_SOURCE_DIR}/src/crcspeed.c
${CMAKE_SOURCE_DIR}/src/crccombine.c
${CMAKE_SOURCE_DIR}/src/crc64.c
${CMAKE_SOURCE_DIR}/src/bitops.c
${CMAKE_SOURCE_DIR}/src/sentinel.c
${CMAKE_SOURCE_DIR}/src/notify.c
${CMAKE_SOURCE_DIR}/src/setproctitle.c
${CMAKE_SOURCE_DIR}/src/blocked.c
${CMAKE_SOURCE_DIR}/src/hyperloglog.c
${CMAKE_SOURCE_DIR}/src/latency.c
${CMAKE_SOURCE_DIR}/src/sparkline.c
${CMAKE_SOURCE_DIR}/src/kv-check-rdb.c
${CMAKE_SOURCE_DIR}/src/kv-check-aof.c
${CMAKE_SOURCE_DIR}/src/geo.c
${CMAKE_SOURCE_DIR}/src/lazyfree.c
${CMAKE_SOURCE_DIR}/src/module.c
${CMAKE_SOURCE_DIR}/src/lrulfu.c
${CMAKE_SOURCE_DIR}/src/evict.c
${CMAKE_SOURCE_DIR}/src/expire.c
${CMAKE_SOURCE_DIR}/src/geohash.c
${CMAKE_SOURCE_DIR}/src/geohash_helper.c
${CMAKE_SOURCE_DIR}/src/childinfo.c
${CMAKE_SOURCE_DIR}/src/allocator_defrag.c
${CMAKE_SOURCE_DIR}/src/defrag.c
${CMAKE_SOURCE_DIR}/src/siphash.c
${CMAKE_SOURCE_DIR}/src/rax.c
${CMAKE_SOURCE_DIR}/src/t_stream.c
${CMAKE_SOURCE_DIR}/src/listpack.c
${CMAKE_SOURCE_DIR}/src/localtime.c
${CMAKE_SOURCE_DIR}/src/lolwut.c
${CMAKE_SOURCE_DIR}/src/lolwut5.c
${CMAKE_SOURCE_DIR}/src/lolwut6.c
${CMAKE_SOURCE_DIR}/src/lolwut9.c
${CMAKE_SOURCE_DIR}/src/acl.c
${CMAKE_SOURCE_DIR}/src/tracking.c
${CMAKE_SOURCE_DIR}/src/socket.c
${CMAKE_SOURCE_DIR}/src/tls.c
${CMAKE_SOURCE_DIR}/src/rdma.c
${CMAKE_SOURCE_DIR}/src/sha256.c
${CMAKE_SOURCE_DIR}/src/timeout.c
${CMAKE_SOURCE_DIR}/src/setcpuaffinity.c
${CMAKE_SOURCE_DIR}/src/monotonic.c
${CMAKE_SOURCE_DIR}/src/mt19937-64.c
${CMAKE_SOURCE_DIR}/src/resp_parser.c
${CMAKE_SOURCE_DIR}/src/call_reply.c
${CMAKE_SOURCE_DIR}/src/script.c
${CMAKE_SOURCE_DIR}/src/functions.c
${CMAKE_SOURCE_DIR}/src/scripting_engine.c
${CMAKE_SOURCE_DIR}/src/trace/trace.c
${CMAKE_SOURCE_DIR}/src/trace/trace_rdb.c
${CMAKE_SOURCE_DIR}/src/trace/trace_aof.c
${CMAKE_SOURCE_DIR}/src/trace/trace_commands.c
${CMAKE_SOURCE_DIR}/src/trace/trace_db.c
${CMAKE_SOURCE_DIR}/src/trace/trace_cluster.c
${CMAKE_SOURCE_DIR}/src/trace/trace_server.c
${CMAKE_SOURCE_DIR}/src/commands.c
${CMAKE_SOURCE_DIR}/src/strl.c
${CMAKE_SOURCE_DIR}/src/connection.c
${CMAKE_SOURCE_DIR}/src/unix.c
${CMAKE_SOURCE_DIR}/src/server.c
${CMAKE_SOURCE_DIR}/src/logreqres.c
${CMAKE_SOURCE_DIR}/src/entry.c
${CMAKE_SOURCE_DIR}/src/vset.c
${CMAKE_SOURCE_DIR}/src/fifo.c
${CMAKE_SOURCE_DIR}/src/mutexqueue.c)
# kv-cli
set(KV_CLI_SRCS
${CMAKE_SOURCE_DIR}/src/anet.c
${CMAKE_SOURCE_DIR}/src/adlist.c
${CMAKE_SOURCE_DIR}/src/dict.c
${CMAKE_SOURCE_DIR}/src/sds.c
${CMAKE_SOURCE_DIR}/src/sha256.c
${CMAKE_SOURCE_DIR}/src/util.c
${CMAKE_SOURCE_DIR}/src/kv-cli.c
${CMAKE_SOURCE_DIR}/src/zmalloc.c
${CMAKE_SOURCE_DIR}/src/release.c
${CMAKE_SOURCE_DIR}/src/ae.c
${CMAKE_SOURCE_DIR}/src/serverassert.c
${CMAKE_SOURCE_DIR}/src/crcspeed.c
${CMAKE_SOURCE_DIR}/src/crccombine.c
${CMAKE_SOURCE_DIR}/src/crc64.c
${CMAKE_SOURCE_DIR}/src/siphash.c
${CMAKE_SOURCE_DIR}/src/crc16.c
${CMAKE_SOURCE_DIR}/src/monotonic.c
${CMAKE_SOURCE_DIR}/src/cli_common.c
${CMAKE_SOURCE_DIR}/src/mt19937-64.c
${CMAKE_SOURCE_DIR}/src/strl.c
${CMAKE_SOURCE_DIR}/src/cli_commands.c)
# kv-benchmark
set(KV_BENCHMARK_SRCS
${CMAKE_SOURCE_DIR}/src/ae.c
${CMAKE_SOURCE_DIR}/src/anet.c
${CMAKE_SOURCE_DIR}/src/sds.c
${CMAKE_SOURCE_DIR}/src/sha256.c
${CMAKE_SOURCE_DIR}/src/util.c
${CMAKE_SOURCE_DIR}/src/kv-benchmark.c
${CMAKE_SOURCE_DIR}/src/adlist.c
${CMAKE_SOURCE_DIR}/src/dict.c
${CMAKE_SOURCE_DIR}/src/zmalloc.c
${CMAKE_SOURCE_DIR}/src/serverassert.c
${CMAKE_SOURCE_DIR}/src/release.c
${CMAKE_SOURCE_DIR}/src/crcspeed.c
${CMAKE_SOURCE_DIR}/src/crccombine.c
${CMAKE_SOURCE_DIR}/src/crc64.c
${CMAKE_SOURCE_DIR}/src/siphash.c
${CMAKE_SOURCE_DIR}/src/crc16.c
${CMAKE_SOURCE_DIR}/src/monotonic.c
${CMAKE_SOURCE_DIR}/src/cli_common.c
${CMAKE_SOURCE_DIR}/src/mt19937-64.c
${CMAKE_SOURCE_DIR}/src/strl.c
${CMAKE_SOURCE_DIR}/src/fuzzer_client.c
${CMAKE_SOURCE_DIR}/src/fuzzer_command_generator.c)
# kv-rdma module
set(KV_RDMA_MODULE_SRCS ${CMAKE_SOURCE_DIR}/src/rdma.c)
# kv-tls module
set(KV_TLS_MODULE_SRCS ${CMAKE_SOURCE_DIR}/src/tls.c)
-115
View File
@@ -1,115 +0,0 @@
# Return the current host distro name. For example: ubuntu, debian, amzn etc
function (kv_get_distro_name DISTRO_NAME)
if (LINUX AND NOT APPLE)
execute_process(
COMMAND /bin/bash "-c" "cat /etc/os-release |grep ^ID=|cut -d = -f 2"
OUTPUT_VARIABLE _OUT_VAR
OUTPUT_STRIP_TRAILING_WHITESPACE)
# clean the output
string(REPLACE "\"" "" _OUT_VAR "${_OUT_VAR}")
string(REPLACE "." "" _OUT_VAR "${_OUT_VAR}")
set(${DISTRO_NAME}
"${_OUT_VAR}"
PARENT_SCOPE)
elseif (APPLE)
set(${DISTRO_NAME}
"darwin"
PARENT_SCOPE)
elseif (IS_FREEBSD)
set(${DISTRO_NAME}
"freebsd"
PARENT_SCOPE)
else ()
set(${DISTRO_NAME}
"unknown"
PARENT_SCOPE)
endif ()
endfunction ()
function (kv_parse_version OUT_MAJOR OUT_MINOR OUT_PATCH)
# Read and parse package version from version.h file
file(STRINGS ${CMAKE_SOURCE_DIR}/src/version.h VERSION_LINES)
foreach (LINE ${VERSION_LINES})
string(FIND "${LINE}" "#define KV_VERSION " VERSION_STR_POS)
if (VERSION_STR_POS GREATER -1)
string(REPLACE "#define KV_VERSION " "" LINE "${LINE}")
string(REPLACE "\"" "" LINE "${LINE}")
# Change "." to ";" to make it a list
string(REPLACE "." ";" LINE "${LINE}")
list(GET LINE 0 _MAJOR)
list(GET LINE 1 _MINOR)
list(GET LINE 2 _PATCH)
message(STATUS "KV version: ${_MAJOR}.${_MINOR}.${_PATCH}")
# Set the output variables
set(${OUT_MAJOR}
${_MAJOR}
PARENT_SCOPE)
set(${OUT_MINOR}
${_MINOR}
PARENT_SCOPE)
set(${OUT_PATCH}
${_PATCH}
PARENT_SCOPE)
endif ()
endforeach ()
endfunction ()
# Given input argument `OPTION_VALUE`, check that the `OPTION_VALUE` is from the allowed values (one of:
# module/yes/no/1/0/true/false)
#
# Return value:
#
# If ARG is valid, return its number where:
#
# ~~~
# - `no` | `0` | `off` => return `0`
# - `yes` | `1` | `on` => return `1`
# - `module` => return `2`
# ~~~
function (kv_parse_build_option OPTION_VALUE OUT_ARG_ENUM)
list(APPEND VALID_OPTIONS "yes")
list(APPEND VALID_OPTIONS "1")
list(APPEND VALID_OPTIONS "on")
list(APPEND VALID_OPTIONS "no")
list(APPEND VALID_OPTIONS "0")
list(APPEND VALID_OPTIONS "off")
list(APPEND VALID_OPTIONS "module")
string(TOLOWER "${OPTION_VALUE}" OPTION_VALUE)
list(FIND VALID_OPTIONS "${ARG}" OPT_INDEX)
if (VERSION_STR_POS GREATER -1)
message(FATAL_ERROR "Invalid value passed ''${OPTION_VALUE}'")
endif ()
if ("${OPTION_VALUE}" STREQUAL "yes"
OR "${OPTION_VALUE}" STREQUAL "1"
OR "${OPTION_VALUE}" STREQUAL "on")
set(${OUT_ARG_ENUM}
1
PARENT_SCOPE)
elseif (
"${OPTION_VALUE}" STREQUAL "no"
OR "${OPTION_VALUE}" STREQUAL "0"
OR "${OPTION_VALUE}" STREQUAL "off")
set(${OUT_ARG_ENUM}
0
PARENT_SCOPE)
else ()
set(${OUT_ARG_ENUM}
2
PARENT_SCOPE)
endif ()
endfunction ()
function (kv_pkg_config PKGNAME OUT_VARIABLE)
if (NOT FOUND_PKGCONFIG)
# Locate pkg-config once
find_package(PkgConfig REQUIRED)
set(FOUND_PKGCONFIG 1)
endif ()
pkg_check_modules(__PREFIX REQUIRED ${PKGNAME})
message(STATUS "Found library for '${PKGNAME}': ${__PREFIX_LIBRARIES}")
set(${OUT_VARIABLE}
"${__PREFIX_LIBRARIES}"
PARENT_SCOPE)
endfunction ()
-38
View File
@@ -1,38 +0,0 @@
if (USE_JEMALLOC)
add_subdirectory(jemalloc)
endif ()
add_subdirectory(lua)
# Set libkv options. We need to disable the defaults set in the OPTION(..) we do this by setting them in the CACHE
set(BUILD_SHARED_LIBS
OFF
CACHE BOOL "Build shared libraries")
set(DISABLE_TESTS
ON
CACHE BOOL "If tests should be compiled or not")
if (USE_TLS) # Module or no module
message(STATUS "Building kv_tls")
set(ENABLE_TLS
ON
CACHE BOOL "If TLS support should be compiled or not")
endif ()
if (USE_RDMA) # Module or no module
message(STATUS "Building kv_rdma")
set(ENABLE_RDMA
ON
CACHE BOOL "If RDMA support should be compiled or not")
endif ()
# Let libkv use sds and dict provided by kv.
set(DICT_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/src)
set(SDS_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/src)
add_subdirectory(libkv)
add_subdirectory(linenoise)
add_subdirectory(fpconv)
add_subdirectory(hdr_histogram)
# Clear any cached variables passed to libkv from the cache
unset(BUILD_SHARED_LIBS CACHE)
unset(DISABLE_TESTS CACHE)
unset(ENABLE_TLS CACHE)
unset(ENABLE_RDMA CACHE)
+8 -20
View File
@@ -36,31 +36,25 @@ ifneq ($(shell sh -c '[ -f .make-ldflags ] && cat .make-ldflags || echo none'),
endif
distclean:
-(cd libkv && $(MAKE) clean) > /dev/null || true
-(cd hiredis && $(MAKE) clean) > /dev/null || true
-(cd linenoise && $(MAKE) clean) > /dev/null || true
-(cd lua && $(MAKE) clean) > /dev/null || true
-(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true
-(cd hdr_histogram && $(MAKE) clean) > /dev/null || true
-(cd fpconv && $(MAKE) clean) > /dev/null || true
-(cd fast_float_c_interface && $(MAKE) clean) > /dev/null || true
-(rm -f .make-*)
.PHONY: distclean
LIBKV_MAKE_FLAGS = SDS_INCLUDE_DIR=../../src/ DICT_INCLUDE_DIR=../../src/
ifneq (,$(filter $(BUILD_TLS),yes module))
LIBKV_MAKE_FLAGS += USE_TLS=1
HIREDIS_MAKE_FLAGS = USE_SSL=1
endif
ifneq (,$(filter $(BUILD_RDMA),yes module))
LIBKV_MAKE_FLAGS += USE_RDMA=1
endif
libkv: .make-prerequisites
hiredis: .make-prerequisites
@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
cd libkv && $(MAKE) static $(LIBKV_MAKE_FLAGS)
cd hiredis && $(MAKE) static $(HIREDIS_MAKE_FLAGS)
.PHONY: libkv
.PHONY: hiredis
linenoise: .make-prerequisites
@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
@@ -76,7 +70,7 @@ hdr_histogram: .make-prerequisites
fpconv: .make-prerequisites
@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
cd fpconv && $(MAKE) CFLAGS="-fPIC $(CFLAGS)"
cd fpconv && $(MAKE)
.PHONY: fpconv
@@ -85,12 +79,12 @@ ifeq ($(uname_S),SunOS)
LUA_CFLAGS= -D__C99FEATURES__=1
endif
LUA_CFLAGS+= -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DLUA_USE_MKSTEMP $(CFLAGS) -fPIC
LUA_CFLAGS+= -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DLUA_USE_MKSTEMP $(CFLAGS)
LUA_LDFLAGS+= $(LDFLAGS)
ifeq ($(LUA_DEBUG),yes)
LUA_CFLAGS+= -O0 -g -DLUA_USE_APICHECK
else
LUA_CFLAGS+= -O2
LUA_CFLAGS+= -O2
endif
ifeq ($(LUA_COVERAGE),yes)
LUA_CFLAGS += -fprofile-arcs -ftest-coverage
@@ -122,9 +116,3 @@ jemalloc: .make-prerequisites
cd jemalloc && $(MAKE) lib/libjemalloc.a
.PHONY: jemalloc
fast_float_c_interface: .make-prerequisites
@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
cd fast_float_c_interface && $(MAKE)
.PHONY: fast_float_c_interface
+16 -33
View File
@@ -1,12 +1,11 @@
This directory contains all KV dependencies, except for the libc that
This directory contains all Valkey dependencies, except for the libc that
should be provided by the operating system.
* **Jemalloc** is our memory allocator, used as replacement for libc malloc on Linux by default. It has good performances and excellent fragmentation behavior. This component is upgraded from time to time.
* **libkv** is the official C client library for KV. It is used by kv-cli, kv-benchmark and KV Sentinel. It is managed in a separate project and updated as needed.
* **linenoise** is a readline replacement. It is developed by the same authors of KV but is managed as a separated project and updated as needed.
* **hiredis** is the official C client library for Redis. It is used by redis-cli, redis-benchmark and Redis Sentinel. It is part of the Redis official ecosystem but is developed externally from the Redis repository, so we just upgrade it as needed.
* **linenoise** is a readline replacement. It is developed by the same authors of Valkey but is managed as a separated project and updated as needed.
* **lua** is Lua 5.1 with minor changes for security and additional libraries.
* **hdr_histogram** Used for per-command latency tracking histograms.
* **fast_float** is a replacement for strtod to convert strings to floats efficiently.
How to upgrade the above dependencies
===
@@ -14,10 +13,10 @@ How to upgrade the above dependencies
Jemalloc
---
Jemalloc is modified with changes that allow us to implement the KV
active defragmentation logic. However this feature of KV is not mandatory
and KV is able to understand if the Jemalloc version it is compiled
against supports such KV-specific modifications. So in theory, if you
Jemalloc is modified with changes that allow us to implement the Valkey
active defragmentation logic. However this feature of Valkey is not mandatory
and Valkey is able to understand if the Jemalloc version it is compiled
against supports such Valkey-specific modifications. So in theory, if you
are not interested in the active defragmentation, you can replace Jemalloc
just following these steps:
@@ -29,7 +28,7 @@ just following these steps:
Jemalloc configuration script is broken and will not work nested in another
git repository.
However note that we change Jemalloc settings via the `configure` script of Jemalloc using the `--with-lg-quantum` option, setting it to the value of 3 instead of 4. This provides us with more size classes that better suit the KV data structures, in order to gain memory efficiency.
However note that we change Jemalloc settings via the `configure` script of Jemalloc using the `--with-lg-quantum` option, setting it to the value of 3 instead of 4. This provides us with more size classes that better suit the Valkey data structures, in order to gain memory efficiency.
If you want to upgrade Jemalloc while also providing support for
active defragmentation, in addition to the above steps you need to perform
@@ -39,7 +38,7 @@ the following additional steps:
to add `#define JEMALLOC_FRAG_HINT`.
6. Implement the function `je_get_defrag_hint()` inside `src/jemalloc.c`. You
can see how it is implemented in the current Jemalloc source tree shipped
with KV, and rewrite it according to the new Jemalloc internals, if they
with Valkey, and rewrite it according to the new Jemalloc internals, if they
changed, otherwise you could just copy the old implementation if you are
upgrading just to a similar version of Jemalloc.
@@ -59,21 +58,20 @@ cd deps/jemalloc
4. Update jemalloc's version in `deps/Makefile`: search for "`--with-version=<old-version-tag>-0-g0`" and update it accordingly.
5. Commit the changes (VERSION,configure,Makefile).
Libkv
Hiredis
---
Libkv is used by Sentinel, `kv-cli` and `kv-benchmark`.
The library is built without its own version of the sds and dict type and uses the KV provided variant instead.
Hiredis is used by Sentinel, `valkey-cli` and `valkey-benchmark`. Like Valkey, uses the SDS string library, but not necessarily the same version. In order to avoid conflicts, this version has all SDS identifiers prefixed by `hi`.
1. `git subtree pull --prefix deps/libkv https://github.com/hanzoai/kv.git <version-tag> --squash`<br>
1. `git subtree pull --prefix deps/hiredis https://github.com/redis/hiredis.git <version-tag> --squash`<br>
This should hopefully merge the local changes into the new version.
2. Commit the changes.
2. Conflicts will arise (due to our changes) you'll need to resolve them and commit.
Linenoise
---
Linenoise is rarely upgraded as needed. The upgrade process is trivial since
KV uses a non modified version of linenoise, so to upgrade just do the
Valkey uses a non modified version of linenoise, so to upgrade just do the
following:
1. Remove the linenoise directory.
@@ -83,11 +81,11 @@ Lua
---
We use Lua 5.1 and no upgrade is planned currently, since we don't want to break
Lua scripts for new Lua features: in the context of KV Lua scripts the
Lua scripts for new Lua features: in the context of Valkey Lua scripts the
capabilities of 5.1 are usually more than enough, the release is rock solid,
and we definitely don't want to break old scripts.
So upgrading of Lua is up to the KV project maintainers and should be a
So upgrading of Lua is up to the Valkey project maintainers and should be a
manual procedure performed by taking a diff between the different versions.
Currently we have at least the following differences between official Lua 5.1
@@ -96,7 +94,6 @@ and our version:
1. Makefile is modified to allow a different compiler than GCC.
2. We have the implementation source code, and directly link to the following external libraries: `lua_cjson.o`, `lua_struct.o`, `lua_cmsgpack.o` and `lua_bit.o`.
3. There is a security fix in `ldo.c`, line 498: The check for `LUA_SIGNATURE[0]` is removed in order to avoid direct bytecode execution.
4. In `lstring.c`, the luaS_newlstr function's hash calculation has been upgraded from a simple hash function to MurmurHash3, implemented within the same file, to enhance performance, particularly for operations involving large strings.
Hdr_Histogram
---
@@ -107,17 +104,3 @@ We use a customized version based on master branch commit e4448cf6d1cd08fff51981
2. Copy updated files from newer version onto files in /hdr_histogram.
3. Apply the changes from 1 above to the updated files.
fast_float
---
The fast_float library provides fast header-only implementations for the C++ from_chars functions for `float` and `double` types as well as integer types. These functions convert ASCII strings representing decimal values (e.g., `1.3e10`) into binary types. The functions are much faster than comparable number-parsing functions from existing C++ standard libraries.
Specifically, `fast_float` provides the following function to parse floating-point numbers with a C++17-like syntax (the library itself only requires C++11):
template <typename T, typename UC = char, typename = FASTFLOAT_ENABLE_IF(is_supported_float_type<T>())>
from_chars_result_t<UC> from_chars(UC const *first, UC const *last, T &value, chars_format fmt = chars_format::general);
To upgrade the library,
1. Check out https://github.com/fastfloat/fast_float/tree/main
2. cd fast_float
3. Invoke "python3 ./script/amalgamate.py --output fast_float.h"
4. Copy fast_float.h file to "deps/fast_float/".
-3912
View File
File diff suppressed because it is too large Load Diff
-37
View File
@@ -1,37 +0,0 @@
CCCOLOR:="\033[34m"
SRCCOLOR:="\033[33m"
ENDCOLOR:="\033[0m"
CXX?=c++
# we need = instead of := so that $@ in QUIET_CXX gets evaluated in the rule and is assigned appropriate value.
TEMP:=$(CXX)
QUIET_CXX=@printf ' %b %b\n' $(CCCOLOR)C++$(ENDCOLOR) $(SRCCOLOR)$@$(ENDCOLOR) 1>&2;
CXX=$(QUIET_CXX)$(TEMP)
WARN=-Wall -W -Wno-missing-field-initializers
STD=-pedantic -std=c++11
OPT?=-O3
CLANG := $(findstring clang,$(shell sh -c '$(CC) --version | head -1'))
ifeq ($(OPT),-O3)
ifeq (clang,$(CLANG))
OPT+=-flto
else
OPT+=-flto=auto -ffat-lto-objects
endif
endif
# 1) Today src/Makefile passes -m32 flag for explicit 32-bit build on 64-bit machine, via CFLAGS. For 32-bit build on
# 32-bit machine and 64-bit on 64-bit machine, CFLAGS are empty. No other flags are set that can conflict with C++,
# therefore let's use CFLAGS without changes for now.
# 2) FASTFLOAT_ALLOWS_LEADING_PLUS allows +inf to be parsed as inf, instead of error.
CXXFLAGS=$(STD) $(OPT) $(WARN) -static -fPIC -fno-exceptions $(CFLAGS) -D FASTFLOAT_ALLOWS_LEADING_PLUS
.PHONY: all clean
all: fast_float_strtod.o
clean:
rm -f *.o || true;
-24
View File
@@ -1,24 +0,0 @@
/*
* Copyright (c) Valkey Contributors
* All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*/
#include "../fast_float/fast_float.h"
#include <cerrno>
extern "C"
{
double fast_float_strtod(const char *str, const char** endptr)
{
double temp = 0;
auto answer = fast_float::from_chars(str, str + strlen(str), temp);
if (answer.ec != std::errc()) {
errno = (answer.ec == std::errc::result_out_of_range) ? ERANGE : EINVAL;
}
if (endptr) {
*endptr = answer.ptr;
}
return temp;
}
}
-6
View File
@@ -1,6 +0,0 @@
project(fpconv)
set(SRCS "${CMAKE_CURRENT_LIST_DIR}/fpconv_dtoa.c" "${CMAKE_CURRENT_LIST_DIR}/fpconv_dtoa.h")
add_library(fpconv STATIC ${SRCS})
# Add -fPIC
set_target_properties(fpconv PROPERTIES POSITION_INDEPENDENT_CODE ON)
-7
View File
@@ -1,7 +0,0 @@
project(hdr_histogram)
set(SRCS "${CMAKE_CURRENT_LIST_DIR}/hdr_histogram.c" "${CMAKE_CURRENT_LIST_DIR}/hdr_histogram.h"
"${CMAKE_CURRENT_LIST_DIR}/hdr_atomic.h" "${CMAKE_CURRENT_LIST_DIR}/hdr_redis_malloc.h")
add_library(hdr_histogram STATIC ${SRCS})
target_compile_definitions(hdr_histogram PRIVATE HDR_MALLOC_INCLUDE=\"hdr_redis_malloc.h\")
+6 -6
View File
@@ -1,13 +1,13 @@
#ifndef HDR_MALLOC_H__
#define HDR_MALLOC_H__
void *kv_malloc(size_t size);
void *zmalloc(size_t size);
void *zcalloc_num(size_t num, size_t size);
void *kv_realloc(void *ptr, size_t size);
void kv_free(void *ptr);
void *zrealloc(void *ptr, size_t size);
void zfree(void *ptr);
#define hdr_malloc kv_malloc
#define hdr_malloc zmalloc
#define hdr_calloc zcalloc_num
#define hdr_realloc kv_realloc
#define hdr_free kv_free
#define hdr_realloc zrealloc
#define hdr_free zfree
#endif
@@ -18,6 +18,7 @@ categories:
- title: 'Breaking Changes'
labels:
- 'breakingchange'
- title: '🧪 Experimental Features'
labels:
- 'experimental'
+177
View File
@@ -0,0 +1,177 @@
name: Build and test
on: [push, pull_request]
jobs:
ubuntu:
name: Ubuntu
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Install dependencies
run: |
curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list
sudo apt-get update
sudo apt-get install -y redis-server valgrind libevent-dev
- name: Build using cmake
env:
EXTRA_CMAKE_OPTS: -DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON -DENABLE_SSL_TESTS:BOOL=ON -DENABLE_ASYNC_TESTS:BOOL=ON
CFLAGS: -Werror
CXXFLAGS: -Werror
run: mkdir build && cd build && cmake .. && make
- name: Build using makefile
run: USE_SSL=1 TEST_ASYNC=1 make
- name: Run tests
env:
SKIPS_AS_FAILS: 1
TEST_SSL: 1
run: $GITHUB_WORKSPACE/test.sh
# - name: Run tests under valgrind
# env:
# SKIPS_AS_FAILS: 1
# TEST_PREFIX: valgrind --error-exitcode=99 --track-origins=yes --leak-check=full
# run: $GITHUB_WORKSPACE/test.sh
centos7:
name: CentOS 7
runs-on: ubuntu-latest
container: centos:7
steps:
- uses: actions/checkout@v3
- name: Install dependencies
run: |
yum -y install http://rpms.remirepo.net/enterprise/remi-release-7.rpm
yum -y --enablerepo=remi install redis
yum -y install gcc gcc-c++ make openssl openssl-devel cmake3 valgrind libevent-devel
- name: Build using cmake
env:
EXTRA_CMAKE_OPTS: -DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON -DENABLE_SSL_TESTS:BOOL=ON -DENABLE_ASYNC_TESTS:BOOL=ON
CFLAGS: -Werror
CXXFLAGS: -Werror
run: mkdir build && cd build && cmake3 .. && make
- name: Build using Makefile
run: USE_SSL=1 TEST_ASYNC=1 make
- name: Run tests
env:
SKIPS_AS_FAILS: 1
TEST_SSL: 1
run: $GITHUB_WORKSPACE/test.sh
- name: Run tests under valgrind
env:
SKIPS_AS_FAILS: 1
TEST_SSL: 1
TEST_PREFIX: valgrind --error-exitcode=99 --track-origins=yes --leak-check=full
run: $GITHUB_WORKSPACE/test.sh
centos8:
name: RockyLinux 8
runs-on: ubuntu-latest
container: rockylinux:8
steps:
- uses: actions/checkout@v3
- name: Install dependencies
run: |
dnf -y upgrade --refresh
dnf -y install https://rpms.remirepo.net/enterprise/remi-release-8.rpm
dnf -y module install redis:remi-6.0
dnf -y group install "Development Tools"
dnf -y install openssl-devel cmake valgrind libevent-devel
- name: Build using cmake
env:
EXTRA_CMAKE_OPTS: -DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON -DENABLE_SSL_TESTS:BOOL=ON -DENABLE_ASYNC_TESTS:BOOL=ON
CFLAGS: -Werror
CXXFLAGS: -Werror
run: mkdir build && cd build && cmake .. && make
- name: Build using Makefile
run: USE_SSL=1 TEST_ASYNC=1 make
- name: Run tests
env:
SKIPS_AS_FAILS: 1
TEST_SSL: 1
run: $GITHUB_WORKSPACE/test.sh
- name: Run tests under valgrind
env:
SKIPS_AS_FAILS: 1
TEST_SSL: 1
TEST_PREFIX: valgrind --error-exitcode=99 --track-origins=yes --leak-check=full
run: $GITHUB_WORKSPACE/test.sh
freebsd:
runs-on: macos-12
name: FreeBSD
steps:
- uses: actions/checkout@v3
- name: Build in FreeBSD
uses: vmactions/freebsd-vm@v0
with:
prepare: pkg install -y gmake cmake
run: |
mkdir build && cd build && cmake .. && make && cd ..
gmake
macos:
name: macOS
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- name: Install dependencies
run: |
brew install openssl redis@7.0
brew link redis@7.0 --force
- name: Build hiredis
run: USE_SSL=1 make
- name: Run tests
env:
TEST_SSL: 1
run: $GITHUB_WORKSPACE/test.sh
windows:
name: Windows
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- name: Install dependencies
run: |
choco install -y ninja memurai-developer
- uses: ilammy/msvc-dev-cmd@v1
- name: Build hiredis
run: |
mkdir build && cd build
cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DENABLE_EXAMPLES=ON
ninja -v
- name: Run tests
run: |
./build/hiredis-test.exe
- name: Install Cygwin Action
uses: cygwin/cygwin-install-action@v2
with:
packages: make git gcc-core
- name: Build in cygwin
env:
HIREDIS_PATH: ${{ github.workspace }}
run: |
make clean && make
@@ -4,14 +4,14 @@ on:
push:
# branches to consider in the event; optional, defaults to all
branches:
- main
- master
jobs:
update_release_draft:
runs-on: ubuntu-latest
steps:
# Drafts your next Release notes as Pull Requests are merged into "master"
- uses: release-drafter/release-drafter@b1476f6e6eb133afa41ed8589daba6dc69b4d3f5 # v6.1.0
- uses: release-drafter/release-drafter@v5
with:
# (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml
config-name: release-drafter-config.yml
+100
View File
@@ -0,0 +1,100 @@
name: C/C++ CI
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
full-build:
name: Build all, plus default examples, run tests against redis
runs-on: ubuntu-latest
env:
# the docker image used by the test.sh
REDIS_DOCKER: redis:alpine
steps:
- name: Install prerequisites
run: sudo apt-get update && sudo apt-get install -y libev-dev libevent-dev libglib2.0-dev libssl-dev valgrind
- uses: actions/checkout@v3
- name: Run make
run: make all examples
- name: Run unittests
run: make check
- name: Run tests with valgrind
env:
TEST_PREFIX: valgrind --error-exitcode=100
SKIPS_ARG: --skip-throughput
run: make check
build-32-bit:
name: Build and test minimal 32 bit linux
runs-on: ubuntu-latest
steps:
- name: Install prerequisites
run: sudo apt-get update && sudo apt-get install gcc-multilib
- uses: actions/checkout@v3
- name: Run make
run: make all
env:
PLATFORM_FLAGS: -m32
- name: Run unittests
env:
REDIS_DOCKER: redis:alpine
run: make check
build-arm:
name: Cross-compile and test arm linux with Qemu
runs-on: ubuntu-latest
strategy:
matrix:
include:
- name: arm
toolset: arm-linux-gnueabi
emulator: qemu-arm
- name: aarch64
toolset: aarch64-linux-gnu
emulator: qemu-aarch64
steps:
- name: Install qemu
if: matrix.emulator
run: sudo apt-get update && sudo apt-get install -y qemu-user
- name: Install platform toolset
if: matrix.toolset
run: sudo apt-get install -y gcc-${{matrix.toolset}}
- uses: actions/checkout@v3
- name: Run make
run: make all
env:
CC: ${{matrix.toolset}}-gcc
AR: ${{matrix.toolset}}-ar
- name: Run unittests
env:
REDIS_DOCKER: redis:alpine
TEST_PREFIX: ${{matrix.emulator}} -L /usr/${{matrix.toolset}}/
run: make check
build-windows:
name: Build and test on windows 64 bit Intel
runs-on: windows-latest
steps:
- uses: microsoft/setup-msbuild@v1.0.2
- uses: actions/checkout@v3
- name: Run CMake (shared lib)
run: cmake -Wno-dev CMakeLists.txt
- name: Build hiredis (shared lib)
run: MSBuild hiredis.vcxproj /p:Configuration=Debug
- name: Run CMake (static lib)
run: cmake -Wno-dev CMakeLists.txt -DBUILD_SHARED_LIBS=OFF
- name: Build hiredis (static lib)
run: MSBuild hiredis.vcxproj /p:Configuration=Debug
- name: Build hiredis-test
run: MSBuild hiredis-test.vcxproj /p:Configuration=Debug
# use memurai, redis compatible server, since it is easy to install. Can't
# install official redis containers on the windows runner
- name: Install Memurai redis server
run: choco install -y memurai-developer.install
- name: Run tests
run: Debug\hiredis-test.exe
+9
View File
@@ -0,0 +1,9 @@
/hiredis-test
/examples/hiredis-example*
/*.o
/*.so
/*.dylib
/*.a
/*.pc
*.dSYM
tags
+125
View File
@@ -0,0 +1,125 @@
language: c
compiler:
- gcc
- clang
os:
- linux
- osx
dist: bionic
branches:
only:
- staging
- trying
- master
- /^release\/.*$/
install:
- if [ "$TRAVIS_COMPILER" != "mingw" ]; then
wget https://github.com/redis/redis/archive/6.0.6.tar.gz;
tar -xzvf 6.0.6.tar.gz;
pushd redis-6.0.6 && BUILD_TLS=yes make && export PATH=$PWD/src:$PATH && popd;
fi;
before_script:
- if [ "$TRAVIS_OS_NAME" == "osx" ]; then
curl -O https://distfiles.macports.org/MacPorts/MacPorts-2.6.2-10.13-HighSierra.pkg;
sudo installer -pkg MacPorts-2.6.2-10.13-HighSierra.pkg -target /;
export PATH=$PATH:/opt/local/bin && sudo port -v selfupdate;
sudo port -N install openssl redis;
fi;
addons:
apt:
packages:
- libc6-dbg
- libc6-dev
- libc6:i386
- libc6-dev-i386
- libc6-dbg:i386
- gcc-multilib
- g++-multilib
- libssl-dev
- libssl-dev:i386
- valgrind
env:
- BITS="32"
- BITS="64"
script:
- EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON -DENABLE_SSL_TESTS:BOOL=ON";
if [ "$TRAVIS_OS_NAME" == "osx" ]; then
if [ "$BITS" == "32" ]; then
CFLAGS="-m32 -Werror";
CXXFLAGS="-m32 -Werror";
LDFLAGS="-m32";
EXTRA_CMAKE_OPTS=;
else
CFLAGS="-Werror";
CXXFLAGS="-Werror";
fi;
else
TEST_PREFIX="valgrind --track-origins=yes --leak-check=full";
if [ "$BITS" == "32" ]; then
CFLAGS="-m32 -Werror";
CXXFLAGS="-m32 -Werror";
LDFLAGS="-m32";
EXTRA_CMAKE_OPTS=;
else
CFLAGS="-Werror";
CXXFLAGS="-Werror";
fi;
fi;
export CFLAGS CXXFLAGS LDFLAGS TEST_PREFIX EXTRA_CMAKE_OPTS
- make && make clean;
if [ "$TRAVIS_OS_NAME" == "osx" ]; then
if [ "$BITS" == "64" ]; then
OPENSSL_PREFIX="$(ls -d /usr/local/Cellar/openssl@1.1/*)" USE_SSL=1 make;
fi;
else
USE_SSL=1 make;
fi;
- mkdir build/ && cd build/
- cmake .. ${EXTRA_CMAKE_OPTS}
- make VERBOSE=1
- if [ "$BITS" == "64" ]; then
TEST_SSL=1 SKIPS_AS_FAILS=1 ctest -V;
else
SKIPS_AS_FAILS=1 ctest -V;
fi;
jobs:
include:
# Windows MinGW cross compile on Linux
- os: linux
dist: xenial
compiler: mingw
addons:
apt:
packages:
- ninja-build
- gcc-mingw-w64-x86-64
- g++-mingw-w64-x86-64
script:
- mkdir build && cd build
- CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_BUILD_WITH_INSTALL_RPATH=on
- ninja -v
# Windows MSVC 2017
- os: windows
compiler: msvc
env:
- MATRIX_EVAL="CC=cl.exe && CXX=cl.exe"
before_install:
- eval "${MATRIX_EVAL}"
install:
- choco install ninja
- choco install -y memurai-developer
script:
- mkdir build && cd build
- cmd.exe //C 'C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat' amd64 '&&'
cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DENABLE_EXAMPLES=ON '&&' ninja -v
- ./hiredis-test.exe
+580
View File
@@ -0,0 +1,580 @@
## [1.2.0](https://github.com/redis/hiredis/tree/v1.2.0) - (2023-06-04)
Announcing Hiredis v1.2.0 with with new adapters, and a great many bug fixes.
## 🚀 New Features
- Add sdevent adapter @Oipo (#1144)
- Allow specifying the keepalive interval @michael-grunder (#1168)
- Add RedisModule adapter @tezc (#1182)
- Helper for setting TCP_USER_TIMEOUT socket option @zuiderkwast (#1188)
## 🐛 Bug Fixes
- Fix a typo in b6a052f. @yossigo (#1190)
- Fix wincrypt symbols conflict @hudayou (#1151)
- Don't attempt to set a timeout if we are in an error state. @michael-grunder (#1180)
- Accept -nan per the RESP3 spec recommendation. @michael-grunder (#1178)
- Fix colliding option values @zuiderkwast (#1172)
- Ensure functionality without `_MSC_VER` definition @windyakin (#1194)
## 🧰 Maintenance
- Add a test for the TCP_USER_TIMEOUT option. @michael-grunder (#1192)
- Add -Werror as a default. @yossigo (#1193)
- CI: Update homebrew Redis version. @yossigo (#1191)
- Fix typo in makefile. @michael-grunder (#1179)
- Write a version file for the CMake package @Neverlord (#1165)
- CMakeLists.txt: respect BUILD_SHARED_LIBS @ffontaine (#1147)
- Cmake static or shared @autoantwort (#1160)
- fix typo @tillkruss (#1153)
- Add a test ensuring we don't clobber connection error. @michael-grunder (#1181)
- Search for openssl on macOS @michael-grunder (#1169)
## Contributors
We'd like to thank all the contributors who worked on this release!
<a href="https://github.com/neverlord"><img src="https://github.com/neverlord.png" width="32" height="32"></a>
<a href="https://github.com/Oipo"><img src="https://github.com/Oipo.png" width="32" height="32"></a>
<a href="https://github.com/autoantwort"><img src="https://github.com/autoantwort.png" width="32" height="32"></a>
<a href="https://github.com/ffontaine"><img src="https://github.com/ffontaine.png" width="32" height="32"></a>
<a href="https://github.com/hudayou"><img src="https://github.com/hudayou.png" width="32" height="32"></a>
<a href="https://github.com/michael-grunder"><img src="https://github.com/michael-grunder.png" width="32" height="32"></a>
<a href="https://github.com/postgraph"><img src="https://github.com/postgraph.png" width="32" height="32"></a>
<a href="https://github.com/tezc"><img src="https://github.com/tezc.png" width="32" height="32"></a>
<a href="https://github.com/tillkruss"><img src="https://github.com/tillkruss.png" width="32" height="32"></a>
<a href="https://github.com/vityafx"><img src="https://github.com/vityafx.png" width="32" height="32"></a>
<a href="https://github.com/windyakin"><img src="https://github.com/windyakin.png" width="32" height="32"></a>
<a href="https://github.com/yossigo"><img src="https://github.com/yossigo.png" width="32" height="32"></a>
<a href="https://github.com/zuiderkwast"><img src="https://github.com/zuiderkwast.png" width="32" height="32"></a>
## [1.1.0](https://github.com/redis/hiredis/tree/v1.1.0) - (2022-11-15)
Announcing Hiredis v1.1.0 GA with better SSL convenience, new async adapters and a great many bug fixes.
**NOTE**: Hiredis can now return `nan` in addition to `-inf` and `inf` when returning a `REDIS_REPLY_DOUBLE`.
## 🐛 Bug Fixes
- Add support for nan in RESP3 double [@filipecosta90](https://github.com/filipecosta90)
([\#1133](https://github.com/redis/hiredis/pull/1133))
## 🧰 Maintenance
- Add an example that calls redisCommandArgv [@michael-grunder](https://github.com/michael-grunder)
([\#1140](https://github.com/redis/hiredis/pull/1140))
- fix flag reference [@pata00](https://github.com/pata00) ([\#1136](https://github.com/redis/hiredis/pull/1136))
- Make freeing a NULL redisAsyncContext a no op. [@michael-grunder](https://github.com/michael-grunder)
([\#1135](https://github.com/redis/hiredis/pull/1135))
- CI updates ([@bjosv](https://github.com/redis/bjosv) ([\#1139](https://github.com/redis/hiredis/pull/1139))
## Contributors
We'd like to thank all the contributors who worked on this release!
<a href="https://github.com/bjsov"><img src="https://github.com/bjosv.png" width="32" height="32"></a>
<a href="https://github.com/filipecosta90"><img src="https://github.com/filipecosta90.png" width="32" height="32"></a>
<a href="https://github.com/michael-grunder"><img src="https://github.com/michael-grunder.png" width="32" height="32"></a>
<a href="https://github.com/pata00"><img src="https://github.com/pata00.png" width="32" height="32"></a>
## [1.1.0-rc1](https://github.com/redis/hiredis/tree/v1.1.0-rc1) - (2022-11-06)
Announcing Hiredis v1.1.0-rc1, with better SSL convenience, new async adapters, and a great many bug fixes.
## 🚀 New Features
- Add possibility to prefer IPv6, IPv4 or unspecified [@zuiderkwast](https://github.com/zuiderkwast)
([\#1096](https://github.com/redis/hiredis/pull/1096))
- Add adapters/libhv [@ithewei](https://github.com/ithewei) ([\#904](https://github.com/redis/hiredis/pull/904))
- Add timeout support to libhv adapter. [@michael-grunder](https://github.com/michael-grunder) ([\#1109](https://github.com/redis/hiredis/pull/1109))
- set default SSL verification path [@adobeturchenko](https://github.com/adobeturchenko) ([\#928](https://github.com/redis/hiredis/pull/928))
- Introduce .close method for redisContextFuncs [@pizhenwei](https://github.com/pizhenwei) ([\#1094](https://github.com/redis/hiredis/pull/1094))
- Make it possible to set SSL verify mode [@stanhu](https://github.com/stanhu) ([\#1085](https://github.com/redis/hiredis/pull/1085))
- Polling adapter and example [@kristjanvalur](https://github.com/kristjanvalur) ([\#932](https://github.com/redis/hiredis/pull/932))
- Unsubscribe handling in async [@bjosv](https://github.com/bjosv) ([\#1047](https://github.com/redis/hiredis/pull/1047))
- Add timeout support for libuv adapter [@MichaelSuen-thePointer](https://github.com/@MichaelSuenthePointer) ([\#1016](https://github.com/redis/hiredis/pull/1016))
## 🐛 Bug Fixes
- Update for MinGW cross compile [@bit0fun](https://github.com/bit0fun) ([\#1127](https://github.com/redis/hiredis/pull/1127))
- fixed CPP build error with adapters/libhv.h [@mtdxc](https://github.com/mtdxc) ([\#1125](https://github.com/redis/hiredis/pull/1125))
- Fix protocol error
[@michael-grunder](https://github.com/michael-grunder),
[@mtuleika-appcast](https://github.com/mtuleika-appcast) ([\#1106](https://github.com/redis/hiredis/pull/1106))
- Use a windows specific keepalive function. [@michael-grunder](https://github.com/michael-grunder) ([\#1104](https://github.com/redis/hiredis/pull/1104))
- Fix CMake config path on Linux. [@xkszltl](https://github.com/xkszltl) ([\#989](https://github.com/redis/hiredis/pull/989))
- Fix potential fault at createDoubleObject [@afcidk](https://github.com/afcidk) ([\#964](https://github.com/redis/hiredis/pull/964))
- Fix some undefined behavior [@jengab](https://github.com/jengab) ([\#1091](https://github.com/redis/hiredis/pull/1091))
- Copy OOM errors to redisAsyncContext when finding subscribe callback [@bjosv](https://github.com/bjosv) ([\#1090](https://github.com/redis/hiredis/pull/1090))
- Maintain backward compatibility with our onConnect callback. [@michael-grunder](https://github.com/michael-grunder) ([\#1087](https://github.com/redis/hiredis/pull/1087))
- Fix PUSH handler tests for Redis >= 7.0.5 [@michael-grunder](https://github.com/michael-grunder) ([\#1121](https://github.com/redis/hiredis/pull/1121))
- fix heap-buffer-overflow [@zhangtaoXT5](https://github.com/zhangtaoXT5) ([\#957](https://github.com/redis/hiredis/pull/957))
- Fix heap-buffer-overflow issue in redisvFormatCommad [@bjosv](https://github.com/bjosv) ([\#1097](https://github.com/redis/hiredis/pull/1097))
- Polling adapter requires sockcompat.h [@michael-grunder](https://github.com/michael-grunder) ([\#1095](https://github.com/redis/hiredis/pull/1095))
- Illumos test fixes, error message difference for bad hostname test. [@devnexen](https://github.com/devnexen) ([\#901](https://github.com/redis/hiredis/pull/901))
- Remove semicolon after do-while in \_EL\_CLEANUP [@sundb](https://github.com/sundb) ([\#905](https://github.com/redis/hiredis/pull/905))
- Stability: Support calling redisAsyncCommand and redisAsyncDisconnect from the onConnected callback [@kristjanvalur](https://github.com/kristjanvalur)
([\#931](https://github.com/redis/hiredis/pull/931))
- Fix async connect on Windows [@kristjanvalur](https://github.com/kristjanvalur) ([\#1073](https://github.com/redis/hiredis/pull/1073))
- Fix tests so they work for Redis 7.0 [@michael-grunder](https://github.com/michael-grunder) ([\#1072](https://github.com/redis/hiredis/pull/1072))
- Fix warnings on Win64 [@orgads](https://github.com/orgads) ([\#1058](https://github.com/redis/hiredis/pull/1058))
- Handle push notifications before or after reply. [@yossigo](https://github.com/yossigo) ([\#1062](https://github.com/redis/hiredis/pull/1062))
- Update hiredis sds with improvements found in redis [@bjosv](https://github.com/bjosv) ([\#1045](https://github.com/redis/hiredis/pull/1045))
- Avoid incorrect call to the previous reply's callback [@bjosv](https://github.com/bjosv) ([\#1040](https://github.com/redis/hiredis/pull/1040))
- fix building on AIX and SunOS [\#1031](https://github.com/redis/hiredis/pull/1031) ([@scddev](https://github.com/scddev))
- Allow sending commands after sending an unsubscribe [@bjosv](https://github.com/bjosv) ([\#1036](https://github.com/redis/hiredis/pull/1036))
- Correction for command timeout during pubsub [@bjosv](https://github.com/bjosv) ([\#1038](https://github.com/redis/hiredis/pull/1038))
- Fix adapters/libevent.h compilation for 64-bit Windows [@pbtummillo](https://github.com/pbtummillo) ([\#937](https://github.com/redis/hiredis/pull/937))
- Fix integer overflow when format command larger than 4GB [@sundb](https://github.com/sundb) ([\#1030](https://github.com/redis/hiredis/pull/1030))
- Handle array response during subscribe in RESP3 [@bjosv](https://github.com/bjosv) ([\#1014](https://github.com/redis/hiredis/pull/1014))
- Support PING while subscribing (RESP2) [@bjosv](https://github.com/bjosv) ([\#1027](https://github.com/redis/hiredis/pull/1027))
## 🧰 Maintenance
- CI fixes in preparation of release [@michael-grunder](https://github.com/michael-grunder) ([\#1130](https://github.com/redis/hiredis/pull/1130))
- Add do while(0) (protection for macros [@afcidk](https://github.com/afcidk) [\#959](https://github.com/redis/hiredis/pull/959))
- Fixup of PR734: Coverage of hiredis.c [@bjosv](https://github.com/bjosv) ([\#1124](https://github.com/redis/hiredis/pull/1124))
- CMake corrections for building on Windows [@bjosv](https://github.com/bjosv) ([\#1122](https://github.com/redis/hiredis/pull/1122))
- Install on windows fixes [@bjosv](https://github.com/bjosv) ([\#1117](https://github.com/redis/hiredis/pull/1117))
- Add libhv example to our standard Makefile [@michael-grunder](https://github.com/michael-grunder) ([\#1108](https://github.com/redis/hiredis/pull/1108))
- Additional include directory given by pkg-config [@bjosv](https://github.com/bjosv) ([\#1118](https://github.com/redis/hiredis/pull/1118))
- Use __attribute__ when building with Clang on Windows [@bjosv](https://github.com/bjosv) ([\#1115](https://github.com/redis/hiredis/pull/1115))
- Minor refactor [@michael-grunder](https://github.com/michael-grunder) ([\#1110](https://github.com/redis/hiredis/pull/1110))
- Fix pkgconfig result for hiredis_ssl [@bjosv](https://github.com/bjosv) ([\#1107](https://github.com/redis/hiredis/pull/1107))
- Update documentation to explain redisConnectWithOptions. [@michael-grunder](https://github.com/michael-grunder) ([\#1099](https://github.com/redis/hiredis/pull/1099))
- uvadapter: reduce number of uv_poll_start calls [@noxiouz](https://github.com/noxiouz) ([\#1098](https://github.com/redis/hiredis/pull/1098))
- Regression test for off-by-one parsing error [@bugwz](https://github.com/bugwz) ([\#1092](https://github.com/redis/hiredis/pull/1092))
- CMake: remove dict.c form hiredis_sources [@Lipraxde](https://github.com/Lipraxde) ([\#1055](https://github.com/redis/hiredis/pull/1055))
- Do store command timeout in the context for redisSetTimeout [@catterer](https://github.com/catterer) ([\#593](https://github.com/redis/hiredis/pull/593), [\#1093](https://github.com/redis/hiredis/pull/1093))
- Add GitHub Actions CI workflow for hiredis: Arm, Arm64, 386, windows. [@kristjanvalur](https://github.com/kristjanvalur) ([\#943](https://github.com/redis/hiredis/pull/943))
- CI: bump macOS runner version [@SukkaW](https://github.com/SukkaW) ([\#1079](https://github.com/redis/hiredis/pull/1079))
- Support for generating release notes [@chayim](https://github.com/chayim) ([\#1083](https://github.com/redis/hiredis/pull/1083))
- Improve example for SSL initialization in README.md [@stanhu](https://github.com/stanhu) ([\#1084](https://github.com/redis/hiredis/pull/1084))
- Fix README typos [@bjosv](https://github.com/bjosv) ([\#1080](https://github.com/redis/hiredis/pull/1080))
- fix cmake version [@smmir-cent](https://github.com/@smmircent) ([\#1050](https://github.com/redis/hiredis/pull/1050))
- Use the same name for static and shared libraries [@orgads](https://github.com/orgads) ([\#1057](https://github.com/redis/hiredis/pull/1057))
- Embed debug information in windows static .lib file [@kristjanvalur](https://github.com/kristjanvalur) ([\#1054](https://github.com/redis/hiredis/pull/1054))
- Improved async documentation [@kristjanvalur](https://github.com/kristjanvalur) ([\#1074](https://github.com/redis/hiredis/pull/1074))
- Use official repository for redis package. [@yossigo](https://github.com/yossigo) ([\#1061](https://github.com/redis/hiredis/pull/1061))
- Whitelist hiredis repo path in cygwin [@michael-grunder](https://github.com/michael-grunder) ([\#1063](https://github.com/redis/hiredis/pull/1063))
- CentOS 8 is EOL, switch to RockyLinux [@michael-grunder](https://github.com/michael-grunder) ([\#1046](https://github.com/redis/hiredis/pull/1046))
- CMakeLists.txt: allow building without a C++ compiler [@ffontaine](https://github.com/ffontaine) ([\#872](https://github.com/redis/hiredis/pull/872))
- Makefile: move SSL options into a block and refine rules [@pizhenwei](https://github.com/pizhenwei) ([\#997](https://github.com/redis/hiredis/pull/997))
- Update CMakeLists.txt for more portability [@EricDeng1001](https://github.com/EricDeng1001) ([\#1005](https://github.com/redis/hiredis/pull/1005))
- FreeBSD build fixes + CI [@michael-grunder](https://github.com/michael-grunder) ([\#1026](https://github.com/redis/hiredis/pull/1026))
- Add asynchronous test for pubsub using RESP3 [@bjosv](https://github.com/bjosv) ([\#1012](https://github.com/redis/hiredis/pull/1012))
- Trigger CI failure when Valgrind issues are found [@bjosv](https://github.com/bjosv) ([\#1011](https://github.com/redis/hiredis/pull/1011))
- Move to using make directly in Cygwin [@michael-grunder](https://github.com/michael-grunder) ([\#1020](https://github.com/redis/hiredis/pull/1020))
- Add asynchronous API tests [@bjosv](https://github.com/bjosv) ([\#1010](https://github.com/redis/hiredis/pull/1010))
- Correcting the build target `coverage` for enabled SSL [@bjosv](https://github.com/bjosv) ([\#1009](https://github.com/redis/hiredis/pull/1009))
- GH Actions: Run SSL tests during CI [@bjosv](https://github.com/bjosv) ([\#1008](https://github.com/redis/hiredis/pull/1008))
- GH: Actions - Add valgrind and CMake [@michael-grunder](https://github.com/michael-grunder) ([\#1004](https://github.com/redis/hiredis/pull/1004))
- Add Centos8 tests in GH Actions [@michael-grunder](https://github.com/michael-grunder) ([\#1001](https://github.com/redis/hiredis/pull/1001))
- We should run actions on PRs [@michael-grunder](https://github.com/michael-grunder) (([\#1000](https://github.com/redis/hiredis/pull/1000))
- Add Cygwin test in GitHub actions [@michael-grunder](https://github.com/michael-grunder) ([\#999](https://github.com/redis/hiredis/pull/999))
- Add Windows tests in GitHub actions [@michael-grunder](https://github.com/michael-grunder) ([\#996](https://github.com/redis/hiredis/pull/996))
- Switch to GitHub actions [@michael-grunder](https://github.com/michael-grunder) ([\#995](https://github.com/redis/hiredis/pull/995))
- Minor refactor of CVE-2021-32765 fix. [@michael-grunder](https://github.com/michael-grunder) ([\#993](https://github.com/redis/hiredis/pull/993))
- Remove extra comma from CMake var. [@xkszltl](https://github.com/xkszltl) ([\#988](https://github.com/redis/hiredis/pull/988))
- Add REDIS\_OPT\_PREFER\_UNSPEC [@michael-grunder](https://github.com/michael-grunder) ([\#1101](https://github.com/redis/hiredis/pull/1101))
## Contributors
We'd like to thank all the contributors who worked on this release!
<a href="https://github.com/EricDeng1001"><img src="https://github.com/EricDeng1001.png" width="32" height="32"></a>
<a href="https://github.com/Lipraxde"><img src="https://github.com/Lipraxde.png" width="32" height="32"></a>
<a href="https://github.com/MichaelSuen-thePointer"><img src="https://github.com/MichaelSuen-thePointer.png" width="32" height="32"></a>
<a href="https://github.com/SukkaW"><img src="https://github.com/SukkaW.png" width="32" height="32"></a>
<a href="https://github.com/adobeturchenko"><img src="https://github.com/adobeturchenko.png" width="32" height="32"></a>
<a href="https://github.com/afcidk"><img src="https://github.com/afcidk.png" width="32" height="32"></a>
<a href="https://github.com/bit0fun"><img src="https://github.com/bit0fun.png" width="32" height="32"></a>
<a href="https://github.com/bjosv"><img src="https://github.com/bjosv.png" width="32" height="32"></a>
<a href="https://github.com/bugwz"><img src="https://github.com/bugwz.png" width="32" height="32"></a>
<a href="https://github.com/catterer"><img src="https://github.com/catterer.png" width="32" height="32"></a>
<a href="https://github.com/chayim"><img src="https://github.com/chayim.png" width="32" height="32"></a>
<a href="https://github.com/devnexen"><img src="https://github.com/devnexen.png" width="32" height="32"></a>
<a href="https://github.com/ffontaine"><img src="https://github.com/ffontaine.png" width="32" height="32"></a>
<a href="https://github.com/ithewei"><img src="https://github.com/ithewei.png" width="32" height="32"></a>
<a href="https://github.com/jengab"><img src="https://github.com/jengab.png" width="32" height="32"></a>
<a href="https://github.com/kristjanvalur"><img src="https://github.com/kristjanvalur.png" width="32" height="32"></a>
<a href="https://github.com/michael-grunder"><img src="https://github.com/michael-grunder.png" width="32" height="32"></a>
<a href="https://github.com/noxiouz"><img src="https://github.com/noxiouz.png" width="32" height="32"></a>
<a href="https://github.com/mtdxc"><img src="https://github.com/mtdxc.png" width="32" height="32"></a>
<a href="https://github.com/orgads"><img src="https://github.com/orgads.png" width="32" height="32"></a>
<a href="https://github.com/pbtummillo"><img src="https://github.com/pbtummillo.png" width="32" height="32"></a>
<a href="https://github.com/pizhenwei"><img src="https://github.com/pizhenwei.png" width="32" height="32"></a>
<a href="https://github.com/scddev"><img src="https://github.com/scddev.png" width="32" height="32"></a>
<a href="https://github.com/smmir-cent"><img src="https://github.com/smmir-cent.png" width="32" height="32"></a>
<a href="https://github.com/stanhu"><img src="https://github.com/stanhu.png" width="32" height="32"></a>
<a href="https://github.com/sundb"><img src="https://github.com/sundb.png" width="32" height="32"></a>
<a href="https://github.com/vturchenko"><img src="https://github.com/vturchenko.png" width="32" height="32"></a>
<a href="https://github.com/xkszltl"><img src="https://github.com/xkszltl.png" width="32" height="32"></a>
<a href="https://github.com/yossigo"><img src="https://github.com/yossigo.png" width="32" height="32"></a>
<a href="https://github.com/zhangtaoXT5"><img src="https://github.com/zhangtaoXT5.png" width="32" height="32"></a>
<a href="https://github.com/zuiderkwast"><img src="https://github.com/zuiderkwast.png" width="32" height="32"></a>
## [1.0.2](https://github.com/redis/hiredis/tree/v1.0.2) - (2021-10-07)
Announcing Hiredis v1.0.2, which fixes CVE-2021-32765 but returns the SONAME to the correct value of `1.0.0`.
- [Revert SONAME bump](https://github.com/redis/hiredis/commit/d4e6f109a064690cde64765c654e679fea1d3548)
([Michael Grunder](https://github.com/michael-grunder))
## [1.0.1](https://github.com/redis/hiredis/tree/v1.0.1) - (2021-10-04)
<span style="color:red">This release erroneously bumped the SONAME, please use [1.0.2](https://github.com/redis/hiredis/tree/v1.0.2)</span>
Announcing Hiredis v1.0.1, a security release fixing CVE-2021-32765
- Fix for [CVE-2021-32765](https://github.com/redis/hiredis/security/advisories/GHSA-hfm9-39pp-55p2)
[commit](https://github.com/redis/hiredis/commit/76a7b10005c70babee357a7d0f2becf28ec7ed1e)
([Yossi Gottlieb](https://github.com/yossigo))
_Thanks to [Yossi Gottlieb](https://github.com/yossigo) for the security fix and to [Microsoft Security Vulnerability Research](https://www.microsoft.com/en-us/msrc/msvr) for finding the bug._ :sparkling_heart:
## [1.0.0](https://github.com/redis/hiredis/tree/v1.0.0) - (2020-08-03)
Announcing Hiredis v1.0.0, which adds support for RESP3, SSL connections, allocator injection, and better Windows support! :tada:
_A big thanks to everyone who helped with this release. The following list includes everyone who contributed at least five lines, sorted by lines contributed._ :sparkling_heart:
[Michael Grunder](https://github.com/michael-grunder), [Yossi Gottlieb](https://github.com/yossigo),
[Mark Nunberg](https://github.com/mnunberg), [Marcus Geelnard](https://github.com/mbitsnbites),
[Justin Brewer](https://github.com/justinbrewer), [Valentino Geron](https://github.com/valentinogeron),
[Minun Dragonation](https://github.com/dragonation), [Omri Steiner](https://github.com/OmriSteiner),
[Sangmoon Yi](https://github.com/jman-krafton), [Jinjiazh](https://github.com/jinjiazhang),
[Odin Hultgren Van Der Horst](https://github.com/Miniwoffer), [Muhammad Zahalqa](https://github.com/tryfinally),
[Nick Rivera](https://github.com/heronr), [Qi Yang](https://github.com/movebean),
[kevin1018](https://github.com/kevin1018)
[Full Changelog](https://github.com/redis/hiredis/compare/v0.14.1...v1.0.0)
**BREAKING CHANGES**:
* `redisOptions` now has two timeout fields. One for connecting, and one for commands. If you're presently using `options->timeout` you will need to change it to use `options->connect_timeout`. (See [example](https://github.com/redis/hiredis/commit/38b5ae543f5c99eb4ccabbe277770fc6bc81226f#diff-86ba39d37aa829c8c82624cce4f049fbL36))
* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now protocol errors. This is consistent
with the RESP specification. On 32-bit platforms, the upper bound is lowered to `SIZE_MAX`.
* `redisReplyObjectFunctions.createArray` now takes `size_t` for its length parameter.
**New features:**
- Support for RESP3
[\#697](https://github.com/redis/hiredis/pull/697),
[\#805](https://github.com/redis/hiredis/pull/805),
[\#819](https://github.com/redis/hiredis/pull/819),
[\#841](https://github.com/redis/hiredis/pull/841)
([Yossi Gottlieb](https://github.com/yossigo), [Michael Grunder](https://github.com/michael-grunder))
- Support for SSL connections
[\#645](https://github.com/redis/hiredis/pull/645),
[\#699](https://github.com/redis/hiredis/pull/699),
[\#702](https://github.com/redis/hiredis/pull/702),
[\#708](https://github.com/redis/hiredis/pull/708),
[\#711](https://github.com/redis/hiredis/pull/711),
[\#821](https://github.com/redis/hiredis/pull/821),
[more](https://github.com/redis/hiredis/pulls?q=is%3Apr+is%3Amerged+SSL)
([Mark Nunberg](https://github.com/mnunberg), [Yossi Gottlieb](https://github.com/yossigo))
- Run-time allocator injection
[\#800](https://github.com/redis/hiredis/pull/800)
([Michael Grunder](https://github.com/michael-grunder))
- Improved Windows support (including MinGW and Windows CI)
[\#652](https://github.com/redis/hiredis/pull/652),
[\#663](https://github.com/redis/hiredis/pull/663)
([Marcus Geelnard](https://www.bitsnbites.eu/author/m/))
- Adds support for distinct connect and command timeouts
[\#839](https://github.com/redis/hiredis/pull/839),
[\#829](https://github.com/redis/hiredis/pull/829)
([Valentino Geron](https://github.com/valentinogeron))
- Add generic pointer and destructor to `redisContext` that users can use for context.
[\#855](https://github.com/redis/hiredis/pull/855)
([Michael Grunder](https://github.com/michael-grunder))
**Closed issues (that involved code changes):**
- Makefile does not install TLS libraries [\#809](https://github.com/redis/hiredis/issues/809)
- redisConnectWithOptions should not set command timeout [\#722](https://github.com/redis/hiredis/issues/722), [\#829](https://github.com/redis/hiredis/pull/829) ([valentinogeron](https://github.com/valentinogeron))
- Fix integer overflow in `sdsrange` [\#827](https://github.com/redis/hiredis/issues/827)
- INFO & CLUSTER commands failed when using RESP3 [\#802](https://github.com/redis/hiredis/issues/802)
- Windows compatibility patches [\#687](https://github.com/redis/hiredis/issues/687), [\#838](https://github.com/redis/hiredis/issues/838), [\#842](https://github.com/redis/hiredis/issues/842)
- RESP3 PUSH messages incorrectly use pending callback [\#825](https://github.com/redis/hiredis/issues/825)
- Asynchronous PSUBSCRIBE command fails when using RESP3 [\#815](https://github.com/redis/hiredis/issues/815)
- New SSL API [\#804](https://github.com/redis/hiredis/issues/804), [\#813](https://github.com/redis/hiredis/issues/813)
- Hard-coded limit of nested reply depth [\#794](https://github.com/redis/hiredis/issues/794)
- Fix TCP_NODELAY in Windows/OSX [\#679](https://github.com/redis/hiredis/issues/679), [\#690](https://github.com/redis/hiredis/issues/690), [\#779](https://github.com/redis/hiredis/issues/779), [\#785](https://github.com/redis/hiredis/issues/785),
- Added timers to libev adapter. [\#778](https://github.com/redis/hiredis/issues/778), [\#795](https://github.com/redis/hiredis/pull/795)
- Initialization discards const qualifier [\#777](https://github.com/redis/hiredis/issues/777)
- \[BUG\]\[MinGW64\] Error setting socket timeout [\#775](https://github.com/redis/hiredis/issues/775)
- undefined reference to hi_malloc [\#769](https://github.com/redis/hiredis/issues/769)
- hiredis pkg-config file incorrectly ignores multiarch libdir spec'n [\#767](https://github.com/redis/hiredis/issues/767)
- Don't use -G to build shared object on Solaris [\#757](https://github.com/redis/hiredis/issues/757)
- error when make USE\_SSL=1 [\#748](https://github.com/redis/hiredis/issues/748)
- Allow to change SSL Mode [\#646](https://github.com/redis/hiredis/issues/646)
- hiredis/adapters/libevent.h memleak [\#618](https://github.com/redis/hiredis/issues/618)
- redisLibuvPoll crash when server closes the connetion [\#545](https://github.com/redis/hiredis/issues/545)
- about redisAsyncDisconnect question [\#518](https://github.com/redis/hiredis/issues/518)
- hiredis adapters libuv error for help [\#508](https://github.com/redis/hiredis/issues/508)
- API/ABI changes analysis [\#506](https://github.com/redis/hiredis/issues/506)
- Memory leak patch in Redis [\#502](https://github.com/redis/hiredis/issues/502)
- Remove the depth limitation [\#421](https://github.com/redis/hiredis/issues/421)
**Merged pull requests:**
- Move SSL management to a distinct private pointer [\#855](https://github.com/redis/hiredis/pull/855) ([michael-grunder](https://github.com/michael-grunder))
- Move include to sockcompat.h to maintain style [\#850](https://github.com/redis/hiredis/pull/850) ([michael-grunder](https://github.com/michael-grunder))
- Remove erroneous tag and add license to push example [\#849](https://github.com/redis/hiredis/pull/849) ([michael-grunder](https://github.com/michael-grunder))
- fix windows compiling with mingw [\#848](https://github.com/redis/hiredis/pull/848) ([rmalizia44](https://github.com/rmalizia44))
- Some Windows quality of life improvements. [\#846](https://github.com/redis/hiredis/pull/846) ([michael-grunder](https://github.com/michael-grunder))
- Use \_WIN32 define instead of WIN32 [\#845](https://github.com/redis/hiredis/pull/845) ([michael-grunder](https://github.com/michael-grunder))
- Non Linux CI fixes [\#844](https://github.com/redis/hiredis/pull/844) ([michael-grunder](https://github.com/michael-grunder))
- Resp3 oob push support [\#841](https://github.com/redis/hiredis/pull/841) ([michael-grunder](https://github.com/michael-grunder))
- fix \#785: defer TCP\_NODELAY in async tcp connections [\#836](https://github.com/redis/hiredis/pull/836) ([OmriSteiner](https://github.com/OmriSteiner))
- sdsrange overflow fix [\#830](https://github.com/redis/hiredis/pull/830) ([michael-grunder](https://github.com/michael-grunder))
- Use explicit pointer casting for c++ compatibility [\#826](https://github.com/redis/hiredis/pull/826) ([aureus1](https://github.com/aureus1))
- Document allocator injection and completeness fix in test.c [\#824](https://github.com/redis/hiredis/pull/824) ([michael-grunder](https://github.com/michael-grunder))
- Use unique names for allocator struct members [\#823](https://github.com/redis/hiredis/pull/823) ([michael-grunder](https://github.com/michael-grunder))
- New SSL API to replace redisSecureConnection\(\). [\#821](https://github.com/redis/hiredis/pull/821) ([yossigo](https://github.com/yossigo))
- Add logic to handle RESP3 push messages [\#819](https://github.com/redis/hiredis/pull/819) ([michael-grunder](https://github.com/michael-grunder))
- Use standrad isxdigit instead of custom helper function. [\#814](https://github.com/redis/hiredis/pull/814) ([tryfinally](https://github.com/tryfinally))
- Fix missing SSL build/install options. [\#812](https://github.com/redis/hiredis/pull/812) ([yossigo](https://github.com/yossigo))
- Add link to ABI tracker [\#808](https://github.com/redis/hiredis/pull/808) ([michael-grunder](https://github.com/michael-grunder))
- Resp3 verbatim string support [\#805](https://github.com/redis/hiredis/pull/805) ([michael-grunder](https://github.com/michael-grunder))
- Allow users to replace allocator and handle OOM everywhere. [\#800](https://github.com/redis/hiredis/pull/800) ([michael-grunder](https://github.com/michael-grunder))
- Remove nested depth limitation. [\#797](https://github.com/redis/hiredis/pull/797) ([michael-grunder](https://github.com/michael-grunder))
- Attempt to fix compilation on Solaris [\#796](https://github.com/redis/hiredis/pull/796) ([michael-grunder](https://github.com/michael-grunder))
- Support timeouts in libev adapater [\#795](https://github.com/redis/hiredis/pull/795) ([michael-grunder](https://github.com/michael-grunder))
- Fix pkgconfig when installing to a custom lib dir [\#793](https://github.com/redis/hiredis/pull/793) ([michael-grunder](https://github.com/michael-grunder))
- Fix USE\_SSL=1 make/cmake on OSX and CMake tests [\#789](https://github.com/redis/hiredis/pull/789) ([michael-grunder](https://github.com/michael-grunder))
- Use correct libuv call on Windows [\#784](https://github.com/redis/hiredis/pull/784) ([michael-grunder](https://github.com/michael-grunder))
- Added CMake package config and fixed hiredis\_ssl on Windows [\#783](https://github.com/redis/hiredis/pull/783) ([michael-grunder](https://github.com/michael-grunder))
- CMake: Set hiredis\_ssl shared object version. [\#780](https://github.com/redis/hiredis/pull/780) ([yossigo](https://github.com/yossigo))
- Win32 tests and timeout fix [\#776](https://github.com/redis/hiredis/pull/776) ([michael-grunder](https://github.com/michael-grunder))
- Provides an optional cleanup callback for async data. [\#768](https://github.com/redis/hiredis/pull/768) ([heronr](https://github.com/heronr))
- Housekeeping fixes [\#764](https://github.com/redis/hiredis/pull/764) ([michael-grunder](https://github.com/michael-grunder))
- install alloc.h [\#756](https://github.com/redis/hiredis/pull/756) ([ch1aki](https://github.com/ch1aki))
- fix spelling mistakes [\#746](https://github.com/redis/hiredis/pull/746) ([ShooterIT](https://github.com/ShooterIT))
- Free the reply in redisGetReply when passed NULL [\#741](https://github.com/redis/hiredis/pull/741) ([michael-grunder](https://github.com/michael-grunder))
- Fix dead code in sslLogCallback relating to should\_log variable. [\#737](https://github.com/redis/hiredis/pull/737) ([natoscott](https://github.com/natoscott))
- Fix typo in dict.c. [\#731](https://github.com/redis/hiredis/pull/731) ([Kevin-Xi](https://github.com/Kevin-Xi))
- Adding an option to DISABLE\_TESTS [\#727](https://github.com/redis/hiredis/pull/727) ([pbotros](https://github.com/pbotros))
- Update README with SSL support. [\#720](https://github.com/redis/hiredis/pull/720) ([yossigo](https://github.com/yossigo))
- Fixes leaks in unit tests [\#715](https://github.com/redis/hiredis/pull/715) ([michael-grunder](https://github.com/michael-grunder))
- SSL Tests [\#711](https://github.com/redis/hiredis/pull/711) ([yossigo](https://github.com/yossigo))
- SSL Reorganization [\#708](https://github.com/redis/hiredis/pull/708) ([yossigo](https://github.com/yossigo))
- Fix MSVC build. [\#706](https://github.com/redis/hiredis/pull/706) ([yossigo](https://github.com/yossigo))
- SSL: Properly report SSL\_connect\(\) errors. [\#702](https://github.com/redis/hiredis/pull/702) ([yossigo](https://github.com/yossigo))
- Silent SSL trace to stdout by default. [\#699](https://github.com/redis/hiredis/pull/699) ([yossigo](https://github.com/yossigo))
- Port RESP3 support from Redis. [\#697](https://github.com/redis/hiredis/pull/697) ([yossigo](https://github.com/yossigo))
- Removed whitespace before newline [\#691](https://github.com/redis/hiredis/pull/691) ([Miniwoffer](https://github.com/Miniwoffer))
- Add install adapters header files [\#688](https://github.com/redis/hiredis/pull/688) ([kevin1018](https://github.com/kevin1018))
- Remove unnecessary null check before free [\#684](https://github.com/redis/hiredis/pull/684) ([qlyoung](https://github.com/qlyoung))
- redisReaderGetReply leak memory [\#671](https://github.com/redis/hiredis/pull/671) ([movebean](https://github.com/movebean))
- fix timeout code in windows [\#670](https://github.com/redis/hiredis/pull/670) ([jman-krafton](https://github.com/jman-krafton))
- test: fix errstr matching for musl libc [\#665](https://github.com/redis/hiredis/pull/665) ([ghost](https://github.com/ghost))
- Windows: MinGW fixes and Windows Travis builders [\#663](https://github.com/redis/hiredis/pull/663) ([mbitsnbites](https://github.com/mbitsnbites))
- The setsockopt and getsockopt API diffs from BSD socket and WSA one [\#662](https://github.com/redis/hiredis/pull/662) ([dragonation](https://github.com/dragonation))
- Fix Compile Error On Windows \(Visual Studio\) [\#658](https://github.com/redis/hiredis/pull/658) ([jinjiazhang](https://github.com/jinjiazhang))
- Fix NXDOMAIN test case [\#653](https://github.com/redis/hiredis/pull/653) ([michael-grunder](https://github.com/michael-grunder))
- Add MinGW support [\#652](https://github.com/redis/hiredis/pull/652) ([mbitsnbites](https://github.com/mbitsnbites))
- SSL Support [\#645](https://github.com/redis/hiredis/pull/645) ([mnunberg](https://github.com/mnunberg))
- Fix Invalid argument after redisAsyncConnectUnix [\#644](https://github.com/redis/hiredis/pull/644) ([codehz](https://github.com/codehz))
- Makefile: use predefined AR [\#632](https://github.com/redis/hiredis/pull/632) ([Mic92](https://github.com/Mic92))
- FreeBSD build fix [\#628](https://github.com/redis/hiredis/pull/628) ([devnexen](https://github.com/devnexen))
- Fix errors not propagating properly with libuv.h. [\#624](https://github.com/redis/hiredis/pull/624) ([yossigo](https://github.com/yossigo))
- Update README.md [\#621](https://github.com/redis/hiredis/pull/621) ([Crunsher](https://github.com/Crunsher))
- Fix redisBufferRead documentation [\#620](https://github.com/redis/hiredis/pull/620) ([hacst](https://github.com/hacst))
- Add CPPFLAGS to REAL\_CFLAGS [\#614](https://github.com/redis/hiredis/pull/614) ([thomaslee](https://github.com/thomaslee))
- Update createArray to take size\_t [\#597](https://github.com/redis/hiredis/pull/597) ([justinbrewer](https://github.com/justinbrewer))
- fix common realloc mistake and add null check more [\#580](https://github.com/redis/hiredis/pull/580) ([charsyam](https://github.com/charsyam))
- Proper error reporting for connect failures [\#578](https://github.com/redis/hiredis/pull/578) ([mnunberg](https://github.com/mnunberg))
\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*
## [1.0.0-rc1](https://github.com/redis/hiredis/tree/v1.0.0-rc1) - (2020-07-29)
_Note: There were no changes to code between v1.0.0-rc1 and v1.0.0 so see v1.0.0 for changelog_
### 0.14.1 (2020-03-13)
* Adds safe allocation wrappers (CVE-2020-7105, #747, #752) (Michael Grunder)
### 0.14.0 (2018-09-25)
**BREAKING CHANGES**:
* Change `redisReply.len` to `size_t`, as it denotes the the size of a string
User code should compare this to `size_t` values as well.
If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before.
* Make string2ll static to fix conflict with Redis (Tom Lee [c3188b])
* Use -dynamiclib instead of -shared for OSX (Ryan Schmidt [a65537])
* Use string2ll from Redis w/added tests (Michael Grunder [7bef04, 60f622])
* Makefile - OSX compilation fixes (Ryan Schmidt [881fcb, 0e9af8])
* Remove redundant NULL checks (Justin Brewer [54acc8, 58e6b8])
* Fix bulk and multi-bulk length truncation (Justin Brewer [109197])
* Fix SIGSEGV in OpenBSD by checking for NULL before calling freeaddrinfo (Justin Brewer [546d94])
* Several POSIX compatibility fixes (Justin Brewer [bbeab8, 49bbaa, d1c1b6])
* Makefile - Compatibility fixes (Dimitri Vorobiev [3238cf, 12a9d1])
* Makefile - Fix make install on FreeBSD (Zach Shipko [a2ef2b])
* Makefile - don't assume $(INSTALL) is cp (Igor Gnatenko [725a96])
* Separate side-effect causing function from assert and small cleanup (amallia [b46413, 3c3234])
* Don't send negative values to `__redisAsyncCommand` (Frederik Deweerdt [706129])
* Fix leak if setsockopt fails (Frederik Deweerdt [e21c9c])
* Fix libevent leak (zfz [515228])
* Clean up GCC warning (Ichito Nagata [2ec774])
* Keep track of errno in `__redisSetErrorFromErrno()` as snprintf may use it (Jin Qing [25cd88])
* Solaris compilation fix (Donald Whyte [41b07d])
* Reorder linker arguments when building examples (Tustfarm-heart [06eedd])
* Keep track of subscriptions in case of rapid subscribe/unsubscribe (Hyungjin Kim [073dc8, be76c5, d46999])
* libuv use after free fix (Paul Scott [cbb956])
* Properly close socket fd on reconnect attempt (WSL [64d1ec])
* Skip valgrind in OSX tests (Jan-Erik Rediger [9deb78])
* Various updates for Travis testing OSX (Ted Nyman [fa3774, 16a459, bc0ea5])
* Update libevent (Chris Xin [386802])
* Change sds.h for building in C++ projects (Ali Volkan ATLI [f5b32e])
* Use proper format specifier in redisFormatSdsCommandArgv (Paulino Huerta, Jan-Erik Rediger [360a06, 8655a6])
* Better handling of NULL reply in example code (Jan-Erik Rediger [1b8ed3])
* Prevent overflow when formatting an error (Jan-Erik Rediger [0335cb])
* Compatibility fix for strerror_r (Tom Lee [bb1747])
* Properly detect integer parse/overflow errors (Justin Brewer [93421f])
* Adds CI for Windows and cygwin fixes (owent, [6c53d6, 6c3e40])
* Catch a buffer overflow when formatting the error message
* Import latest upstream sds. This breaks applications that are linked against the old hiredis v0.13
* Fix warnings, when compiled with -Wshadow
* Make hiredis compile in Cygwin on Windows, now CI-tested
* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now
protocol errors. This is consistent with the RESP specification. On 32-bit
platforms, the upper bound is lowered to `SIZE_MAX`.
* Remove backwards compatibility macro's
This removes the following old function aliases, use the new name now:
| Old | New |
| --------------------------- | ---------------------- |
| redisReplyReaderCreate | redisReaderCreate |
| redisReplyReaderCreate | redisReaderCreate |
| redisReplyReaderFree | redisReaderFree |
| redisReplyReaderFeed | redisReaderFeed |
| redisReplyReaderGetReply | redisReaderGetReply |
| redisReplyReaderSetPrivdata | redisReaderSetPrivdata |
| redisReplyReaderGetObject | redisReaderGetObject |
| redisReplyReaderGetError | redisReaderGetError |
* The `DEBUG` variable in the Makefile was renamed to `DEBUG_FLAGS`
Previously it broke some builds for people that had `DEBUG` set to some arbitrary value,
due to debugging other software.
By renaming we avoid unintentional name clashes.
Simply rename `DEBUG` to `DEBUG_FLAGS` in your environment to make it working again.
### 0.13.3 (2015-09-16)
* Revert "Clear `REDIS_CONNECTED` flag when connection is closed".
* Make tests pass on FreeBSD (Thanks, Giacomo Olgeni)
If the `REDIS_CONNECTED` flag is cleared,
the async onDisconnect callback function will never be called.
This causes problems as the disconnect is never reported back to the user.
### 0.13.2 (2015-08-25)
* Prevent crash on pending replies in async code (Thanks, @switch-st)
* Clear `REDIS_CONNECTED` flag when connection is closed (Thanks, Jerry Jacobs)
* Add MacOS X addapter (Thanks, @dizzus)
* Add Qt adapter (Thanks, Pietro Cerutti)
* Add Ivykis adapter (Thanks, Gergely Nagy)
All adapters are provided as is and are only tested where possible.
### 0.13.1 (2015-05-03)
This is a bug fix release.
The new `reconnect` method introduced new struct members, which clashed with pre-defined names in pre-C99 code.
Another commit forced C99 compilation just to make it work, but of course this is not desirable for outside projects.
Other non-C99 code can now use hiredis as usual again.
Sorry for the inconvenience.
* Fix memory leak in async reply handling (Salvatore Sanfilippo)
* Rename struct member to avoid name clash with pre-c99 code (Alex Balashov, ncopa)
### 0.13.0 (2015-04-16)
This release adds a minimal Windows compatibility layer.
The parser, standalone since v0.12.0, can now be compiled on Windows
(and thus used in other client libraries as well)
* Windows compatibility layer for parser code (tzickel)
* Properly escape data printed to PKGCONF file (Dan Skorupski)
* Fix tests when assert() undefined (Keith Bennett, Matt Stancliff)
* Implement a reconnect method for the client context, this changes the structure of `redisContext` (Aaron Bedra)
### 0.12.1 (2015-01-26)
* Fix `make install`: DESTDIR support, install all required files, install PKGCONF in proper location
* Fix `make test` as 32 bit build on 64 bit platform
### 0.12.0 (2015-01-22)
* Add optional KeepAlive support
* Try again on EINTR errors
* Add libuv adapter
* Add IPv6 support
* Remove possibility of multiple close on same fd
* Add ability to bind source address on connect
* Add redisConnectFd() and redisFreeKeepFd()
* Fix getaddrinfo() memory leak
* Free string if it is unused (fixes memory leak)
* Improve redisAppendCommandArgv performance 2.5x
* Add support for SO_REUSEADDR
* Fix redisvFormatCommand format parsing
* Add GLib 2.0 adapter
* Refactor reading code into read.c
* Fix errno error buffers to not clobber errors
* Generate pkgconf during build
* Silence _BSD_SOURCE warnings
* Improve digit counting for multibulk creation
### 0.11.0
* Increase the maximum multi-bulk reply depth to 7.
* Increase the read buffer size from 2k to 16k.
* Use poll(2) instead of select(2) to support large fds (>= 1024).
### 0.10.1
* Makefile overhaul. Important to check out if you override one or more
variables using environment variables or via arguments to the "make" tool.
* Issue #45: Fix potential memory leak for a multi bulk reply with 0 elements
being created by the default reply object functions.
* Issue #43: Don't crash in an asynchronous context when Redis returns an error
reply after the connection has been made (this happens when the maximum
number of connections is reached).
### 0.10.0
* See commit log.
+237
View File
@@ -0,0 +1,237 @@
CMAKE_MINIMUM_REQUIRED(VERSION 3.0.0)
OPTION(BUILD_SHARED_LIBS "Build shared libraries" ON)
OPTION(ENABLE_SSL "Build hiredis_ssl for SSL support" OFF)
OPTION(DISABLE_TESTS "If tests should be compiled or not" OFF)
OPTION(ENABLE_SSL_TESTS "Should we test SSL connections" OFF)
OPTION(ENABLE_EXAMPLES "Enable building hiredis examples" OFF)
OPTION(ENABLE_ASYNC_TESTS "Should we run all asynchronous API tests" OFF)
MACRO(getVersionBit name)
SET(VERSION_REGEX "^#define ${name} (.+)$")
FILE(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/hiredis.h"
VERSION_BIT REGEX ${VERSION_REGEX})
STRING(REGEX REPLACE ${VERSION_REGEX} "\\1" ${name} "${VERSION_BIT}")
ENDMACRO(getVersionBit)
getVersionBit(HIREDIS_MAJOR)
getVersionBit(HIREDIS_MINOR)
getVersionBit(HIREDIS_PATCH)
getVersionBit(HIREDIS_SONAME)
SET(VERSION "${HIREDIS_MAJOR}.${HIREDIS_MINOR}.${HIREDIS_PATCH}")
MESSAGE("Detected version: ${VERSION}")
PROJECT(hiredis LANGUAGES "C" VERSION "${VERSION}")
INCLUDE(GNUInstallDirs)
# Hiredis requires C99
SET(CMAKE_C_STANDARD 99)
SET(CMAKE_DEBUG_POSTFIX d)
SET(hiredis_sources
alloc.c
async.c
hiredis.c
net.c
read.c
sds.c
sockcompat.c)
SET(hiredis_sources ${hiredis_sources})
IF(WIN32)
ADD_DEFINITIONS(-D_CRT_SECURE_NO_WARNINGS -DWIN32_LEAN_AND_MEAN)
ENDIF()
ADD_LIBRARY(hiredis ${hiredis_sources})
ADD_LIBRARY(hiredis::hiredis ALIAS hiredis)
set(hiredis_export_name hiredis CACHE STRING "Name of the exported target")
set_target_properties(hiredis PROPERTIES EXPORT_NAME ${hiredis_export_name})
SET_TARGET_PROPERTIES(hiredis
PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE
VERSION "${HIREDIS_SONAME}")
IF(MSVC)
SET_TARGET_PROPERTIES(hiredis
PROPERTIES COMPILE_FLAGS /Z7)
ENDIF()
IF(WIN32)
TARGET_LINK_LIBRARIES(hiredis PUBLIC ws2_32 crypt32)
ELSEIF(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
TARGET_LINK_LIBRARIES(hiredis PUBLIC m)
ELSEIF(CMAKE_SYSTEM_NAME MATCHES "SunOS")
TARGET_LINK_LIBRARIES(hiredis PUBLIC socket)
ENDIF()
TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
CONFIGURE_FILE(hiredis.pc.in hiredis.pc @ONLY)
set(CPACK_PACKAGE_VENDOR "Redis")
set(CPACK_PACKAGE_DESCRIPTION "\
Hiredis is a minimalistic C client library for the Redis database.
It is minimalistic because it just adds minimal support for the protocol, \
but at the same time it uses a high level printf-alike API in order to make \
it much higher level than otherwise suggested by its minimal code base and the \
lack of explicit bindings for every Redis command.
Apart from supporting sending commands and receiving replies, it comes with a \
reply parser that is decoupled from the I/O layer. It is a stream parser designed \
for easy reusability, which can for instance be used in higher level language bindings \
for efficient reply parsing.
Hiredis only supports the binary-safe Redis protocol, so you can use it with any Redis \
version >= 1.2.0.
The library comes with multiple APIs. There is the synchronous API, the asynchronous API \
and the reply parsing API.")
set(CPACK_PACKAGE_HOMEPAGE_URL "https://github.com/redis/hiredis")
set(CPACK_PACKAGE_CONTACT "michael dot grunder at gmail dot com")
set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
set(CPACK_RPM_PACKAGE_AUTOREQPROV ON)
include(CPack)
INSTALL(TARGETS hiredis
EXPORT hiredis-targets
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
if (MSVC AND BUILD_SHARED_LIBS)
INSTALL(FILES $<TARGET_PDB_FILE:hiredis>
DESTINATION ${CMAKE_INSTALL_BINDIR}
CONFIGURATIONS Debug RelWithDebInfo)
endif()
# For NuGet packages
INSTALL(FILES hiredis.targets
DESTINATION build/native)
INSTALL(FILES hiredis.h read.h sds.h async.h alloc.h sockcompat.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis)
INSTALL(DIRECTORY adapters
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis)
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis.pc
DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
export(EXPORT hiredis-targets
FILE "${CMAKE_CURRENT_BINARY_DIR}/hiredis-targets.cmake"
NAMESPACE hiredis::)
if(WIN32)
SET(CMAKE_CONF_INSTALL_DIR share/hiredis)
else()
SET(CMAKE_CONF_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR}/cmake/hiredis)
endif()
SET(INCLUDE_INSTALL_DIR include)
include(CMakePackageConfigHelpers)
write_basic_package_version_file("${CMAKE_CURRENT_BINARY_DIR}/hiredis-config-version.cmake"
COMPATIBILITY SameMajorVersion)
configure_package_config_file(hiredis-config.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/hiredis-config.cmake
INSTALL_DESTINATION ${CMAKE_CONF_INSTALL_DIR}
PATH_VARS INCLUDE_INSTALL_DIR)
INSTALL(EXPORT hiredis-targets
FILE hiredis-targets.cmake
NAMESPACE hiredis::
DESTINATION ${CMAKE_CONF_INSTALL_DIR})
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis-config.cmake
${CMAKE_CURRENT_BINARY_DIR}/hiredis-config-version.cmake
DESTINATION ${CMAKE_CONF_INSTALL_DIR})
IF(ENABLE_SSL)
IF (NOT OPENSSL_ROOT_DIR)
IF (APPLE)
SET(OPENSSL_ROOT_DIR "/usr/local/opt/openssl")
ENDIF()
ENDIF()
FIND_PACKAGE(OpenSSL REQUIRED)
SET(hiredis_ssl_sources
ssl.c)
ADD_LIBRARY(hiredis_ssl ${hiredis_ssl_sources})
ADD_LIBRARY(hiredis::hiredis_ssl ALIAS hiredis_ssl)
IF (APPLE AND BUILD_SHARED_LIBS)
SET_PROPERTY(TARGET hiredis_ssl PROPERTY LINK_FLAGS "-Wl,-undefined -Wl,dynamic_lookup")
ENDIF()
SET_TARGET_PROPERTIES(hiredis_ssl
PROPERTIES
WINDOWS_EXPORT_ALL_SYMBOLS TRUE
VERSION "${HIREDIS_SONAME}")
IF(MSVC)
SET_TARGET_PROPERTIES(hiredis_ssl
PROPERTIES COMPILE_FLAGS /Z7)
ENDIF()
TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE OpenSSL::SSL)
IF(WIN32)
TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE hiredis)
ENDIF()
CONFIGURE_FILE(hiredis_ssl.pc.in hiredis_ssl.pc @ONLY)
INSTALL(TARGETS hiredis_ssl
EXPORT hiredis_ssl-targets
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
if (MSVC AND BUILD_SHARED_LIBS)
INSTALL(FILES $<TARGET_PDB_FILE:hiredis_ssl>
DESTINATION ${CMAKE_INSTALL_BINDIR}
CONFIGURATIONS Debug RelWithDebInfo)
endif()
INSTALL(FILES hiredis_ssl.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis)
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl.pc
DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
export(EXPORT hiredis_ssl-targets
FILE "${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl-targets.cmake"
NAMESPACE hiredis::)
if(WIN32)
SET(CMAKE_CONF_INSTALL_DIR share/hiredis_ssl)
else()
SET(CMAKE_CONF_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR}/cmake/hiredis_ssl)
endif()
configure_package_config_file(hiredis_ssl-config.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl-config.cmake
INSTALL_DESTINATION ${CMAKE_CONF_INSTALL_DIR}
PATH_VARS INCLUDE_INSTALL_DIR)
INSTALL(EXPORT hiredis_ssl-targets
FILE hiredis_ssl-targets.cmake
NAMESPACE hiredis::
DESTINATION ${CMAKE_CONF_INSTALL_DIR})
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl-config.cmake
DESTINATION ${CMAKE_CONF_INSTALL_DIR})
ENDIF()
IF(NOT DISABLE_TESTS)
ENABLE_TESTING()
ADD_EXECUTABLE(hiredis-test test.c)
TARGET_LINK_LIBRARIES(hiredis-test hiredis)
IF(ENABLE_SSL_TESTS)
ADD_DEFINITIONS(-DHIREDIS_TEST_SSL=1)
TARGET_LINK_LIBRARIES(hiredis-test hiredis_ssl)
ENDIF()
IF(ENABLE_ASYNC_TESTS)
ADD_DEFINITIONS(-DHIREDIS_TEST_ASYNC=1)
TARGET_LINK_LIBRARIES(hiredis-test event)
ENDIF()
ADD_TEST(NAME hiredis-test
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test.sh)
ENDIF()
# Add examples
IF(ENABLE_EXAMPLES)
ADD_SUBDIRECTORY(examples)
ENDIF(ENABLE_EXAMPLES)
+29
View File
@@ -0,0 +1,29 @@
Copyright (c) 2009-2011, Redis Ltd.
Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of Redis nor the names of its contributors may be used
to endorse or promote products derived from this software without specific
prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+354
View File
@@ -0,0 +1,354 @@
# Hiredis Makefile
# Copyright (C) 2010-2011 Redis Ltd.
# Copyright (C) 2010-2011 Pieter Noordhuis <pcnoordhuis at gmail dot com>
# This file is released under the BSD license, see the COPYING file
OBJ=alloc.o net.o hiredis.o sds.o async.o read.o sockcompat.o
EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib hiredis-example-push hiredis-example-poll
TESTS=hiredis-test
LIBNAME=libhiredis
PKGCONFNAME=hiredis.pc
HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}')
HIREDIS_MINOR=$(shell grep HIREDIS_MINOR hiredis.h | awk '{print $$3}')
HIREDIS_PATCH=$(shell grep HIREDIS_PATCH hiredis.h | awk '{print $$3}')
HIREDIS_SONAME=$(shell grep HIREDIS_SONAME hiredis.h | awk '{print $$3}')
# Installation related variables and target
PREFIX?=/usr/local
INCLUDE_PATH?=include/hiredis
LIBRARY_PATH?=lib
PKGCONF_PATH?=pkgconfig
INSTALL_INCLUDE_PATH= $(DESTDIR)$(PREFIX)/$(INCLUDE_PATH)
INSTALL_LIBRARY_PATH= $(DESTDIR)$(PREFIX)/$(LIBRARY_PATH)
INSTALL_PKGCONF_PATH= $(INSTALL_LIBRARY_PATH)/$(PKGCONF_PATH)
# redis-server configuration used for testing
REDIS_PORT=56379
REDIS_SERVER=redis-server
define REDIS_TEST_CONFIG
daemonize yes
pidfile /tmp/hiredis-test-redis.pid
port $(REDIS_PORT)
bind 127.0.0.1
unixsocket /tmp/hiredis-test-redis.sock
endef
export REDIS_TEST_CONFIG
# Fallback to gcc when $CC is not in $PATH.
CC:=$(shell sh -c 'type $${CC%% *} >/dev/null 2>/dev/null && echo $(CC) || echo gcc')
CXX:=$(shell sh -c 'type $${CXX%% *} >/dev/null 2>/dev/null && echo $(CXX) || echo g++')
OPTIMIZATION?=-O3
WARNINGS=-Wall -Wextra -Werror -Wstrict-prototypes -Wwrite-strings -Wno-missing-field-initializers
DEBUG_FLAGS?= -g -ggdb
REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CPPFLAGS) $(CFLAGS) $(WARNINGS) $(DEBUG_FLAGS) $(PLATFORM_FLAGS)
REAL_LDFLAGS=$(LDFLAGS)
DYLIBSUFFIX=so
STLIBSUFFIX=a
DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME)
DYLIB_MAJOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR)
DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX)
DYLIB_MAKE_CMD=$(CC) $(PLATFORM_FLAGS) -shared -Wl,-soname,$(DYLIB_MINOR_NAME)
STLIBNAME=$(LIBNAME).$(STLIBSUFFIX)
STLIB_MAKE_CMD=$(AR) rcs
#################### SSL variables start ####################
SSL_OBJ=ssl.o
SSL_LIBNAME=libhiredis_ssl
SSL_PKGCONFNAME=hiredis_ssl.pc
SSL_INSTALLNAME=install-ssl
SSL_DYLIB_MINOR_NAME=$(SSL_LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME)
SSL_DYLIB_MAJOR_NAME=$(SSL_LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR)
SSL_DYLIBNAME=$(SSL_LIBNAME).$(DYLIBSUFFIX)
SSL_STLIBNAME=$(SSL_LIBNAME).$(STLIBSUFFIX)
SSL_DYLIB_MAKE_CMD=$(CC) $(PLATFORM_FLAGS) -shared -Wl,-soname,$(SSL_DYLIB_MINOR_NAME)
USE_SSL?=0
ifeq ($(USE_SSL),1)
# This is required for test.c only
CFLAGS+=-DHIREDIS_TEST_SSL
EXAMPLES+=hiredis-example-ssl hiredis-example-libevent-ssl
SSL_STLIB=$(SSL_STLIBNAME)
SSL_DYLIB=$(SSL_DYLIBNAME)
SSL_PKGCONF=$(SSL_PKGCONFNAME)
SSL_INSTALL=$(SSL_INSTALLNAME)
else
SSL_STLIB=
SSL_DYLIB=
SSL_PKGCONF=
SSL_INSTALL=
endif
##################### SSL variables end #####################
# Platform-specific overrides
uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
# This is required for test.c only
ifeq ($(TEST_ASYNC),1)
export CFLAGS+=-DHIREDIS_TEST_ASYNC
endif
ifeq ($(USE_SSL),1)
ifndef OPENSSL_PREFIX
ifeq ($(uname_S),Darwin)
SEARCH_PATH1=/opt/homebrew/opt/openssl
SEARCH_PATH2=/usr/local/opt/openssl
ifneq ($(wildcard $(SEARCH_PATH1)),)
OPENSSL_PREFIX=$(SEARCH_PATH1)
else ifneq ($(wildcard $(SEARCH_PATH2)),)
OPENSSL_PREFIX=$(SEARCH_PATH2)
endif
endif
endif
ifdef OPENSSL_PREFIX
CFLAGS+=-I$(OPENSSL_PREFIX)/include
SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib
endif
SSL_LDFLAGS+=-lssl -lcrypto
endif
ifeq ($(uname_S),FreeBSD)
LDFLAGS+=-lm
IS_GCC=$(shell sh -c '$(CC) --version 2>/dev/null |egrep -i -c "gcc"')
ifeq ($(IS_GCC),1)
REAL_CFLAGS+=-pedantic
endif
else
REAL_CFLAGS+=-pedantic
endif
ifeq ($(uname_S),SunOS)
IS_SUN_CC=$(shell sh -c '$(CC) -V 2>&1 |egrep -i -c "sun|studio"')
ifeq ($(IS_SUN_CC),1)
SUN_SHARED_FLAG=-G
else
SUN_SHARED_FLAG=-shared
endif
REAL_LDFLAGS+= -ldl -lnsl -lsocket
DYLIB_MAKE_CMD=$(CC) $(SUN_SHARED_FLAG) -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS)
SSL_DYLIB_MAKE_CMD=$(CC) $(SUN_SHARED_FLAG) -o $(SSL_DYLIBNAME) -h $(SSL_DYLIB_MINOR_NAME) $(LDFLAGS) $(SSL_LDFLAGS)
endif
ifeq ($(uname_S),Darwin)
DYLIBSUFFIX=dylib
DYLIB_MINOR_NAME=$(LIBNAME).$(HIREDIS_SONAME).$(DYLIBSUFFIX)
DYLIB_MAKE_CMD=$(CC) -dynamiclib -Wl,-install_name,$(PREFIX)/$(LIBRARY_PATH)/$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS)
SSL_DYLIB_MAKE_CMD=$(CC) -dynamiclib -Wl,-install_name,$(PREFIX)/$(LIBRARY_PATH)/$(SSL_DYLIB_MINOR_NAME) -o $(SSL_DYLIBNAME) $(LDFLAGS) $(SSL_LDFLAGS)
DYLIB_PLUGIN=-Wl,-undefined -Wl,dynamic_lookup
endif
all: dynamic static hiredis-test pkgconfig
dynamic: $(DYLIBNAME) $(SSL_DYLIB)
static: $(STLIBNAME) $(SSL_STLIB)
pkgconfig: $(PKGCONFNAME) $(SSL_PKGCONF)
# Deps (use make dep to generate this)
alloc.o: alloc.c fmacros.h alloc.h
async.o: async.c fmacros.h alloc.h async.h hiredis.h read.h sds.h net.h dict.c dict.h win32.h async_private.h
dict.o: dict.c fmacros.h alloc.h dict.h
hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h alloc.h net.h async.h win32.h
net.o: net.c fmacros.h net.h hiredis.h read.h sds.h alloc.h sockcompat.h win32.h
read.o: read.c fmacros.h alloc.h read.h sds.h win32.h
sds.o: sds.c sds.h sdsalloc.h alloc.h
sockcompat.o: sockcompat.c sockcompat.h
test.o: test.c fmacros.h hiredis.h read.h sds.h alloc.h net.h sockcompat.h win32.h
$(DYLIBNAME): $(OBJ)
$(DYLIB_MAKE_CMD) -o $(DYLIBNAME) $(OBJ) $(REAL_LDFLAGS)
$(STLIBNAME): $(OBJ)
$(STLIB_MAKE_CMD) $(STLIBNAME) $(OBJ)
#################### SSL building rules start ####################
$(SSL_DYLIBNAME): $(SSL_OBJ)
$(SSL_DYLIB_MAKE_CMD) $(DYLIB_PLUGIN) -o $(SSL_DYLIBNAME) $(SSL_OBJ) $(REAL_LDFLAGS) $(LDFLAGS) $(SSL_LDFLAGS)
$(SSL_STLIBNAME): $(SSL_OBJ)
$(STLIB_MAKE_CMD) $(SSL_STLIBNAME) $(SSL_OBJ)
$(SSL_OBJ): ssl.c hiredis.h read.h sds.h alloc.h async.h win32.h async_private.h
#################### SSL building rules end ####################
# Binaries:
hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(REAL_LDFLAGS)
hiredis-example-libevent-ssl: examples/example-libevent-ssl.c adapters/libevent.h $(STLIBNAME) $(SSL_STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS)
hiredis-example-libev: examples/example-libev.c adapters/libev.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -lev $(STLIBNAME) $(REAL_LDFLAGS)
hiredis-example-libhv: examples/example-libhv.c adapters/libhv.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -lhv $(STLIBNAME) $(REAL_LDFLAGS)
hiredis-example-glib: examples/example-glib.c adapters/glib.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(shell pkg-config --cflags --libs glib-2.0) $(STLIBNAME) $(REAL_LDFLAGS)
hiredis-example-ivykis: examples/example-ivykis.c adapters/ivykis.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -livykis $(STLIBNAME) $(REAL_LDFLAGS)
hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) $(REAL_LDFLAGS)
hiredis-example-ssl: examples/example-ssl.c $(STLIBNAME) $(SSL_STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS)
hiredis-example-poll: examples/example-poll.c adapters/poll.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS)
ifndef AE_DIR
hiredis-example-ae:
@echo "Please specify AE_DIR (e.g. <redis repository>/src)"
@false
else
hiredis-example-ae: examples/example-ae.c adapters/ae.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(AE_DIR) $< $(AE_DIR)/ae.o $(AE_DIR)/zmalloc.o $(AE_DIR)/../deps/jemalloc/lib/libjemalloc.a -pthread $(STLIBNAME)
endif
ifndef LIBUV_DIR
# dynamic link libuv.so
hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) -I. -I$(LIBUV_DIR)/include $< -luv -lpthread -lrt $(STLIBNAME) $(REAL_LDFLAGS)
else
# use user provided static lib
hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) $(REAL_LDFLAGS)
endif
ifeq ($(and $(QT_MOC),$(QT_INCLUDE_DIR),$(QT_LIBRARY_DIR)),)
hiredis-example-qt:
@echo "Please specify QT_MOC, QT_INCLUDE_DIR AND QT_LIBRARY_DIR"
@false
else
hiredis-example-qt: examples/example-qt.cpp adapters/qt.h $(STLIBNAME)
$(QT_MOC) adapters/qt.h -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore | \
$(CXX) -x c++ -o qt-adapter-moc.o -c - $(REAL_CFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore
$(QT_MOC) examples/example-qt.h -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore | \
$(CXX) -x c++ -o qt-example-moc.o -c - $(REAL_CFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore
$(CXX) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore -L$(QT_LIBRARY_DIR) qt-adapter-moc.o qt-example-moc.o $< -pthread $(STLIBNAME) -lQtCore
endif
hiredis-example: examples/example.c $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS)
hiredis-example-push: examples/example-push.c $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS)
examples: $(EXAMPLES)
TEST_LIBS = $(STLIBNAME) $(SSL_STLIB)
TEST_LDFLAGS = $(SSL_LDFLAGS)
ifeq ($(USE_SSL),1)
TEST_LDFLAGS += -pthread
endif
ifeq ($(TEST_ASYNC),1)
TEST_LDFLAGS += -levent
endif
hiredis-test: test.o $(TEST_LIBS)
$(CC) -o $@ $(REAL_CFLAGS) -I. $^ $(REAL_LDFLAGS) $(TEST_LDFLAGS)
hiredis-%: %.o $(STLIBNAME)
$(CC) $(REAL_CFLAGS) -o $@ $< $(TEST_LIBS) $(REAL_LDFLAGS)
test: hiredis-test
./hiredis-test
check: hiredis-test
TEST_SSL=$(USE_SSL) ./test.sh
.c.o:
$(CC) -std=c99 -c $(REAL_CFLAGS) $<
clean:
rm -rf $(DYLIBNAME) $(STLIBNAME) $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov
dep:
$(CC) $(CPPFLAGS) $(CFLAGS) -MM *.c
INSTALL?= cp -pPR
$(PKGCONFNAME): hiredis.h
@echo "Generating $@ for pkgconfig..."
@echo prefix=$(PREFIX) > $@
@echo exec_prefix=\$${prefix} >> $@
@echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@
@echo includedir=$(PREFIX)/include >> $@
@echo pkgincludedir=$(PREFIX)/$(INCLUDE_PATH) >> $@
@echo >> $@
@echo Name: hiredis >> $@
@echo Description: Minimalistic C client library for Redis. >> $@
@echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@
@echo Libs: -L\$${libdir} -lhiredis >> $@
@echo Cflags: -I\$${pkgincludedir} -I\$${includedir} -D_FILE_OFFSET_BITS=64 >> $@
$(SSL_PKGCONFNAME): hiredis_ssl.h
@echo "Generating $@ for pkgconfig..."
@echo prefix=$(PREFIX) > $@
@echo exec_prefix=\$${prefix} >> $@
@echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@
@echo includedir=$(PREFIX)/include >> $@
@echo pkgincludedir=$(PREFIX)/$(INCLUDE_PATH) >> $@
@echo >> $@
@echo Name: hiredis_ssl >> $@
@echo Description: SSL Support for hiredis. >> $@
@echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@
@echo Requires: hiredis >> $@
@echo Libs: -L\$${libdir} -lhiredis_ssl >> $@
@echo Libs.private: -lssl -lcrypto >> $@
install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) $(SSL_INSTALL)
mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL_LIBRARY_PATH)
$(INSTALL) hiredis.h async.h read.h sds.h alloc.h sockcompat.h $(INSTALL_INCLUDE_PATH)
$(INSTALL) adapters/*.h $(INSTALL_INCLUDE_PATH)/adapters
$(INSTALL) $(DYLIBNAME) $(INSTALL_LIBRARY_PATH)/$(DYLIB_MINOR_NAME)
cd $(INSTALL_LIBRARY_PATH) && ln -sf $(DYLIB_MINOR_NAME) $(DYLIBNAME) && ln -sf $(DYLIB_MINOR_NAME) $(DYLIB_MAJOR_NAME)
$(INSTALL) $(STLIBNAME) $(INSTALL_LIBRARY_PATH)
mkdir -p $(INSTALL_PKGCONF_PATH)
$(INSTALL) $(PKGCONFNAME) $(INSTALL_PKGCONF_PATH)
install-ssl: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME)
mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_LIBRARY_PATH)
$(INSTALL) hiredis_ssl.h $(INSTALL_INCLUDE_PATH)
$(INSTALL) $(SSL_DYLIBNAME) $(INSTALL_LIBRARY_PATH)/$(SSL_DYLIB_MINOR_NAME)
cd $(INSTALL_LIBRARY_PATH) && ln -sf $(SSL_DYLIB_MINOR_NAME) $(SSL_DYLIBNAME) && ln -sf $(SSL_DYLIB_MINOR_NAME) $(SSL_DYLIB_MAJOR_NAME)
$(INSTALL) $(SSL_STLIBNAME) $(INSTALL_LIBRARY_PATH)
mkdir -p $(INSTALL_PKGCONF_PATH)
$(INSTALL) $(SSL_PKGCONFNAME) $(INSTALL_PKGCONF_PATH)
32bit:
@echo ""
@echo "WARNING: if this fails under Linux you probably need to install libc6-dev-i386"
@echo ""
$(MAKE) CFLAGS="-m32" LDFLAGS="-m32"
32bit-vars:
$(eval CFLAGS=-m32)
$(eval LDFLAGS=-m32)
gprof:
$(MAKE) CFLAGS="-pg" LDFLAGS="-pg"
gcov:
$(MAKE) CFLAGS+="-fprofile-arcs -ftest-coverage" LDFLAGS="-fprofile-arcs"
coverage: gcov
make check
mkdir -p tmp/lcov
lcov -d . -c --exclude '/usr*' -o tmp/lcov/hiredis.info
lcov -q -l tmp/lcov/hiredis.info
genhtml --legend -q -o tmp/lcov/report tmp/lcov/hiredis.info
noopt:
$(MAKE) OPTIMIZATION=""
.PHONY: all test check clean dep install 32bit 32bit-vars gprof gcov noopt
+821
View File
@@ -0,0 +1,821 @@
[![Build Status](https://github.com/redis/hiredis/actions/workflows/build.yml/badge.svg)](https://github.com/redis/hiredis/actions/workflows/build.yml)
**This Readme reflects the latest changed in the master branch. See [v1.0.0](https://github.com/redis/hiredis/tree/v1.0.0) for the Readme and documentation for the latest release ([API/ABI history](https://abi-laboratory.pro/?view=timeline&l=hiredis)).**
# HIREDIS
Hiredis is a minimalistic C client library for the [Redis](https://redis.io/) database.
It is minimalistic because it just adds minimal support for the protocol, but
at the same time it uses a high level printf-alike API in order to make it
much higher level than otherwise suggested by its minimal code base and the
lack of explicit bindings for every Redis command.
Apart from supporting sending commands and receiving replies, it comes with
a reply parser that is decoupled from the I/O layer. It
is a stream parser designed for easy reusability, which can for instance be used
in higher level language bindings for efficient reply parsing.
Hiredis only supports the binary-safe Redis protocol, so you can use it with any
Redis version >= 1.2.0.
The library comes with multiple APIs. There is the
*synchronous API*, the *asynchronous API* and the *reply parsing API*.
## Upgrading to `1.1.0`
Almost all users will simply need to recompile their applications against the newer version of hiredis.
**NOTE**: Hiredis can now return `nan` in addition to `-inf` and `inf` in a `REDIS_REPLY_DOUBLE`.
Applications that deal with `RESP3` doubles should make sure to account for this.
## Upgrading to `1.0.2`
<span style="color:red">NOTE: v1.0.1 erroneously bumped SONAME, which is why it is skipped here.</span>
Version 1.0.2 is simply 1.0.0 with a fix for [CVE-2021-32765](https://github.com/redis/hiredis/security/advisories/GHSA-hfm9-39pp-55p2). They are otherwise identical.
## Upgrading to `1.0.0`
Version 1.0.0 marks the first stable release of Hiredis.
It includes some minor breaking changes, mostly to make the exposed API more uniform and self-explanatory.
It also bundles the updated `sds` library, to sync up with upstream and Redis.
For code changes see the [Changelog](CHANGELOG.md).
_Note: As described below, a few member names have been changed but most applications should be able to upgrade with minor code changes and recompiling._
## IMPORTANT: Breaking changes from `0.14.1` -> `1.0.0`
* `redisContext` has two additional members (`free_privdata`, and `privctx`).
* `redisOptions.timeout` has been renamed to `redisOptions.connect_timeout`, and we've added `redisOptions.command_timeout`.
* `redisReplyObjectFunctions.createArray` now takes `size_t` instead of `int` for its length parameter.
## IMPORTANT: Breaking changes when upgrading from 0.13.x -> 0.14.x
Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now
protocol errors. This is consistent with the RESP specification. On 32-bit
platforms, the upper bound is lowered to `SIZE_MAX`.
Change `redisReply.len` to `size_t`, as it denotes the the size of a string
User code should compare this to `size_t` values as well. If it was used to
compare to other values, casting might be necessary or can be removed, if
casting was applied before.
## Upgrading from `<0.9.0`
Version 0.9.0 is a major overhaul of hiredis in every aspect. However, upgrading existing
code using hiredis should not be a big pain. The key thing to keep in mind when
upgrading is that hiredis >= 0.9.0 uses a `redisContext*` to keep state, in contrast to
the stateless 0.0.1 that only has a file descriptor to work with.
## Synchronous API
To consume the synchronous API, there are only a few function calls that need to be introduced:
```c
redisContext *redisConnect(const char *ip, int port);
void *redisCommand(redisContext *c, const char *format, ...);
void freeReplyObject(void *reply);
```
### Connecting
The function `redisConnect` is used to create a so-called `redisContext`. The
context is where Hiredis holds state for a connection. The `redisContext`
struct has an integer `err` field that is non-zero when the connection is in
an error state. The field `errstr` will contain a string with a description of
the error. More information on errors can be found in the **Errors** section.
After trying to connect to Redis using `redisConnect` you should
check the `err` field to see if establishing the connection was successful:
```c
redisContext *c = redisConnect("127.0.0.1", 6379);
if (c == NULL || c->err) {
if (c) {
printf("Error: %s\n", c->errstr);
// handle error
} else {
printf("Can't allocate redis context\n");
}
}
```
One can also use `redisConnectWithOptions` which takes a `redisOptions` argument
that can be configured with endpoint information as well as many different flags
to change how the `redisContext` will be configured.
```c
redisOptions opt = {0};
/* One can set the endpoint with one of our helper macros */
if (tcp) {
REDIS_OPTIONS_SET_TCP(&opt, "localhost", 6379);
} else {
REDIS_OPTIONS_SET_UNIX(&opt, "/tmp/redis.sock");
}
/* And privdata can be specified with another helper */
REDIS_OPTIONS_SET_PRIVDATA(&opt, myPrivData, myPrivDataDtor);
/* Finally various options may be set via the `options` member, as described below */
opt->options |= REDIS_OPT_PREFER_IPV4;
```
If a connection is lost, `int redisReconnect(redisContext *c)` can be used to restore the connection using the same endpoint and options as the given context.
### Configurable redisOptions flags
There are several flags you may set in the `redisOptions` struct to change default behavior. You can specify the flags via the `redisOptions->options` member.
| Flag | Description |
| --- | --- |
| REDIS\_OPT\_NONBLOCK | Tells hiredis to make a non-blocking connection. |
| REDIS\_OPT\_REUSEADDR | Tells hiredis to set the [SO_REUSEADDR](https://man7.org/linux/man-pages/man7/socket.7.html) socket option |
| REDIS\_OPT\_PREFER\_IPV4<br>REDIS\_OPT\_PREFER_IPV6<br>REDIS\_OPT\_PREFER\_IP\_UNSPEC | Informs hiredis to either prefer IPv4 or IPv6 when invoking [getaddrinfo](https://man7.org/linux/man-pages/man3/gai_strerror.3.html). `REDIS_OPT_PREFER_IP_UNSPEC` will cause hiredis to specify `AF_UNSPEC` in the getaddrinfo call, which means both IPv4 and IPv6 addresses will be searched simultaneously.<br>Hiredis prefers IPv4 by default. |
| REDIS\_OPT\_NO\_PUSH\_AUTOFREE | Tells hiredis to not install the default RESP3 PUSH handler (which just intercepts and frees the replies). This is useful in situations where you want to process these messages in-band. |
| REDIS\_OPT\_NOAUTOFREEREPLIES | **ASYNC**: tells hiredis not to automatically invoke `freeReplyObject` after executing the reply callback. |
| REDIS\_OPT\_NOAUTOFREE | **ASYNC**: Tells hiredis not to automatically free the `redisAsyncContext` on connection/communication failure, but only if the user makes an explicit call to `redisAsyncDisconnect` or `redisAsyncFree` |
*Note: A `redisContext` is not thread-safe.*
### Other configuration using socket options
The following socket options are applied directly to the underlying socket.
The values are not stored in the `redisContext`, so they are not automatically applied when reconnecting using `redisReconnect()`.
These functions return `REDIS_OK` on success.
On failure, `REDIS_ERR` is returned and the underlying connection is closed.
To configure these for an asyncronous context (see *Asynchronous API* below), use `ac->c` to get the redisContext out of an asyncRedisContext.
```C
int redisEnableKeepAlive(redisContext *c);
int redisEnableKeepAliveWithInterval(redisContext *c, int interval);
```
Enables TCP keepalive by setting the following socket options (with some variations depending on OS):
* `SO_KEEPALIVE`;
* `TCP_KEEPALIVE` or `TCP_KEEPIDLE`, value configurable using the `interval` parameter, default 15 seconds;
* `TCP_KEEPINTVL` set to 1/3 of `interval`;
* `TCP_KEEPCNT` set to 3.
```C
int redisSetTcpUserTimeout(redisContext *c, unsigned int timeout);
```
Set the `TCP_USER_TIMEOUT` Linux-specific socket option which is as described in the `tcp` man page:
> When the value is greater than 0, it specifies the maximum amount of time in milliseconds that trans mitted data may remain unacknowledged before TCP will forcibly close the corresponding connection and return ETIMEDOUT to the application.
> If the option value is specified as 0, TCP will use the system default.
### Sending commands
There are several ways to issue commands to Redis. The first that will be introduced is
`redisCommand`. This function takes a format similar to printf. In the simplest form,
it is used like this:
```c
reply = redisCommand(context, "SET foo bar");
```
The specifier `%s` interpolates a string in the command, and uses `strlen` to
determine the length of the string:
```c
reply = redisCommand(context, "SET foo %s", value);
```
When you need to pass binary safe strings in a command, the `%b` specifier can be
used. Together with a pointer to the string, it requires a `size_t` length argument
of the string:
```c
reply = redisCommand(context, "SET foo %b", value, (size_t) valuelen);
```
Internally, Hiredis splits the command in different arguments and will
convert it to the protocol used to communicate with Redis.
One or more spaces separates arguments, so you can use the specifiers
anywhere in an argument:
```c
reply = redisCommand(context, "SET key:%s %s", myid, value);
```
### Using replies
The return value of `redisCommand` holds a reply when the command was
successfully executed. When an error occurs, the return value is `NULL` and
the `err` field in the context will be set (see section on **Errors**).
Once an error is returned the context cannot be reused and you should set up
a new connection.
The standard replies that `redisCommand` are of the type `redisReply`. The
`type` field in the `redisReply` should be used to test what kind of reply
was received:
### RESP2
* **`REDIS_REPLY_STATUS`**:
* The command replied with a status reply. The status string can be accessed using `reply->str`.
The length of this string can be accessed using `reply->len`.
* **`REDIS_REPLY_ERROR`**:
* The command replied with an error. The error string can be accessed identical to `REDIS_REPLY_STATUS`.
* **`REDIS_REPLY_INTEGER`**:
* The command replied with an integer. The integer value can be accessed using the
`reply->integer` field of type `long long`.
* **`REDIS_REPLY_NIL`**:
* The command replied with a **nil** object. There is no data to access.
* **`REDIS_REPLY_STRING`**:
* A bulk (string) reply. The value of the reply can be accessed using `reply->str`.
The length of this string can be accessed using `reply->len`.
* **`REDIS_REPLY_ARRAY`**:
* A multi bulk reply. The number of elements in the multi bulk reply is stored in
`reply->elements`. Every element in the multi bulk reply is a `redisReply` object as well
and can be accessed via `reply->element[..index..]`.
Redis may reply with nested arrays but this is fully supported.
### RESP3
Hiredis also supports every new `RESP3` data type which are as follows. For more information about the protocol see the `RESP3` [specification.](https://github.com/antirez/RESP3/blob/master/spec.md)
* **`REDIS_REPLY_DOUBLE`**:
* The command replied with a double-precision floating point number.
The value is stored as a string in the `str` member, and can be converted with `strtod` or similar.
* **`REDIS_REPLY_BOOL`**:
* A boolean true/false reply.
The value is stored in the `integer` member and will be either `0` or `1`.
* **`REDIS_REPLY_MAP`**:
* An array with the added invariant that there will always be an even number of elements.
The MAP is functionally equivalent to `REDIS_REPLY_ARRAY` except for the previously mentioned invariant.
* **`REDIS_REPLY_SET`**:
* An array response where each entry is unique.
Like the MAP type, the data is identical to an array response except there are no duplicate values.
* **`REDIS_REPLY_PUSH`**:
* An array that can be generated spontaneously by Redis.
This array response will always contain at least two subelements. The first contains the type of `PUSH` message (e.g. `message`, or `invalidate`), and the second being a sub-array with the `PUSH` payload itself.
* **`REDIS_REPLY_ATTR`**:
* An array structurally identical to a `MAP` but intended as meta-data about a reply.
_As of Redis 6.0.6 this reply type is not used in Redis_
* **`REDIS_REPLY_BIGNUM`**:
* A string representing an arbitrarily large signed or unsigned integer value.
The number will be encoded as a string in the `str` member of `redisReply`.
* **`REDIS_REPLY_VERB`**:
* A verbatim string, intended to be presented to the user without modification.
The string payload is stored in the `str` member, and type data is stored in the `vtype` member (e.g. `txt` for raw text or `md` for markdown).
Replies should be freed using the `freeReplyObject()` function.
Note that this function will take care of freeing sub-reply objects
contained in arrays and nested arrays, so there is no need for the user to
free the sub replies (it is actually harmful and will corrupt the memory).
**Important:** the current version of hiredis (1.0.0) frees replies when the
asynchronous API is used. This means you should not call `freeReplyObject` when
you use this API. The reply is cleaned up by hiredis _after_ the callback
returns. We may introduce a flag to make this configurable in future versions of the library.
### Cleaning up
To disconnect and free the context the following function can be used:
```c
void redisFree(redisContext *c);
```
This function immediately closes the socket and then frees the allocations done in
creating the context.
### Sending commands (cont'd)
Together with `redisCommand`, the function `redisCommandArgv` can be used to issue commands.
It has the following prototype:
```c
void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
```
It takes the number of arguments `argc`, an array of strings `argv` and the lengths of the
arguments `argvlen`. For convenience, `argvlen` may be set to `NULL` and the function will
use `strlen(3)` on every argument to determine its length. Obviously, when any of the arguments
need to be binary safe, the entire array of lengths `argvlen` should be provided.
The return value has the same semantic as `redisCommand`.
### Pipelining
To explain how Hiredis supports pipelining in a blocking connection, there needs to be
understanding of the internal execution flow.
When any of the functions in the `redisCommand` family is called, Hiredis first formats the
command according to the Redis protocol. The formatted command is then put in the output buffer
of the context. This output buffer is dynamic, so it can hold any number of commands.
After the command is put in the output buffer, `redisGetReply` is called. This function has the
following two execution paths:
1. The input buffer is non-empty:
* Try to parse a single reply from the input buffer and return it
* If no reply could be parsed, continue at *2*
2. The input buffer is empty:
* Write the **entire** output buffer to the socket
* Read from the socket until a single reply could be parsed
The function `redisGetReply` is exported as part of the Hiredis API and can be used when a reply
is expected on the socket. To pipeline commands, the only thing that needs to be done is
filling up the output buffer. For this cause, two commands can be used that are identical
to the `redisCommand` family, apart from not returning a reply:
```c
void redisAppendCommand(redisContext *c, const char *format, ...);
void redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
```
After calling either function one or more times, `redisGetReply` can be used to receive the
subsequent replies. The return value for this function is either `REDIS_OK` or `REDIS_ERR`, where
the latter means an error occurred while reading a reply. Just as with the other commands,
the `err` field in the context can be used to find out what the cause of this error is.
The following examples shows a simple pipeline (resulting in only a single call to `write(2)` and
a single call to `read(2)`):
```c
redisReply *reply;
redisAppendCommand(context,"SET foo bar");
redisAppendCommand(context,"GET foo");
redisGetReply(context,(void**)&reply); // reply for SET
freeReplyObject(reply);
redisGetReply(context,(void**)&reply); // reply for GET
freeReplyObject(reply);
```
This API can also be used to implement a blocking subscriber:
```c
reply = redisCommand(context,"SUBSCRIBE foo");
freeReplyObject(reply);
while(redisGetReply(context,(void *)&reply) == REDIS_OK) {
// consume message
freeReplyObject(reply);
}
```
### Errors
When a function call is not successful, depending on the function either `NULL` or `REDIS_ERR` is
returned. The `err` field inside the context will be non-zero and set to one of the
following constants:
* **`REDIS_ERR_IO`**:
There was an I/O error while creating the connection, trying to write
to the socket or read from the socket. If you included `errno.h` in your
application, you can use the global `errno` variable to find out what is
wrong.
* **`REDIS_ERR_EOF`**:
The server closed the connection which resulted in an empty read.
* **`REDIS_ERR_PROTOCOL`**:
There was an error while parsing the protocol.
* **`REDIS_ERR_OTHER`**:
Any other error. Currently, it is only used when a specified hostname to connect
to cannot be resolved.
In every case, the `errstr` field in the context will be set to hold a string representation
of the error.
## Asynchronous API
Hiredis comes with an asynchronous API that works easily with any event library.
Examples are bundled that show using Hiredis with [libev](http://software.schmorp.de/pkg/libev.html)
and [libevent](http://monkey.org/~provos/libevent/).
### Connecting
The function `redisAsyncConnect` can be used to establish a non-blocking connection to
Redis. It returns a pointer to the newly created `redisAsyncContext` struct. The `err` field
should be checked after creation to see if there were errors creating the connection.
Because the connection that will be created is non-blocking, the kernel is not able to
instantly return if the specified host and port is able to accept a connection.
In case of error, it is the caller's responsibility to free the context using `redisAsyncFree()`
*Note: A `redisAsyncContext` is not thread-safe.*
An application function creating a connection might look like this:
```c
void appConnect(myAppData *appData)
{
redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
if (c->err) {
printf("Error: %s\n", c->errstr);
// handle error
redisAsyncFree(c);
c = NULL;
} else {
appData->context = c;
appData->connecting = 1;
c->data = appData; /* store application pointer for the callbacks */
redisAsyncSetConnectCallback(c, appOnConnect);
redisAsyncSetDisconnectCallback(c, appOnDisconnect);
}
}
```
The asynchronous context _should_ hold a *connect* callback function that is called when the connection
attempt completes, either successfully or with an error.
It _can_ also hold a *disconnect* callback function that is called when the
connection is disconnected (either because of an error or per user request). Both callbacks should
have the following prototype:
```c
void(const redisAsyncContext *c, int status);
```
On a *connect*, the `status` argument is set to `REDIS_OK` if the connection attempt succeeded. In this
case, the context is ready to accept commands. If it is called with `REDIS_ERR` then the
connection attempt failed. The `err` field in the context can be accessed to find out the cause of the error.
After a failed connection attempt, the context object is automatically freed by the library after calling
the connect callback. This may be a good point to create a new context and retry the connection.
On a disconnect, the `status` argument is set to `REDIS_OK` when disconnection was initiated by the
user, or `REDIS_ERR` when the disconnection was caused by an error. When it is `REDIS_ERR`, the `err`
field in the context can be accessed to find out the cause of the error.
The context object is always freed after the disconnect callback fired. When a reconnect is needed,
the disconnect callback is a good point to do so.
Setting the connect or disconnect callbacks can only be done once per context. For subsequent calls the
api will return `REDIS_ERR`. The function to set the callbacks have the following prototype:
```c
/* Alternatively you can use redisAsyncSetConnectCallbackNC which will be passed a non-const
redisAsyncContext* on invocation (e.g. allowing writes to the privdata member). */
int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn);
int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn);
```
`ac->data` may be used to pass user data to both callbacks. A typical implementation
might look something like this:
```c
void appOnConnect(redisAsyncContext *c, int status)
{
myAppData *appData = (myAppData*)c->data; /* get my application specific context*/
appData->connecting = 0;
if (status == REDIS_OK) {
appData->connected = 1;
} else {
appData->connected = 0;
appData->err = c->err;
appData->context = NULL; /* avoid stale pointer when callback returns */
}
appAttemptReconnect();
}
void appOnDisconnect(redisAsyncContext *c, int status)
{
myAppData *appData = (myAppData*)c->data; /* get my application specific context*/
appData->connected = 0;
appData->err = c->err;
appData->context = NULL; /* avoid stale pointer when callback returns */
if (status == REDIS_OK) {
appNotifyDisconnectCompleted(mydata);
} else {
appNotifyUnexpectedDisconnect(mydata);
appAttemptReconnect();
}
}
```
### Sending commands and their callbacks
In an asynchronous context, commands are automatically pipelined due to the nature of an event loop.
Therefore, unlike the synchronous API, there is only a single way to send commands.
Because commands are sent to Redis asynchronously, issuing a command requires a callback function
that is called when the reply is received. Reply callbacks should have the following prototype:
```c
void(redisAsyncContext *c, void *reply, void *privdata);
```
The `privdata` argument can be used to curry arbitrary data to the callback from the point where
the command is initially queued for execution.
The functions that can be used to issue commands in an asynchronous context are:
```c
int redisAsyncCommand(
redisAsyncContext *ac, redisCallbackFn *fn, void *privdata,
const char *format, ...);
int redisAsyncCommandArgv(
redisAsyncContext *ac, redisCallbackFn *fn, void *privdata,
int argc, const char **argv, const size_t *argvlen);
```
Both functions work like their blocking counterparts. The return value is `REDIS_OK` when the command
was successfully added to the output buffer and `REDIS_ERR` otherwise. Example: when the connection
is being disconnected per user-request, no new commands may be added to the output buffer and `REDIS_ERR` is
returned on calls to the `redisAsyncCommand` family.
If the reply for a command with a `NULL` callback is read, it is immediately freed. When the callback
for a command is non-`NULL`, the memory is freed immediately following the callback: the reply is only
valid for the duration of the callback.
All pending callbacks are called with a `NULL` reply when the context encountered an error.
For every command issued, with the exception of **SUBSCRIBE** and **PSUBSCRIBE**, the callback is
called exactly once. Even if the context object id disconnected or deleted, every pending callback
will be called with a `NULL` reply.
For **SUBSCRIBE** and **PSUBSCRIBE**, the callbacks may be called repeatedly until an `unsubscribe`
message arrives. This will be the last invocation of the callback. In case of error, the callbacks
may receive a final `NULL` reply instead.
### Disconnecting
An asynchronous connection can be terminated using:
```c
void redisAsyncDisconnect(redisAsyncContext *ac);
```
When this function is called, the connection is **not** immediately terminated. Instead, new
commands are no longer accepted and the connection is only terminated when all pending commands
have been written to the socket, their respective replies have been read and their respective
callbacks have been executed. After this, the disconnection callback is executed with the
`REDIS_OK` status and the context object is freed.
The connection can be forcefully disconnected using
```c
void redisAsyncFree(redisAsyncContext *ac);
```
In this case, nothing more is written to the socket, all pending callbacks are called with a `NULL`
reply and the disconnection callback is called with `REDIS_OK`, after which the context object
is freed.
### Hooking it up to event library *X*
There are a few hooks that need to be set on the context object after it is created.
See the `adapters/` directory for bindings to *libev* and *libevent*.
## Reply parsing API
Hiredis comes with a reply parsing API that makes it easy for writing higher
level language bindings.
The reply parsing API consists of the following functions:
```c
redisReader *redisReaderCreate(void);
void redisReaderFree(redisReader *reader);
int redisReaderFeed(redisReader *reader, const char *buf, size_t len);
int redisReaderGetReply(redisReader *reader, void **reply);
```
The same set of functions are used internally by hiredis when creating a
normal Redis context, the above API just exposes it to the user for a direct
usage.
### Usage
The function `redisReaderCreate` creates a `redisReader` structure that holds a
buffer with unparsed data and state for the protocol parser.
Incoming data -- most likely from a socket -- can be placed in the internal
buffer of the `redisReader` using `redisReaderFeed`. This function will make a
copy of the buffer pointed to by `buf` for `len` bytes. This data is parsed
when `redisReaderGetReply` is called. This function returns an integer status
and a reply object (as described above) via `void **reply`. The returned status
can be either `REDIS_OK` or `REDIS_ERR`, where the latter means something went
wrong (either a protocol error, or an out of memory error).
The parser limits the level of nesting for multi bulk payloads to 7. If the
multi bulk nesting level is higher than this, the parser returns an error.
### Customizing replies
The function `redisReaderGetReply` creates `redisReply` and makes the function
argument `reply` point to the created `redisReply` variable. For instance, if
the response of type `REDIS_REPLY_STATUS` then the `str` field of `redisReply`
will hold the status as a vanilla C string. However, the functions that are
responsible for creating instances of the `redisReply` can be customized by
setting the `fn` field on the `redisReader` struct. This should be done
immediately after creating the `redisReader`.
For example, [hiredis-rb](https://github.com/pietern/hiredis-rb/blob/master/ext/hiredis_ext/reader.c)
uses customized reply object functions to create Ruby objects.
### Reader max buffer
Both when using the Reader API directly or when using it indirectly via a
normal Redis context, the redisReader structure uses a buffer in order to
accumulate data from the server.
Usually this buffer is destroyed when it is empty and is larger than 16
KiB in order to avoid wasting memory in unused buffers
However when working with very big payloads destroying the buffer may slow
down performances considerably, so it is possible to modify the max size of
an idle buffer changing the value of the `maxbuf` field of the reader structure
to the desired value. The special value of 0 means that there is no maximum
value for an idle buffer, so the buffer will never get freed.
For instance if you have a normal Redis context you can set the maximum idle
buffer to zero (unlimited) just with:
```c
context->reader->maxbuf = 0;
```
This should be done only in order to maximize performances when working with
large payloads. The context should be set back to `REDIS_READER_MAX_BUF` again
as soon as possible in order to prevent allocation of useless memory.
### Reader max array elements
By default the hiredis reply parser sets the maximum number of multi-bulk elements
to 2^32 - 1 or 4,294,967,295 entries. If you need to process multi-bulk replies
with more than this many elements you can set the value higher or to zero, meaning
unlimited with:
```c
context->reader->maxelements = 0;
```
## SSL/TLS Support
### Building
SSL/TLS support is not built by default and requires an explicit flag:
make USE_SSL=1
This requires OpenSSL development package (e.g. including header files to be
available.
When enabled, SSL/TLS support is built into extra `libhiredis_ssl.a` and
`libhiredis_ssl.so` static/dynamic libraries. This leaves the original libraries
unaffected so no additional dependencies are introduced.
### Using it
First, you'll need to make sure you include the SSL header file:
```c
#include <hiredis/hiredis.h>
#include <hiredis/hiredis_ssl.h>
```
You will also need to link against `libhiredis_ssl`, **in addition** to
`libhiredis` and add `-lssl -lcrypto` to satisfy its dependencies.
Hiredis implements SSL/TLS on top of its normal `redisContext` or
`redisAsyncContext`, so you will need to establish a connection first and then
initiate an SSL/TLS handshake.
#### Hiredis OpenSSL Wrappers
Before Hiredis can negotiate an SSL/TLS connection, it is necessary to
initialize OpenSSL and create a context. You can do that in two ways:
1. Work directly with the OpenSSL API to initialize the library's global context
and create `SSL_CTX *` and `SSL *` contexts. With an `SSL *` object you can
call `redisInitiateSSL()`.
2. Work with a set of Hiredis-provided wrappers around OpenSSL, create a
`redisSSLContext` object to hold configuration and use
`redisInitiateSSLWithContext()` to initiate the SSL/TLS handshake.
```c
/* An Hiredis SSL context. It holds SSL configuration and can be reused across
* many contexts.
*/
redisSSLContext *ssl_context;
/* An error variable to indicate what went wrong, if the context fails to
* initialize.
*/
redisSSLContextError ssl_error = REDIS_SSL_CTX_NONE;
/* Initialize global OpenSSL state.
*
* You should call this only once when your app initializes, and only if
* you don't explicitly or implicitly initialize OpenSSL it elsewhere.
*/
redisInitOpenSSL();
/* Create SSL context */
ssl_context = redisCreateSSLContext(
"cacertbundle.crt", /* File name of trusted CA/ca bundle file, optional */
"/path/to/certs", /* Path of trusted certificates, optional */
"client_cert.pem", /* File name of client certificate file, optional */
"client_key.pem", /* File name of client private key, optional */
"redis.mydomain.com", /* Server name to request (SNI), optional */
&ssl_error);
if(ssl_context == NULL || ssl_error != REDIS_SSL_CTX_NONE) {
/* Handle error and abort... */
/* e.g.
printf("SSL error: %s\n",
(ssl_error != REDIS_SSL_CTX_NONE) ?
redisSSLContextGetError(ssl_error) : "Unknown error");
// Abort
*/
}
/* Create Redis context and establish connection */
c = redisConnect("localhost", 6443);
if (c == NULL || c->err) {
/* Handle error and abort... */
}
/* Negotiate SSL/TLS */
if (redisInitiateSSLWithContext(c, ssl_context) != REDIS_OK) {
/* Handle error, in c->err / c->errstr */
}
```
## RESP3 PUSH replies
Redis 6.0 introduced PUSH replies with the reply-type `>`. These messages are generated spontaneously and can arrive at any time, so must be handled using callbacks.
### Default behavior
Hiredis installs handlers on `redisContext` and `redisAsyncContext` by default, which will intercept and free any PUSH replies detected. This means existing code will work as-is after upgrading to Redis 6 and switching to `RESP3`.
### Custom PUSH handler prototypes
The callback prototypes differ between `redisContext` and `redisAsyncContext`.
#### redisContext
```c
void my_push_handler(void *privdata, void *reply) {
/* Handle the reply */
/* Note: We need to free the reply in our custom handler for
blocking contexts. This lets us keep the reply if
we want. */
freeReplyObject(reply);
}
```
#### redisAsyncContext
```c
void my_async_push_handler(redisAsyncContext *ac, void *reply) {
/* Handle the reply */
/* Note: Because async hiredis always frees replies, you should
not call freeReplyObject in an async push callback. */
}
```
### Installing a custom handler
There are two ways to set your own PUSH handlers.
1. Set `push_cb` or `async_push_cb` in the `redisOptions` struct and connect with `redisConnectWithOptions` or `redisAsyncConnectWithOptions`.
```c
redisOptions = {0};
REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379);
options->push_cb = my_push_handler;
redisContext *context = redisConnectWithOptions(&options);
```
2. Call `redisSetPushCallback` or `redisAsyncSetPushCallback` on a connected context.
```c
redisContext *context = redisConnect("127.0.0.1", 6379);
redisSetPushCallback(context, my_push_handler);
```
_Note `redisSetPushCallback` and `redisAsyncSetPushCallback` both return any currently configured handler, making it easy to override and then return to the old value._
### Specifying no handler
If you have a unique use-case where you don't want hiredis to automatically intercept and free PUSH replies, you will want to configure no handler at all. This can be done in two ways.
1. Set the `REDIS_OPT_NO_PUSH_AUTOFREE` flag in `redisOptions` and leave the callback function pointer `NULL`.
```c
redisOptions = {0};
REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379);
options->options |= REDIS_OPT_NO_PUSH_AUTOFREE;
redisContext *context = redisConnectWithOptions(&options);
```
3. Call `redisSetPushCallback` with `NULL` once connected.
```c
redisContext *context = redisConnect("127.0.0.1", 6379);
redisSetPushCallback(context, NULL);
```
_Note: With no handler configured, calls to `redisCommand` may generate more than one reply, so this strategy is only applicable when there's some kind of blocking `redisGetReply()` loop (e.g. `MONITOR` or `SUBSCRIBE` workloads)._
## Allocator injection
Hiredis uses a pass-thru structure of function pointers defined in [alloc.h](https://github.com/redis/hiredis/blob/f5d25850/alloc.h#L41) that contain the currently configured allocation and deallocation functions. By default they just point to libc (`malloc`, `calloc`, `realloc`, etc).
### Overriding
One can override the allocators like so:
```c
hiredisAllocFuncs myfuncs = {
.mallocFn = my_malloc,
.callocFn = my_calloc,
.reallocFn = my_realloc,
.strdupFn = my_strdup,
.freeFn = my_free,
};
// Override allocators (function returns current allocators if needed)
hiredisAllocFuncs orig = hiredisSetAllocators(&myfuncs);
```
To reset the allocators to their default libc function simply call:
```c
hiredisResetAllocators();
```
## AUTHORS
Salvatore Sanfilippo (antirez at gmail),\
Pieter Noordhuis (pcnoordhuis at gmail)\
Michael Grunder (michael dot grunder at gmail)
_Hiredis is released under the BSD license._
@@ -28,95 +28,89 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef KV_ADAPTERS_AE_H
#define KV_ADAPTERS_AE_H
#include "../async.h"
#include "../cluster.h"
#include "../kv.h"
#include <ae.h>
#ifndef __HIREDIS_AE_H__
#define __HIREDIS_AE_H__
#include <sys/types.h>
#include <ae.h>
#include "../hiredis.h"
#include "../async.h"
typedef struct kvAeEvents {
kvAsyncContext *context;
typedef struct redisAeEvents {
redisAsyncContext *context;
aeEventLoop *loop;
int fd;
int reading, writing;
} kvAeEvents;
} redisAeEvents;
static void kvAeReadEvent(aeEventLoop *el, int fd, void *privdata, int mask) {
((void)el);
((void)fd);
((void)mask);
static void redisAeReadEvent(aeEventLoop *el, int fd, void *privdata, int mask) {
((void)el); ((void)fd); ((void)mask);
kvAeEvents *e = (kvAeEvents *)privdata;
kvAsyncHandleRead(e->context);
redisAeEvents *e = (redisAeEvents*)privdata;
redisAsyncHandleRead(e->context);
}
static void kvAeWriteEvent(aeEventLoop *el, int fd, void *privdata, int mask) {
((void)el);
((void)fd);
((void)mask);
static void redisAeWriteEvent(aeEventLoop *el, int fd, void *privdata, int mask) {
((void)el); ((void)fd); ((void)mask);
kvAeEvents *e = (kvAeEvents *)privdata;
kvAsyncHandleWrite(e->context);
redisAeEvents *e = (redisAeEvents*)privdata;
redisAsyncHandleWrite(e->context);
}
static void kvAeAddRead(void *privdata) {
kvAeEvents *e = (kvAeEvents *)privdata;
static void redisAeAddRead(void *privdata) {
redisAeEvents *e = (redisAeEvents*)privdata;
aeEventLoop *loop = e->loop;
if (!e->reading) {
e->reading = 1;
aeCreateFileEvent(loop, e->fd, AE_READABLE, kvAeReadEvent, e);
aeCreateFileEvent(loop,e->fd,AE_READABLE,redisAeReadEvent,e);
}
}
static void kvAeDelRead(void *privdata) {
kvAeEvents *e = (kvAeEvents *)privdata;
static void redisAeDelRead(void *privdata) {
redisAeEvents *e = (redisAeEvents*)privdata;
aeEventLoop *loop = e->loop;
if (e->reading) {
e->reading = 0;
aeDeleteFileEvent(loop, e->fd, AE_READABLE);
aeDeleteFileEvent(loop,e->fd,AE_READABLE);
}
}
static void kvAeAddWrite(void *privdata) {
kvAeEvents *e = (kvAeEvents *)privdata;
static void redisAeAddWrite(void *privdata) {
redisAeEvents *e = (redisAeEvents*)privdata;
aeEventLoop *loop = e->loop;
if (!e->writing) {
e->writing = 1;
aeCreateFileEvent(loop, e->fd, AE_WRITABLE, kvAeWriteEvent, e);
aeCreateFileEvent(loop,e->fd,AE_WRITABLE,redisAeWriteEvent,e);
}
}
static void kvAeDelWrite(void *privdata) {
kvAeEvents *e = (kvAeEvents *)privdata;
static void redisAeDelWrite(void *privdata) {
redisAeEvents *e = (redisAeEvents*)privdata;
aeEventLoop *loop = e->loop;
if (e->writing) {
e->writing = 0;
aeDeleteFileEvent(loop, e->fd, AE_WRITABLE);
aeDeleteFileEvent(loop,e->fd,AE_WRITABLE);
}
}
static void kvAeCleanup(void *privdata) {
kvAeEvents *e = (kvAeEvents *)privdata;
kvAeDelRead(privdata);
kvAeDelWrite(privdata);
vk_free(e);
static void redisAeCleanup(void *privdata) {
redisAeEvents *e = (redisAeEvents*)privdata;
redisAeDelRead(privdata);
redisAeDelWrite(privdata);
hi_free(e);
}
static int kvAeAttach(aeEventLoop *loop, kvAsyncContext *ac) {
kvContext *c = &(ac->c);
kvAeEvents *e;
static int redisAeAttach(aeEventLoop *loop, redisAsyncContext *ac) {
redisContext *c = &(ac->c);
redisAeEvents *e;
/* Nothing should be attached when something is already attached */
if (ac->ev.data != NULL)
return KV_ERR;
return REDIS_ERR;
/* Create container for context and r/w events */
e = (kvAeEvents *)vk_malloc(sizeof(*e));
e = (redisAeEvents*)hi_malloc(sizeof(*e));
if (e == NULL)
return KV_ERR;
return REDIS_ERR;
e->context = ac;
e->loop = loop;
@@ -124,31 +118,13 @@ static int kvAeAttach(aeEventLoop *loop, kvAsyncContext *ac) {
e->reading = e->writing = 0;
/* Register functions to start/stop listening for events */
ac->ev.addRead = kvAeAddRead;
ac->ev.delRead = kvAeDelRead;
ac->ev.addWrite = kvAeAddWrite;
ac->ev.delWrite = kvAeDelWrite;
ac->ev.cleanup = kvAeCleanup;
ac->ev.addRead = redisAeAddRead;
ac->ev.delRead = redisAeDelRead;
ac->ev.addWrite = redisAeAddWrite;
ac->ev.delWrite = redisAeDelWrite;
ac->ev.cleanup = redisAeCleanup;
ac->ev.data = e;
return KV_OK;
return REDIS_OK;
}
/* Internal adapter function with correct function signature. */
static int kvAeAttachAdapter(kvAsyncContext *ac, void *loop) {
return kvAeAttach((aeEventLoop *)loop, ac);
}
KV_UNUSED
static int kvClusterOptionsUseAe(kvClusterOptions *options,
aeEventLoop *loop) {
if (options == NULL || loop == NULL) {
return KV_ERR;
}
options->attach_fn = kvAeAttachAdapter;
options->attach_data = loop;
return KV_OK;
}
#endif /* KV_ADAPTERS_AE_H */
#endif
+156
View File
@@ -0,0 +1,156 @@
#ifndef __HIREDIS_GLIB_H__
#define __HIREDIS_GLIB_H__
#include <glib.h>
#include "../hiredis.h"
#include "../async.h"
typedef struct
{
GSource source;
redisAsyncContext *ac;
GPollFD poll_fd;
} RedisSource;
static void
redis_source_add_read (gpointer data)
{
RedisSource *source = (RedisSource *)data;
g_return_if_fail(source);
source->poll_fd.events |= G_IO_IN;
g_main_context_wakeup(g_source_get_context((GSource *)data));
}
static void
redis_source_del_read (gpointer data)
{
RedisSource *source = (RedisSource *)data;
g_return_if_fail(source);
source->poll_fd.events &= ~G_IO_IN;
g_main_context_wakeup(g_source_get_context((GSource *)data));
}
static void
redis_source_add_write (gpointer data)
{
RedisSource *source = (RedisSource *)data;
g_return_if_fail(source);
source->poll_fd.events |= G_IO_OUT;
g_main_context_wakeup(g_source_get_context((GSource *)data));
}
static void
redis_source_del_write (gpointer data)
{
RedisSource *source = (RedisSource *)data;
g_return_if_fail(source);
source->poll_fd.events &= ~G_IO_OUT;
g_main_context_wakeup(g_source_get_context((GSource *)data));
}
static void
redis_source_cleanup (gpointer data)
{
RedisSource *source = (RedisSource *)data;
g_return_if_fail(source);
redis_source_del_read(source);
redis_source_del_write(source);
/*
* It is not our responsibility to remove ourself from the
* current main loop. However, we will remove the GPollFD.
*/
if (source->poll_fd.fd >= 0) {
g_source_remove_poll((GSource *)data, &source->poll_fd);
source->poll_fd.fd = -1;
}
}
static gboolean
redis_source_prepare (GSource *source,
gint *timeout_)
{
RedisSource *redis = (RedisSource *)source;
*timeout_ = -1;
return !!(redis->poll_fd.events & redis->poll_fd.revents);
}
static gboolean
redis_source_check (GSource *source)
{
RedisSource *redis = (RedisSource *)source;
return !!(redis->poll_fd.events & redis->poll_fd.revents);
}
static gboolean
redis_source_dispatch (GSource *source,
GSourceFunc callback,
gpointer user_data)
{
RedisSource *redis = (RedisSource *)source;
if ((redis->poll_fd.revents & G_IO_OUT)) {
redisAsyncHandleWrite(redis->ac);
redis->poll_fd.revents &= ~G_IO_OUT;
}
if ((redis->poll_fd.revents & G_IO_IN)) {
redisAsyncHandleRead(redis->ac);
redis->poll_fd.revents &= ~G_IO_IN;
}
if (callback) {
return callback(user_data);
}
return TRUE;
}
static void
redis_source_finalize (GSource *source)
{
RedisSource *redis = (RedisSource *)source;
if (redis->poll_fd.fd >= 0) {
g_source_remove_poll(source, &redis->poll_fd);
redis->poll_fd.fd = -1;
}
}
static GSource *
redis_source_new (redisAsyncContext *ac)
{
static GSourceFuncs source_funcs = {
.prepare = redis_source_prepare,
.check = redis_source_check,
.dispatch = redis_source_dispatch,
.finalize = redis_source_finalize,
};
redisContext *c = &ac->c;
RedisSource *source;
g_return_val_if_fail(ac != NULL, NULL);
source = (RedisSource *)g_source_new(&source_funcs, sizeof *source);
if (source == NULL)
return NULL;
source->ac = ac;
source->poll_fd.fd = c->fd;
source->poll_fd.events = 0;
source->poll_fd.revents = 0;
g_source_add_poll((GSource *)source, &source->poll_fd);
ac->ev.addRead = redis_source_add_read;
ac->ev.delRead = redis_source_del_read;
ac->ev.addWrite = redis_source_add_write;
ac->ev.delWrite = redis_source_del_write;
ac->ev.cleanup = redis_source_cleanup;
ac->ev.data = source;
return (GSource *)source;
}
#endif /* __HIREDIS_GLIB_H__ */
+84
View File
@@ -0,0 +1,84 @@
#ifndef __HIREDIS_IVYKIS_H__
#define __HIREDIS_IVYKIS_H__
#include <iv.h>
#include "../hiredis.h"
#include "../async.h"
typedef struct redisIvykisEvents {
redisAsyncContext *context;
struct iv_fd fd;
} redisIvykisEvents;
static void redisIvykisReadEvent(void *arg) {
redisAsyncContext *context = (redisAsyncContext *)arg;
redisAsyncHandleRead(context);
}
static void redisIvykisWriteEvent(void *arg) {
redisAsyncContext *context = (redisAsyncContext *)arg;
redisAsyncHandleWrite(context);
}
static void redisIvykisAddRead(void *privdata) {
redisIvykisEvents *e = (redisIvykisEvents*)privdata;
iv_fd_set_handler_in(&e->fd, redisIvykisReadEvent);
}
static void redisIvykisDelRead(void *privdata) {
redisIvykisEvents *e = (redisIvykisEvents*)privdata;
iv_fd_set_handler_in(&e->fd, NULL);
}
static void redisIvykisAddWrite(void *privdata) {
redisIvykisEvents *e = (redisIvykisEvents*)privdata;
iv_fd_set_handler_out(&e->fd, redisIvykisWriteEvent);
}
static void redisIvykisDelWrite(void *privdata) {
redisIvykisEvents *e = (redisIvykisEvents*)privdata;
iv_fd_set_handler_out(&e->fd, NULL);
}
static void redisIvykisCleanup(void *privdata) {
redisIvykisEvents *e = (redisIvykisEvents*)privdata;
iv_fd_unregister(&e->fd);
hi_free(e);
}
static int redisIvykisAttach(redisAsyncContext *ac) {
redisContext *c = &(ac->c);
redisIvykisEvents *e;
/* Nothing should be attached when something is already attached */
if (ac->ev.data != NULL)
return REDIS_ERR;
/* Create container for context and r/w events */
e = (redisIvykisEvents*)hi_malloc(sizeof(*e));
if (e == NULL)
return REDIS_ERR;
e->context = ac;
/* Register functions to start/stop listening for events */
ac->ev.addRead = redisIvykisAddRead;
ac->ev.delRead = redisIvykisDelRead;
ac->ev.addWrite = redisIvykisAddWrite;
ac->ev.delWrite = redisIvykisDelWrite;
ac->ev.cleanup = redisIvykisCleanup;
ac->ev.data = e;
/* Initialize and install read/write events */
IV_FD_INIT(&e->fd);
e->fd.fd = c->fd;
e->fd.handler_in = redisIvykisReadEvent;
e->fd.handler_out = redisIvykisWriteEvent;
e->fd.handler_err = NULL;
e->fd.cookie = e->context;
iv_fd_register(&e->fd);
return REDIS_OK;
}
#endif
@@ -28,140 +28,138 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef KV_ADAPTERS_LIBEV_H
#define KV_ADAPTERS_LIBEV_H
#include "../async.h"
#include "../cluster.h"
#include "../kv.h"
#include <ev.h>
#ifndef __HIREDIS_LIBEV_H__
#define __HIREDIS_LIBEV_H__
#include <stdlib.h>
#include <sys/types.h>
#include <ev.h>
#include "../hiredis.h"
#include "../async.h"
typedef struct kvLibevEvents {
kvAsyncContext *context;
typedef struct redisLibevEvents {
redisAsyncContext *context;
struct ev_loop *loop;
int reading, writing;
ev_io rev, wev;
ev_timer timer;
} kvLibevEvents;
} redisLibevEvents;
static void kvLibevReadEvent(EV_P_ ev_io *watcher, int revents) {
static void redisLibevReadEvent(EV_P_ ev_io *watcher, int revents) {
#if EV_MULTIPLICITY
((void)EV_A);
#endif
((void)revents);
kvLibevEvents *e = (kvLibevEvents *)watcher->data;
kvAsyncHandleRead(e->context);
redisLibevEvents *e = (redisLibevEvents*)watcher->data;
redisAsyncHandleRead(e->context);
}
static void kvLibevWriteEvent(EV_P_ ev_io *watcher, int revents) {
static void redisLibevWriteEvent(EV_P_ ev_io *watcher, int revents) {
#if EV_MULTIPLICITY
((void)EV_A);
#endif
((void)revents);
kvLibevEvents *e = (kvLibevEvents *)watcher->data;
kvAsyncHandleWrite(e->context);
redisLibevEvents *e = (redisLibevEvents*)watcher->data;
redisAsyncHandleWrite(e->context);
}
static void kvLibevAddRead(void *privdata) {
kvLibevEvents *e = (kvLibevEvents *)privdata;
static void redisLibevAddRead(void *privdata) {
redisLibevEvents *e = (redisLibevEvents*)privdata;
#if EV_MULTIPLICITY
struct ev_loop *loop = e->loop;
#endif
if (!e->reading) {
e->reading = 1;
ev_io_start(EV_A_ & e->rev);
ev_io_start(EV_A_ &e->rev);
}
}
static void kvLibevDelRead(void *privdata) {
kvLibevEvents *e = (kvLibevEvents *)privdata;
static void redisLibevDelRead(void *privdata) {
redisLibevEvents *e = (redisLibevEvents*)privdata;
#if EV_MULTIPLICITY
struct ev_loop *loop = e->loop;
#endif
if (e->reading) {
e->reading = 0;
ev_io_stop(EV_A_ & e->rev);
ev_io_stop(EV_A_ &e->rev);
}
}
static void kvLibevAddWrite(void *privdata) {
kvLibevEvents *e = (kvLibevEvents *)privdata;
static void redisLibevAddWrite(void *privdata) {
redisLibevEvents *e = (redisLibevEvents*)privdata;
#if EV_MULTIPLICITY
struct ev_loop *loop = e->loop;
#endif
if (!e->writing) {
e->writing = 1;
ev_io_start(EV_A_ & e->wev);
ev_io_start(EV_A_ &e->wev);
}
}
static void kvLibevDelWrite(void *privdata) {
kvLibevEvents *e = (kvLibevEvents *)privdata;
static void redisLibevDelWrite(void *privdata) {
redisLibevEvents *e = (redisLibevEvents*)privdata;
#if EV_MULTIPLICITY
struct ev_loop *loop = e->loop;
#endif
if (e->writing) {
e->writing = 0;
ev_io_stop(EV_A_ & e->wev);
ev_io_stop(EV_A_ &e->wev);
}
}
static void kvLibevStopTimer(void *privdata) {
kvLibevEvents *e = (kvLibevEvents *)privdata;
static void redisLibevStopTimer(void *privdata) {
redisLibevEvents *e = (redisLibevEvents*)privdata;
#if EV_MULTIPLICITY
struct ev_loop *loop = e->loop;
#endif
ev_timer_stop(EV_A_ & e->timer);
ev_timer_stop(EV_A_ &e->timer);
}
static void kvLibevCleanup(void *privdata) {
kvLibevEvents *e = (kvLibevEvents *)privdata;
kvLibevDelRead(privdata);
kvLibevDelWrite(privdata);
kvLibevStopTimer(privdata);
vk_free(e);
static void redisLibevCleanup(void *privdata) {
redisLibevEvents *e = (redisLibevEvents*)privdata;
redisLibevDelRead(privdata);
redisLibevDelWrite(privdata);
redisLibevStopTimer(privdata);
hi_free(e);
}
static void kvLibevTimeout(EV_P_ ev_timer *timer, int revents) {
static void redisLibevTimeout(EV_P_ ev_timer *timer, int revents) {
#if EV_MULTIPLICITY
((void)EV_A);
#endif
((void)revents);
kvLibevEvents *e = (kvLibevEvents *)timer->data;
kvAsyncHandleTimeout(e->context);
redisLibevEvents *e = (redisLibevEvents*)timer->data;
redisAsyncHandleTimeout(e->context);
}
static void kvLibevSetTimeout(void *privdata, struct timeval tv) {
kvLibevEvents *e = (kvLibevEvents *)privdata;
static void redisLibevSetTimeout(void *privdata, struct timeval tv) {
redisLibevEvents *e = (redisLibevEvents*)privdata;
#if EV_MULTIPLICITY
struct ev_loop *loop = e->loop;
#endif
if (!ev_is_active(&e->timer)) {
ev_init(&e->timer, kvLibevTimeout);
ev_init(&e->timer, redisLibevTimeout);
e->timer.data = e;
}
e->timer.repeat = tv.tv_sec + tv.tv_usec / 1000000.00;
ev_timer_again(EV_A_ & e->timer);
ev_timer_again(EV_A_ &e->timer);
}
static int kvLibevAttach(EV_P_ kvAsyncContext *ac) {
kvContext *c = &(ac->c);
kvLibevEvents *e;
static int redisLibevAttach(EV_P_ redisAsyncContext *ac) {
redisContext *c = &(ac->c);
redisLibevEvents *e;
/* Nothing should be attached when something is already attached */
if (ac->ev.data != NULL)
return KV_ERR;
return REDIS_ERR;
/* Create container for context and r/w events */
e = (kvLibevEvents *)vk_calloc(1, sizeof(*e));
e = (redisLibevEvents*)hi_calloc(1, sizeof(*e));
if (e == NULL)
return KV_ERR;
return REDIS_ERR;
e->context = ac;
#if EV_MULTIPLICITY
@@ -173,35 +171,18 @@ static int kvLibevAttach(EV_P_ kvAsyncContext *ac) {
e->wev.data = e;
/* Register functions to start/stop listening for events */
ac->ev.addRead = kvLibevAddRead;
ac->ev.delRead = kvLibevDelRead;
ac->ev.addWrite = kvLibevAddWrite;
ac->ev.delWrite = kvLibevDelWrite;
ac->ev.cleanup = kvLibevCleanup;
ac->ev.scheduleTimer = kvLibevSetTimeout;
ac->ev.addRead = redisLibevAddRead;
ac->ev.delRead = redisLibevDelRead;
ac->ev.addWrite = redisLibevAddWrite;
ac->ev.delWrite = redisLibevDelWrite;
ac->ev.cleanup = redisLibevCleanup;
ac->ev.scheduleTimer = redisLibevSetTimeout;
ac->ev.data = e;
/* Initialize read/write events */
ev_io_init(&e->rev, kvLibevReadEvent, c->fd, EV_READ);
ev_io_init(&e->wev, kvLibevWriteEvent, c->fd, EV_WRITE);
return KV_OK;
ev_io_init(&e->rev,redisLibevReadEvent,c->fd,EV_READ);
ev_io_init(&e->wev,redisLibevWriteEvent,c->fd,EV_WRITE);
return REDIS_OK;
}
/* Internal adapter function with correct function signature. */
static int kvLibevAttachAdapter(kvAsyncContext *ac, void *loop) {
return kvLibevAttach((struct ev_loop *)loop, ac);
}
KV_UNUSED
static int kvClusterOptionsUseLibev(kvClusterOptions *options,
struct ev_loop *loop) {
if (options == NULL || loop == NULL) {
return KV_ERR;
}
options->attach_fn = kvLibevAttachAdapter;
options->attach_data = loop;
return KV_OK;
}
#endif /* KV_ADAPTERS_LIBEV_H */
#endif
+175
View File
@@ -0,0 +1,175 @@
/*
* Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __HIREDIS_LIBEVENT_H__
#define __HIREDIS_LIBEVENT_H__
#include <event2/event.h>
#include "../hiredis.h"
#include "../async.h"
#define REDIS_LIBEVENT_DELETED 0x01
#define REDIS_LIBEVENT_ENTERED 0x02
typedef struct redisLibeventEvents {
redisAsyncContext *context;
struct event *ev;
struct event_base *base;
struct timeval tv;
short flags;
short state;
} redisLibeventEvents;
static void redisLibeventDestroy(redisLibeventEvents *e) {
hi_free(e);
}
static void redisLibeventHandler(evutil_socket_t fd, short event, void *arg) {
((void)fd);
redisLibeventEvents *e = (redisLibeventEvents*)arg;
e->state |= REDIS_LIBEVENT_ENTERED;
#define CHECK_DELETED() if (e->state & REDIS_LIBEVENT_DELETED) {\
redisLibeventDestroy(e);\
return; \
}
if ((event & EV_TIMEOUT) && (e->state & REDIS_LIBEVENT_DELETED) == 0) {
redisAsyncHandleTimeout(e->context);
CHECK_DELETED();
}
if ((event & EV_READ) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) {
redisAsyncHandleRead(e->context);
CHECK_DELETED();
}
if ((event & EV_WRITE) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) {
redisAsyncHandleWrite(e->context);
CHECK_DELETED();
}
e->state &= ~REDIS_LIBEVENT_ENTERED;
#undef CHECK_DELETED
}
static void redisLibeventUpdate(void *privdata, short flag, int isRemove) {
redisLibeventEvents *e = (redisLibeventEvents *)privdata;
const struct timeval *tv = e->tv.tv_sec || e->tv.tv_usec ? &e->tv : NULL;
if (isRemove) {
if ((e->flags & flag) == 0) {
return;
} else {
e->flags &= ~flag;
}
} else {
if (e->flags & flag) {
return;
} else {
e->flags |= flag;
}
}
event_del(e->ev);
event_assign(e->ev, e->base, e->context->c.fd, e->flags | EV_PERSIST,
redisLibeventHandler, privdata);
event_add(e->ev, tv);
}
static void redisLibeventAddRead(void *privdata) {
redisLibeventUpdate(privdata, EV_READ, 0);
}
static void redisLibeventDelRead(void *privdata) {
redisLibeventUpdate(privdata, EV_READ, 1);
}
static void redisLibeventAddWrite(void *privdata) {
redisLibeventUpdate(privdata, EV_WRITE, 0);
}
static void redisLibeventDelWrite(void *privdata) {
redisLibeventUpdate(privdata, EV_WRITE, 1);
}
static void redisLibeventCleanup(void *privdata) {
redisLibeventEvents *e = (redisLibeventEvents*)privdata;
if (!e) {
return;
}
event_del(e->ev);
event_free(e->ev);
e->ev = NULL;
if (e->state & REDIS_LIBEVENT_ENTERED) {
e->state |= REDIS_LIBEVENT_DELETED;
} else {
redisLibeventDestroy(e);
}
}
static void redisLibeventSetTimeout(void *privdata, struct timeval tv) {
redisLibeventEvents *e = (redisLibeventEvents *)privdata;
short flags = e->flags;
e->flags = 0;
e->tv = tv;
redisLibeventUpdate(e, flags, 0);
}
static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) {
redisContext *c = &(ac->c);
redisLibeventEvents *e;
/* Nothing should be attached when something is already attached */
if (ac->ev.data != NULL)
return REDIS_ERR;
/* Create container for context and r/w events */
e = (redisLibeventEvents*)hi_calloc(1, sizeof(*e));
if (e == NULL)
return REDIS_ERR;
e->context = ac;
/* Register functions to start/stop listening for events */
ac->ev.addRead = redisLibeventAddRead;
ac->ev.delRead = redisLibeventDelRead;
ac->ev.addWrite = redisLibeventAddWrite;
ac->ev.delWrite = redisLibeventDelWrite;
ac->ev.cleanup = redisLibeventCleanup;
ac->ev.scheduleTimer = redisLibeventSetTimeout;
ac->ev.data = e;
/* Initialize and install read/write events */
e->ev = event_new(base, c->fd, EV_READ | EV_WRITE, redisLibeventHandler, e);
e->base = base;
return REDIS_OK;
}
#endif
+123
View File
@@ -0,0 +1,123 @@
#ifndef __HIREDIS_LIBHV_H__
#define __HIREDIS_LIBHV_H__
#include <hv/hloop.h>
#include "../hiredis.h"
#include "../async.h"
typedef struct redisLibhvEvents {
hio_t *io;
htimer_t *timer;
} redisLibhvEvents;
static void redisLibhvHandleEvents(hio_t* io) {
redisAsyncContext* context = (redisAsyncContext*)hevent_userdata(io);
int events = hio_events(io);
int revents = hio_revents(io);
if (context && (events & HV_READ) && (revents & HV_READ)) {
redisAsyncHandleRead(context);
}
if (context && (events & HV_WRITE) && (revents & HV_WRITE)) {
redisAsyncHandleWrite(context);
}
}
static void redisLibhvAddRead(void *privdata) {
redisLibhvEvents* events = (redisLibhvEvents*)privdata;
hio_add(events->io, redisLibhvHandleEvents, HV_READ);
}
static void redisLibhvDelRead(void *privdata) {
redisLibhvEvents* events = (redisLibhvEvents*)privdata;
hio_del(events->io, HV_READ);
}
static void redisLibhvAddWrite(void *privdata) {
redisLibhvEvents* events = (redisLibhvEvents*)privdata;
hio_add(events->io, redisLibhvHandleEvents, HV_WRITE);
}
static void redisLibhvDelWrite(void *privdata) {
redisLibhvEvents* events = (redisLibhvEvents*)privdata;
hio_del(events->io, HV_WRITE);
}
static void redisLibhvCleanup(void *privdata) {
redisLibhvEvents* events = (redisLibhvEvents*)privdata;
if (events->timer)
htimer_del(events->timer);
hio_close(events->io);
hevent_set_userdata(events->io, NULL);
hi_free(events);
}
static void redisLibhvTimeout(htimer_t* timer) {
hio_t* io = (hio_t*)hevent_userdata(timer);
redisAsyncHandleTimeout((redisAsyncContext*)hevent_userdata(io));
}
static void redisLibhvSetTimeout(void *privdata, struct timeval tv) {
redisLibhvEvents* events;
uint32_t millis;
hloop_t* loop;
events = (redisLibhvEvents*)privdata;
millis = tv.tv_sec * 1000 + tv.tv_usec / 1000;
if (millis == 0) {
/* Libhv disallows zero'd timers so treat this as a delete or NO OP */
if (events->timer) {
htimer_del(events->timer);
events->timer = NULL;
}
} else if (events->timer == NULL) {
/* Add new timer */
loop = hevent_loop(events->io);
events->timer = htimer_add(loop, redisLibhvTimeout, millis, 1);
hevent_set_userdata(events->timer, events->io);
} else {
/* Update existing timer */
htimer_reset(events->timer, millis);
}
}
static int redisLibhvAttach(redisAsyncContext* ac, hloop_t* loop) {
redisContext *c = &(ac->c);
redisLibhvEvents *events;
hio_t* io = NULL;
if (ac->ev.data != NULL) {
return REDIS_ERR;
}
/* Create container struct to keep track of our io and any timer */
events = (redisLibhvEvents*)hi_malloc(sizeof(*events));
if (events == NULL) {
return REDIS_ERR;
}
io = hio_get(loop, c->fd);
if (io == NULL) {
hi_free(events);
return REDIS_ERR;
}
hevent_set_userdata(io, ac);
events->io = io;
events->timer = NULL;
ac->ev.addRead = redisLibhvAddRead;
ac->ev.delRead = redisLibhvDelRead;
ac->ev.addWrite = redisLibhvAddWrite;
ac->ev.delWrite = redisLibhvDelWrite;
ac->ev.cleanup = redisLibhvCleanup;
ac->ev.scheduleTimer = redisLibhvSetTimeout;
ac->ev.data = events;
return REDIS_OK;
}
#endif
+177
View File
@@ -0,0 +1,177 @@
#ifndef HIREDIS_LIBSDEVENT_H
#define HIREDIS_LIBSDEVENT_H
#include <systemd/sd-event.h>
#include "../hiredis.h"
#include "../async.h"
#define REDIS_LIBSDEVENT_DELETED 0x01
#define REDIS_LIBSDEVENT_ENTERED 0x02
typedef struct redisLibsdeventEvents {
redisAsyncContext *context;
struct sd_event *event;
struct sd_event_source *fdSource;
struct sd_event_source *timerSource;
int fd;
short flags;
short state;
} redisLibsdeventEvents;
static void redisLibsdeventDestroy(redisLibsdeventEvents *e) {
if (e->fdSource) {
e->fdSource = sd_event_source_disable_unref(e->fdSource);
}
if (e->timerSource) {
e->timerSource = sd_event_source_disable_unref(e->timerSource);
}
sd_event_unref(e->event);
hi_free(e);
}
static int redisLibsdeventTimeoutHandler(sd_event_source *s, uint64_t usec, void *userdata) {
((void)s);
((void)usec);
redisLibsdeventEvents *e = (redisLibsdeventEvents*)userdata;
redisAsyncHandleTimeout(e->context);
return 0;
}
static int redisLibsdeventHandler(sd_event_source *s, int fd, uint32_t event, void *userdata) {
((void)s);
((void)fd);
redisLibsdeventEvents *e = (redisLibsdeventEvents*)userdata;
e->state |= REDIS_LIBSDEVENT_ENTERED;
#define CHECK_DELETED() if (e->state & REDIS_LIBSDEVENT_DELETED) {\
redisLibsdeventDestroy(e);\
return 0; \
}
if ((event & EPOLLIN) && e->context && (e->state & REDIS_LIBSDEVENT_DELETED) == 0) {
redisAsyncHandleRead(e->context);
CHECK_DELETED();
}
if ((event & EPOLLOUT) && e->context && (e->state & REDIS_LIBSDEVENT_DELETED) == 0) {
redisAsyncHandleWrite(e->context);
CHECK_DELETED();
}
e->state &= ~REDIS_LIBSDEVENT_ENTERED;
#undef CHECK_DELETED
return 0;
}
static void redisLibsdeventAddRead(void *userdata) {
redisLibsdeventEvents *e = (redisLibsdeventEvents*)userdata;
if (e->flags & EPOLLIN) {
return;
}
e->flags |= EPOLLIN;
if (e->flags & EPOLLOUT) {
sd_event_source_set_io_events(e->fdSource, e->flags);
} else {
sd_event_add_io(e->event, &e->fdSource, e->fd, e->flags, redisLibsdeventHandler, e);
}
}
static void redisLibsdeventDelRead(void *userdata) {
redisLibsdeventEvents *e = (redisLibsdeventEvents*)userdata;
e->flags &= ~EPOLLIN;
if (e->flags) {
sd_event_source_set_io_events(e->fdSource, e->flags);
} else {
e->fdSource = sd_event_source_disable_unref(e->fdSource);
}
}
static void redisLibsdeventAddWrite(void *userdata) {
redisLibsdeventEvents *e = (redisLibsdeventEvents*)userdata;
if (e->flags & EPOLLOUT) {
return;
}
e->flags |= EPOLLOUT;
if (e->flags & EPOLLIN) {
sd_event_source_set_io_events(e->fdSource, e->flags);
} else {
sd_event_add_io(e->event, &e->fdSource, e->fd, e->flags, redisLibsdeventHandler, e);
}
}
static void redisLibsdeventDelWrite(void *userdata) {
redisLibsdeventEvents *e = (redisLibsdeventEvents*)userdata;
e->flags &= ~EPOLLOUT;
if (e->flags) {
sd_event_source_set_io_events(e->fdSource, e->flags);
} else {
e->fdSource = sd_event_source_disable_unref(e->fdSource);
}
}
static void redisLibsdeventCleanup(void *userdata) {
redisLibsdeventEvents *e = (redisLibsdeventEvents*)userdata;
if (!e) {
return;
}
if (e->state & REDIS_LIBSDEVENT_ENTERED) {
e->state |= REDIS_LIBSDEVENT_DELETED;
} else {
redisLibsdeventDestroy(e);
}
}
static void redisLibsdeventSetTimeout(void *userdata, struct timeval tv) {
redisLibsdeventEvents *e = (redisLibsdeventEvents *)userdata;
uint64_t usec = tv.tv_sec * 1000000 + tv.tv_usec;
if (!e->timerSource) {
sd_event_add_time_relative(e->event, &e->timerSource, CLOCK_MONOTONIC, usec, 1, redisLibsdeventTimeoutHandler, e);
} else {
sd_event_source_set_time_relative(e->timerSource, usec);
}
}
static int redisLibsdeventAttach(redisAsyncContext *ac, struct sd_event *event) {
redisContext *c = &(ac->c);
redisLibsdeventEvents *e;
/* Nothing should be attached when something is already attached */
if (ac->ev.data != NULL)
return REDIS_ERR;
/* Create container for context and r/w events */
e = (redisLibsdeventEvents*)hi_calloc(1, sizeof(*e));
if (e == NULL)
return REDIS_ERR;
/* Initialize and increase event refcount */
e->context = ac;
e->event = event;
e->fd = c->fd;
sd_event_ref(event);
/* Register functions to start/stop listening for events */
ac->ev.addRead = redisLibsdeventAddRead;
ac->ev.delRead = redisLibsdeventDelRead;
ac->ev.addWrite = redisLibsdeventAddWrite;
ac->ev.delWrite = redisLibsdeventDelWrite;
ac->ev.cleanup = redisLibsdeventCleanup;
ac->ev.scheduleTimer = redisLibsdeventSetTimeout;
ac->ev.data = e;
return REDIS_OK;
}
#endif
+171
View File
@@ -0,0 +1,171 @@
#ifndef __HIREDIS_LIBUV_H__
#define __HIREDIS_LIBUV_H__
#include <stdlib.h>
#include <uv.h>
#include "../hiredis.h"
#include "../async.h"
#include <string.h>
typedef struct redisLibuvEvents {
redisAsyncContext* context;
uv_poll_t handle;
uv_timer_t timer;
int events;
} redisLibuvEvents;
static void redisLibuvPoll(uv_poll_t* handle, int status, int events) {
redisLibuvEvents* p = (redisLibuvEvents*)handle->data;
int ev = (status ? p->events : events);
if (p->context != NULL && (ev & UV_READABLE)) {
redisAsyncHandleRead(p->context);
}
if (p->context != NULL && (ev & UV_WRITABLE)) {
redisAsyncHandleWrite(p->context);
}
}
static void redisLibuvAddRead(void *privdata) {
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
if (p->events & UV_READABLE) {
return;
}
p->events |= UV_READABLE;
uv_poll_start(&p->handle, p->events, redisLibuvPoll);
}
static void redisLibuvDelRead(void *privdata) {
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
p->events &= ~UV_READABLE;
if (p->events) {
uv_poll_start(&p->handle, p->events, redisLibuvPoll);
} else {
uv_poll_stop(&p->handle);
}
}
static void redisLibuvAddWrite(void *privdata) {
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
if (p->events & UV_WRITABLE) {
return;
}
p->events |= UV_WRITABLE;
uv_poll_start(&p->handle, p->events, redisLibuvPoll);
}
static void redisLibuvDelWrite(void *privdata) {
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
p->events &= ~UV_WRITABLE;
if (p->events) {
uv_poll_start(&p->handle, p->events, redisLibuvPoll);
} else {
uv_poll_stop(&p->handle);
}
}
static void on_timer_close(uv_handle_t *handle) {
redisLibuvEvents* p = (redisLibuvEvents*)handle->data;
p->timer.data = NULL;
if (!p->handle.data) {
// both timer and handle are closed
hi_free(p);
}
// else, wait for `on_handle_close`
}
static void on_handle_close(uv_handle_t *handle) {
redisLibuvEvents* p = (redisLibuvEvents*)handle->data;
p->handle.data = NULL;
if (!p->timer.data) {
// timer never started, or timer already destroyed
hi_free(p);
}
// else, wait for `on_timer_close`
}
// libuv removed `status` parameter since v0.11.23
// see: https://github.com/libuv/libuv/blob/v0.11.23/include/uv.h
#if (UV_VERSION_MAJOR == 0 && UV_VERSION_MINOR < 11) || \
(UV_VERSION_MAJOR == 0 && UV_VERSION_MINOR == 11 && UV_VERSION_PATCH < 23)
static void redisLibuvTimeout(uv_timer_t *timer, int status) {
(void)status; // unused
#else
static void redisLibuvTimeout(uv_timer_t *timer) {
#endif
redisLibuvEvents *e = (redisLibuvEvents*)timer->data;
redisAsyncHandleTimeout(e->context);
}
static void redisLibuvSetTimeout(void *privdata, struct timeval tv) {
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
uint64_t millsec = tv.tv_sec * 1000 + tv.tv_usec / 1000.0;
if (!p->timer.data) {
// timer is uninitialized
if (uv_timer_init(p->handle.loop, &p->timer) != 0) {
return;
}
p->timer.data = p;
}
// updates the timeout if the timer has already started
// or start the timer
uv_timer_start(&p->timer, redisLibuvTimeout, millsec, 0);
}
static void redisLibuvCleanup(void *privdata) {
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
p->context = NULL; // indicate that context might no longer exist
if (p->timer.data) {
uv_close((uv_handle_t*)&p->timer, on_timer_close);
}
uv_close((uv_handle_t*)&p->handle, on_handle_close);
}
static int redisLibuvAttach(redisAsyncContext* ac, uv_loop_t* loop) {
redisContext *c = &(ac->c);
if (ac->ev.data != NULL) {
return REDIS_ERR;
}
ac->ev.addRead = redisLibuvAddRead;
ac->ev.delRead = redisLibuvDelRead;
ac->ev.addWrite = redisLibuvAddWrite;
ac->ev.delWrite = redisLibuvDelWrite;
ac->ev.cleanup = redisLibuvCleanup;
ac->ev.scheduleTimer = redisLibuvSetTimeout;
redisLibuvEvents* p = (redisLibuvEvents*)hi_malloc(sizeof(*p));
if (p == NULL)
return REDIS_ERR;
memset(p, 0, sizeof(*p));
if (uv_poll_init_socket(loop, &p->handle, c->fd) != 0) {
return REDIS_ERR;
}
ac->ev.data = p;
p->handle.data = p;
p->context = ac;
return REDIS_OK;
}
#endif
+115
View File
@@ -0,0 +1,115 @@
//
// Created by Дмитрий Бахвалов on 13.07.15.
// Copyright (c) 2015 Dmitry Bakhvalov. All rights reserved.
//
#ifndef __HIREDIS_MACOSX_H__
#define __HIREDIS_MACOSX_H__
#include <CoreFoundation/CoreFoundation.h>
#include "../hiredis.h"
#include "../async.h"
typedef struct {
redisAsyncContext *context;
CFSocketRef socketRef;
CFRunLoopSourceRef sourceRef;
} RedisRunLoop;
static int freeRedisRunLoop(RedisRunLoop* redisRunLoop) {
if( redisRunLoop != NULL ) {
if( redisRunLoop->sourceRef != NULL ) {
CFRunLoopSourceInvalidate(redisRunLoop->sourceRef);
CFRelease(redisRunLoop->sourceRef);
}
if( redisRunLoop->socketRef != NULL ) {
CFSocketInvalidate(redisRunLoop->socketRef);
CFRelease(redisRunLoop->socketRef);
}
hi_free(redisRunLoop);
}
return REDIS_ERR;
}
static void redisMacOSAddRead(void *privdata) {
RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
CFSocketEnableCallBacks(redisRunLoop->socketRef, kCFSocketReadCallBack);
}
static void redisMacOSDelRead(void *privdata) {
RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
CFSocketDisableCallBacks(redisRunLoop->socketRef, kCFSocketReadCallBack);
}
static void redisMacOSAddWrite(void *privdata) {
RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
CFSocketEnableCallBacks(redisRunLoop->socketRef, kCFSocketWriteCallBack);
}
static void redisMacOSDelWrite(void *privdata) {
RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
CFSocketDisableCallBacks(redisRunLoop->socketRef, kCFSocketWriteCallBack);
}
static void redisMacOSCleanup(void *privdata) {
RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
freeRedisRunLoop(redisRunLoop);
}
static void redisMacOSAsyncCallback(CFSocketRef __unused s, CFSocketCallBackType callbackType, CFDataRef __unused address, const void __unused *data, void *info) {
redisAsyncContext* context = (redisAsyncContext*) info;
switch (callbackType) {
case kCFSocketReadCallBack:
redisAsyncHandleRead(context);
break;
case kCFSocketWriteCallBack:
redisAsyncHandleWrite(context);
break;
default:
break;
}
}
static int redisMacOSAttach(redisAsyncContext *redisAsyncCtx, CFRunLoopRef runLoop) {
redisContext *redisCtx = &(redisAsyncCtx->c);
/* Nothing should be attached when something is already attached */
if( redisAsyncCtx->ev.data != NULL ) return REDIS_ERR;
RedisRunLoop* redisRunLoop = (RedisRunLoop*) hi_calloc(1, sizeof(RedisRunLoop));
if (redisRunLoop == NULL)
return REDIS_ERR;
/* Setup redis stuff */
redisRunLoop->context = redisAsyncCtx;
redisAsyncCtx->ev.addRead = redisMacOSAddRead;
redisAsyncCtx->ev.delRead = redisMacOSDelRead;
redisAsyncCtx->ev.addWrite = redisMacOSAddWrite;
redisAsyncCtx->ev.delWrite = redisMacOSDelWrite;
redisAsyncCtx->ev.cleanup = redisMacOSCleanup;
redisAsyncCtx->ev.data = redisRunLoop;
/* Initialize and install read/write events */
CFSocketContext socketCtx = { 0, redisAsyncCtx, NULL, NULL, NULL };
redisRunLoop->socketRef = CFSocketCreateWithNative(NULL, redisCtx->fd,
kCFSocketReadCallBack | kCFSocketWriteCallBack,
redisMacOSAsyncCallback,
&socketCtx);
if( !redisRunLoop->socketRef ) return freeRedisRunLoop(redisRunLoop);
redisRunLoop->sourceRef = CFSocketCreateRunLoopSource(NULL, redisRunLoop->socketRef, 0);
if( !redisRunLoop->sourceRef ) return freeRedisRunLoop(redisRunLoop);
CFRunLoopAddSource(runLoop, redisRunLoop->sourceRef, kCFRunLoopDefaultMode);
return REDIS_OK;
}
#endif
@@ -1,43 +1,41 @@
#ifndef KV_ADAPTERS_POLL_H
#define KV_ADAPTERS_POLL_H
#ifndef HIREDIS_POLL_H
#define HIREDIS_POLL_H
#include "../async.h"
#include "../cluster.h"
#include "../sockcompat.h"
#include <errno.h>
#include <string.h> // for memset
#include <errno.h>
/* Values to return from kvPollTick */
#define KV_POLL_HANDLED_READ 1
#define KV_POLL_HANDLED_WRITE 2
#define KV_POLL_HANDLED_TIMEOUT 4
/* Values to return from redisPollTick */
#define REDIS_POLL_HANDLED_READ 1
#define REDIS_POLL_HANDLED_WRITE 2
#define REDIS_POLL_HANDLED_TIMEOUT 4
/* An adapter to allow manual polling of the async context by checking the state
* of the underlying file descriptor. Useful in cases where there is no formal
* IO event loop but regular ticking can be used, such as in game engines. */
typedef struct kvPollEvents {
kvAsyncContext *context;
kvFD fd;
typedef struct redisPollEvents {
redisAsyncContext *context;
redisFD fd;
char reading, writing;
char in_tick;
char deleted;
double deadline;
} kvPollEvents;
} redisPollEvents;
static double kvPollTimevalToDouble(struct timeval *tv) {
static double redisPollTimevalToDouble(struct timeval *tv) {
if (tv == NULL)
return 0.0;
return tv->tv_sec + tv->tv_usec / 1000000.00;
}
static double kvPollGetNow(void) {
static double redisPollGetNow(void) {
#ifndef _MSC_VER
struct timeval tv;
gettimeofday(&tv, NULL);
return kvPollTimevalToDouble(&tv);
gettimeofday(&tv,NULL);
return redisPollTimevalToDouble(&tv);
#else
FILETIME ft;
ULARGE_INTEGER li;
@@ -51,14 +49,14 @@ static double kvPollGetNow(void) {
/* Poll for io, handling any pending callbacks. The timeout argument can be
* positive to wait for a maximum given time for IO, zero to poll, or negative
* to wait forever */
static int kvPollTick(kvAsyncContext *ac, double timeout) {
static int redisPollTick(redisAsyncContext *ac, double timeout) {
int reading, writing;
struct pollfd pfd;
int handled;
int ns;
int itimeout;
kvPollEvents *e = (kvPollEvents *)ac->ev.data;
redisPollEvents *e = (redisPollEvents*)ac->ev.data;
if (!e)
return 0;
@@ -71,7 +69,7 @@ static int kvPollTick(kvAsyncContext *ac, double timeout) {
pfd.fd = e->fd;
pfd.events = 0;
if (reading)
pfd.events = POLLIN;
pfd.events = POLLIN;
if (writing)
pfd.events |= POLLOUT;
@@ -88,94 +86,95 @@ static int kvPollTick(kvAsyncContext *ac, double timeout) {
return ns;
ns = 0;
}
handled = 0;
e->in_tick = 1;
if (ns) {
if (reading && (pfd.revents & POLLIN)) {
kvAsyncHandleRead(ac);
handled |= KV_POLL_HANDLED_READ;
redisAsyncHandleRead(ac);
handled |= REDIS_POLL_HANDLED_READ;
}
/* on Windows, connection failure is indicated with the Exception fdset.
* handle it the same as writable. */
if (writing && (pfd.revents & (POLLOUT | POLLERR))) {
/* context Read callback may have caused context to be deleted, e.g.
by doing a kvAsyncDisconnect() */
by doing an redisAsyncDisconnect() */
if (!e->deleted) {
kvAsyncHandleWrite(ac);
handled |= KV_POLL_HANDLED_WRITE;
redisAsyncHandleWrite(ac);
handled |= REDIS_POLL_HANDLED_WRITE;
}
}
}
/* perform timeouts */
if (!e->deleted && e->deadline != 0.0) {
double now = kvPollGetNow();
double now = redisPollGetNow();
if (now >= e->deadline) {
/* deadline has passed. disable timeout and perform callback */
e->deadline = 0.0;
kvAsyncHandleTimeout(ac);
handled |= KV_POLL_HANDLED_TIMEOUT;
redisAsyncHandleTimeout(ac);
handled |= REDIS_POLL_HANDLED_TIMEOUT;
}
}
/* do a delayed cleanup if required */
if (e->deleted)
vk_free(e);
hi_free(e);
else
e->in_tick = 0;
return handled;
}
static void kvPollAddRead(void *data) {
kvPollEvents *e = (kvPollEvents *)data;
static void redisPollAddRead(void *data) {
redisPollEvents *e = (redisPollEvents*)data;
e->reading = 1;
}
static void kvPollDelRead(void *data) {
kvPollEvents *e = (kvPollEvents *)data;
static void redisPollDelRead(void *data) {
redisPollEvents *e = (redisPollEvents*)data;
e->reading = 0;
}
static void kvPollAddWrite(void *data) {
kvPollEvents *e = (kvPollEvents *)data;
static void redisPollAddWrite(void *data) {
redisPollEvents *e = (redisPollEvents*)data;
e->writing = 1;
}
static void kvPollDelWrite(void *data) {
kvPollEvents *e = (kvPollEvents *)data;
static void redisPollDelWrite(void *data) {
redisPollEvents *e = (redisPollEvents*)data;
e->writing = 0;
}
static void kvPollCleanup(void *data) {
kvPollEvents *e = (kvPollEvents *)data;
static void redisPollCleanup(void *data) {
redisPollEvents *e = (redisPollEvents*)data;
/* if we are currently processing a tick, postpone deletion */
if (e->in_tick)
e->deleted = 1;
else
vk_free(e);
hi_free(e);
}
static void kvPollScheduleTimer(void *data, struct timeval tv) {
kvPollEvents *e = (kvPollEvents *)data;
double now = kvPollGetNow();
e->deadline = now + kvPollTimevalToDouble(&tv);
static void redisPollScheduleTimer(void *data, struct timeval tv)
{
redisPollEvents *e = (redisPollEvents*)data;
double now = redisPollGetNow();
e->deadline = now + redisPollTimevalToDouble(&tv);
}
static int kvPollAttach(kvAsyncContext *ac) {
kvContext *c = &(ac->c);
kvPollEvents *e;
static int redisPollAttach(redisAsyncContext *ac) {
redisContext *c = &(ac->c);
redisPollEvents *e;
/* Nothing should be attached when something is already attached */
if (ac->ev.data != NULL)
return KV_ERR;
return REDIS_ERR;
/* Create container for context and r/w events */
e = (kvPollEvents *)vk_malloc(sizeof(*e));
e = (redisPollEvents*)hi_malloc(sizeof(*e));
if (e == NULL)
return KV_ERR;
return REDIS_ERR;
memset(e, 0, sizeof(*e));
e->context = ac;
@@ -185,30 +184,14 @@ static int kvPollAttach(kvAsyncContext *ac) {
e->deadline = 0.0;
/* Register functions to start/stop listening for events */
ac->ev.addRead = kvPollAddRead;
ac->ev.delRead = kvPollDelRead;
ac->ev.addWrite = kvPollAddWrite;
ac->ev.delWrite = kvPollDelWrite;
ac->ev.scheduleTimer = kvPollScheduleTimer;
ac->ev.cleanup = kvPollCleanup;
ac->ev.addRead = redisPollAddRead;
ac->ev.delRead = redisPollDelRead;
ac->ev.addWrite = redisPollAddWrite;
ac->ev.delWrite = redisPollDelWrite;
ac->ev.scheduleTimer = redisPollScheduleTimer;
ac->ev.cleanup = redisPollCleanup;
ac->ev.data = e;
return KV_OK;
return REDIS_OK;
}
/* Internal adapter function with correct function signature. */
static int kvPollAttachAdapter(kvAsyncContext *ac, KV_UNUSED void *unused) {
return kvPollAttach(ac);
}
KV_UNUSED
static int kvClusterOptionsUsePoll(kvClusterOptions *options) {
if (options == NULL) {
return KV_ERR;
}
options->attach_fn = kvPollAttachAdapter;
return KV_OK;
}
#endif /* KV_ADAPTERS_POLL_H */
#endif /* HIREDIS_POLL_H */
+135
View File
@@ -0,0 +1,135 @@
/*-
* Copyright (C) 2014 Pietro Cerutti <gahr@gahr.ch>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef __HIREDIS_QT_H__
#define __HIREDIS_QT_H__
#include <QSocketNotifier>
#include "../async.h"
static void RedisQtAddRead(void *);
static void RedisQtDelRead(void *);
static void RedisQtAddWrite(void *);
static void RedisQtDelWrite(void *);
static void RedisQtCleanup(void *);
class RedisQtAdapter : public QObject {
Q_OBJECT
friend
void RedisQtAddRead(void * adapter) {
RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
a->addRead();
}
friend
void RedisQtDelRead(void * adapter) {
RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
a->delRead();
}
friend
void RedisQtAddWrite(void * adapter) {
RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
a->addWrite();
}
friend
void RedisQtDelWrite(void * adapter) {
RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
a->delWrite();
}
friend
void RedisQtCleanup(void * adapter) {
RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
a->cleanup();
}
public:
RedisQtAdapter(QObject * parent = 0)
: QObject(parent), m_ctx(0), m_read(0), m_write(0) { }
~RedisQtAdapter() {
if (m_ctx != 0) {
m_ctx->ev.data = NULL;
}
}
int setContext(redisAsyncContext * ac) {
if (ac->ev.data != NULL) {
return REDIS_ERR;
}
m_ctx = ac;
m_ctx->ev.data = this;
m_ctx->ev.addRead = RedisQtAddRead;
m_ctx->ev.delRead = RedisQtDelRead;
m_ctx->ev.addWrite = RedisQtAddWrite;
m_ctx->ev.delWrite = RedisQtDelWrite;
m_ctx->ev.cleanup = RedisQtCleanup;
return REDIS_OK;
}
private:
void addRead() {
if (m_read) return;
m_read = new QSocketNotifier(m_ctx->c.fd, QSocketNotifier::Read, 0);
connect(m_read, SIGNAL(activated(int)), this, SLOT(read()));
}
void delRead() {
if (!m_read) return;
delete m_read;
m_read = 0;
}
void addWrite() {
if (m_write) return;
m_write = new QSocketNotifier(m_ctx->c.fd, QSocketNotifier::Write, 0);
connect(m_write, SIGNAL(activated(int)), this, SLOT(write()));
}
void delWrite() {
if (!m_write) return;
delete m_write;
m_write = 0;
}
void cleanup() {
delRead();
delWrite();
}
private slots:
void read() { redisAsyncHandleRead(m_ctx); }
void write() { redisAsyncHandleWrite(m_ctx); }
private:
redisAsyncContext * m_ctx;
QSocketNotifier * m_read;
QSocketNotifier * m_write;
};
#endif /* !__HIREDIS_QT_H__ */
+144
View File
@@ -0,0 +1,144 @@
#ifndef __HIREDIS_REDISMODULEAPI_H__
#define __HIREDIS_REDISMODULEAPI_H__
#include "redismodule.h"
#include "../async.h"
#include "../hiredis.h"
#include <sys/types.h>
typedef struct redisModuleEvents {
redisAsyncContext *context;
RedisModuleCtx *module_ctx;
int fd;
int reading, writing;
int timer_active;
RedisModuleTimerID timer_id;
} redisModuleEvents;
static inline void redisModuleReadEvent(int fd, void *privdata, int mask) {
(void) fd;
(void) mask;
redisModuleEvents *e = (redisModuleEvents*)privdata;
redisAsyncHandleRead(e->context);
}
static inline void redisModuleWriteEvent(int fd, void *privdata, int mask) {
(void) fd;
(void) mask;
redisModuleEvents *e = (redisModuleEvents*)privdata;
redisAsyncHandleWrite(e->context);
}
static inline void redisModuleAddRead(void *privdata) {
redisModuleEvents *e = (redisModuleEvents*)privdata;
if (!e->reading) {
e->reading = 1;
RedisModule_EventLoopAdd(e->fd, REDISMODULE_EVENTLOOP_READABLE, redisModuleReadEvent, e);
}
}
static inline void redisModuleDelRead(void *privdata) {
redisModuleEvents *e = (redisModuleEvents*)privdata;
if (e->reading) {
e->reading = 0;
RedisModule_EventLoopDel(e->fd, REDISMODULE_EVENTLOOP_READABLE);
}
}
static inline void redisModuleAddWrite(void *privdata) {
redisModuleEvents *e = (redisModuleEvents*)privdata;
if (!e->writing) {
e->writing = 1;
RedisModule_EventLoopAdd(e->fd, REDISMODULE_EVENTLOOP_WRITABLE, redisModuleWriteEvent, e);
}
}
static inline void redisModuleDelWrite(void *privdata) {
redisModuleEvents *e = (redisModuleEvents*)privdata;
if (e->writing) {
e->writing = 0;
RedisModule_EventLoopDel(e->fd, REDISMODULE_EVENTLOOP_WRITABLE);
}
}
static inline void redisModuleStopTimer(void *privdata) {
redisModuleEvents *e = (redisModuleEvents*)privdata;
if (e->timer_active) {
RedisModule_StopTimer(e->module_ctx, e->timer_id, NULL);
}
e->timer_active = 0;
}
static inline void redisModuleCleanup(void *privdata) {
redisModuleEvents *e = (redisModuleEvents*)privdata;
redisModuleDelRead(privdata);
redisModuleDelWrite(privdata);
redisModuleStopTimer(privdata);
hi_free(e);
}
static inline void redisModuleTimeout(RedisModuleCtx *ctx, void *privdata) {
(void) ctx;
redisModuleEvents *e = (redisModuleEvents*)privdata;
e->timer_active = 0;
redisAsyncHandleTimeout(e->context);
}
static inline void redisModuleSetTimeout(void *privdata, struct timeval tv) {
redisModuleEvents* e = (redisModuleEvents*)privdata;
redisModuleStopTimer(privdata);
mstime_t millis = tv.tv_sec * 1000 + tv.tv_usec / 1000.0;
e->timer_id = RedisModule_CreateTimer(e->module_ctx, millis, redisModuleTimeout, e);
e->timer_active = 1;
}
/* Check if Redis version is compatible with the adapter. */
static inline int redisModuleCompatibilityCheck(void) {
if (!RedisModule_EventLoopAdd ||
!RedisModule_EventLoopDel ||
!RedisModule_CreateTimer ||
!RedisModule_StopTimer) {
return REDIS_ERR;
}
return REDIS_OK;
}
static inline int redisModuleAttach(redisAsyncContext *ac, RedisModuleCtx *module_ctx) {
redisContext *c = &(ac->c);
redisModuleEvents *e;
/* Nothing should be attached when something is already attached */
if (ac->ev.data != NULL)
return REDIS_ERR;
/* Create container for context and r/w events */
e = (redisModuleEvents*)hi_malloc(sizeof(*e));
if (e == NULL)
return REDIS_ERR;
e->context = ac;
e->module_ctx = module_ctx;
e->fd = c->fd;
e->reading = e->writing = 0;
e->timer_active = 0;
/* Register functions to start/stop listening for events */
ac->ev.addRead = redisModuleAddRead;
ac->ev.delRead = redisModuleDelRead;
ac->ev.addWrite = redisModuleAddWrite;
ac->ev.delWrite = redisModuleDelWrite;
ac->ev.cleanup = redisModuleCleanup;
ac->ev.scheduleTimer = redisModuleSetTimeout;
ac->ev.data = e;
return REDIS_OK;
}
#endif
+18 -20
View File
@@ -29,13 +29,11 @@
*/
#include "fmacros.h"
#include "alloc.h"
#include <stdlib.h>
#include <string.h>
#include <stdlib.h>
kvAllocFuncs kvAllocFns = {
hiredisAllocFuncs hiredisAllocFns = {
.mallocFn = malloc,
.callocFn = calloc,
.reallocFn = realloc,
@@ -43,18 +41,18 @@ kvAllocFuncs kvAllocFns = {
.freeFn = free,
};
/* Override kv' allocators with ones supplied by the user */
kvAllocFuncs kvSetAllocators(kvAllocFuncs *override) {
kvAllocFuncs orig = kvAllocFns;
/* Override hiredis' allocators with ones supplied by the user */
hiredisAllocFuncs hiredisSetAllocators(hiredisAllocFuncs *override) {
hiredisAllocFuncs orig = hiredisAllocFns;
kvAllocFns = *override;
hiredisAllocFns = *override;
return orig;
}
/* Reset allocators to use libc defaults */
void kvResetAllocators(void) {
kvAllocFns = (kvAllocFuncs){
void hiredisResetAllocators(void) {
hiredisAllocFns = (hiredisAllocFuncs) {
.mallocFn = malloc,
.callocFn = calloc,
.reallocFn = realloc,
@@ -65,28 +63,28 @@ void kvResetAllocators(void) {
#ifdef _WIN32
void *vk_malloc(size_t size) {
return kvAllocFns.mallocFn(size);
void *hi_malloc(size_t size) {
return hiredisAllocFns.mallocFn(size);
}
void *vk_calloc(size_t nmemb, size_t size) {
void *hi_calloc(size_t nmemb, size_t size) {
/* Overflow check as the user can specify any arbitrary allocator */
if (SIZE_MAX / size < nmemb)
return NULL;
return kvAllocFns.callocFn(nmemb, size);
return hiredisAllocFns.callocFn(nmemb, size);
}
void *vk_realloc(void *ptr, size_t size) {
return kvAllocFns.reallocFn(ptr, size);
void *hi_realloc(void *ptr, size_t size) {
return hiredisAllocFns.reallocFn(ptr, size);
}
char *vk_strdup(const char *str) {
return kvAllocFns.strdupFn(str);
char *hi_strdup(const char *str) {
return hiredisAllocFns.strdupFn(str);
}
void vk_free(void *ptr) {
kvAllocFns.freeFn(ptr);
void hi_free(void *ptr) {
hiredisAllocFns.freeFn(ptr);
}
#endif
@@ -28,9 +28,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef KV_ALLOC_H
#define KV_ALLOC_H
#include "visibility.h"
#ifndef HIREDIS_ALLOC_H
#define HIREDIS_ALLOC_H
#include <stddef.h> /* for size_t */
#include <stdint.h>
@@ -40,53 +39,53 @@ extern "C" {
#endif
/* Structure pointing to our actually configured allocators */
typedef struct kvAllocFuncs {
typedef struct hiredisAllocFuncs {
void *(*mallocFn)(size_t);
void *(*callocFn)(size_t, size_t);
void *(*reallocFn)(void *, size_t);
char *(*strdupFn)(const char *);
void (*freeFn)(void *);
} kvAllocFuncs;
void *(*callocFn)(size_t,size_t);
void *(*reallocFn)(void*,size_t);
char *(*strdupFn)(const char*);
void (*freeFn)(void*);
} hiredisAllocFuncs;
LIBKV_API kvAllocFuncs kvSetAllocators(kvAllocFuncs *fns);
LIBKV_API void kvResetAllocators(void);
hiredisAllocFuncs hiredisSetAllocators(hiredisAllocFuncs *ha);
void hiredisResetAllocators(void);
#ifndef _WIN32
/* kv' configured allocator function pointer struct */
LIBKV_API extern kvAllocFuncs kvAllocFns;
/* Hiredis' configured allocator function pointer struct */
extern hiredisAllocFuncs hiredisAllocFns;
static inline void *vk_malloc(size_t size) {
return kvAllocFns.mallocFn(size);
static inline void *hi_malloc(size_t size) {
return hiredisAllocFns.mallocFn(size);
}
static inline void *vk_calloc(size_t nmemb, size_t size) {
static inline void *hi_calloc(size_t nmemb, size_t size) {
/* Overflow check as the user can specify any arbitrary allocator */
if (SIZE_MAX / size < nmemb)
return NULL;
return kvAllocFns.callocFn(nmemb, size);
return hiredisAllocFns.callocFn(nmemb, size);
}
static inline void *vk_realloc(void *ptr, size_t size) {
return kvAllocFns.reallocFn(ptr, size);
static inline void *hi_realloc(void *ptr, size_t size) {
return hiredisAllocFns.reallocFn(ptr, size);
}
static inline char *vk_strdup(const char *str) {
return kvAllocFns.strdupFn(str);
static inline char *hi_strdup(const char *str) {
return hiredisAllocFns.strdupFn(str);
}
static inline void vk_free(void *ptr) {
kvAllocFns.freeFn(ptr);
static inline void hi_free(void *ptr) {
hiredisAllocFns.freeFn(ptr);
}
#else
LIBKV_API void *vk_malloc(size_t size);
LIBKV_API void *vk_calloc(size_t nmemb, size_t size);
LIBKV_API void *vk_realloc(void *ptr, size_t size);
LIBKV_API char *vk_strdup(const char *str);
LIBKV_API void vk_free(void *ptr);
void *hi_malloc(size_t size);
void *hi_calloc(size_t nmemb, size_t size);
void *hi_realloc(void *ptr, size_t size);
char *hi_strdup(const char *str);
void hi_free(void *ptr);
#endif
@@ -94,4 +93,4 @@ LIBKV_API void vk_free(void *ptr);
}
#endif
#endif /* KV_ALLOC_H */
#endif /* HIREDIS_ALLOC_H */
+24
View File
@@ -0,0 +1,24 @@
# Appveyor configuration file for CI build of hiredis on Windows (under Cygwin)
environment:
matrix:
- CYG_BASH: C:\cygwin64\bin\bash
CC: gcc
- CYG_BASH: C:\cygwin\bin\bash
CC: gcc
CFLAGS: -m32
CXXFLAGS: -m32
LDFLAGS: -m32
clone_depth: 1
# Attempt to ensure we don't try to convert line endings to Win32 CRLF as this will cause build to fail
init:
- git config --global core.autocrlf input
# Install needed build dependencies
install:
- '%CYG_BASH% -lc "cygcheck -dc cygwin"'
build_script:
- 'echo building...'
- '%CYG_BASH% -lc "cd $APPVEYOR_BUILD_FOLDER; exec 0</dev/null; mkdir build && cd build && cmake .. -G \"Unix Makefiles\" && make VERBOSE=1"'
+1034
View File
File diff suppressed because it is too large Load Diff
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2009-2011, Redis Ltd.
* Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
*
* All rights reserved.
@@ -29,56 +29,48 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef KV_ASYNC_H
#define KV_ASYNC_H
#include "kv.h"
#include "visibility.h"
#ifndef __HIREDIS_ASYNC_H
#define __HIREDIS_ASYNC_H
#include "hiredis.h"
#ifdef __cplusplus
extern "C" {
#endif
/* For the async cluster attach functions. */
#if defined(__GNUC__) || defined(__clang__)
#define KV_UNUSED __attribute__((unused))
#else
#define KV_UNUSED
#endif
struct kvAsyncContext; /* need forward declaration of kvAsyncContext */
struct dict; /* dictionary header is included in async.c */
struct redisAsyncContext; /* need forward declaration of redisAsyncContext */
struct dict; /* dictionary header is included in async.c */
/* Reply callback prototype and container */
typedef void(kvCallbackFn)(struct kvAsyncContext *, void *, void *);
typedef struct kvCallback {
struct kvCallback *next; /* simple singly linked list */
kvCallbackFn *fn;
typedef void (redisCallbackFn)(struct redisAsyncContext*, void*, void*);
typedef struct redisCallback {
struct redisCallback *next; /* simple singly linked list */
redisCallbackFn *fn;
int pending_subs;
int unsubscribe_sent;
void *privdata;
int subscribed;
} kvCallback;
} redisCallback;
/* List of callbacks for either regular replies or pub/sub */
typedef struct kvCallbackList {
kvCallback *head, *tail;
} kvCallbackList;
typedef struct redisCallbackList {
redisCallback *head, *tail;
} redisCallbackList;
/* Connection callback prototypes */
typedef void(kvDisconnectCallback)(const struct kvAsyncContext *, int status);
typedef void(kvConnectCallback)(struct kvAsyncContext *, int status);
typedef void(kvTimerCallback)(void *timer, void *privdata);
typedef void (redisDisconnectCallback)(const struct redisAsyncContext*, int status);
typedef void (redisConnectCallback)(const struct redisAsyncContext*, int status);
typedef void (redisConnectCallbackNC)(struct redisAsyncContext *, int status);
typedef void(redisTimerCallback)(void *timer, void *privdata);
/* Context for an async connection to KV */
typedef struct kvAsyncContext {
/* Context for an async connection to Redis */
typedef struct redisAsyncContext {
/* Hold the regular context, so it can be realloc'ed. */
kvContext c;
redisContext c;
/* Setup error flags so they can be used directly. */
int err;
char *errstr;
/* Not used by libkv */
/* Not used by hiredis */
void *data;
void (*dataCleanup)(void *privdata);
@@ -97,14 +89,15 @@ typedef struct kvAsyncContext {
} ev;
/* Called when either the connection is terminated due to an error or per
* user request. The status is set accordingly (KV_OK, KV_ERR). */
kvDisconnectCallback *onDisconnect;
* user request. The status is set accordingly (REDIS_OK, REDIS_ERR). */
redisDisconnectCallback *onDisconnect;
/* Called when the first write event was received. */
kvConnectCallback *onConnect;
redisConnectCallback *onConnect;
redisConnectCallbackNC *onConnectNC;
/* Regular command callbacks */
kvCallbackList replies;
redisCallbackList replies;
/* Address used for connect() */
struct sockaddr *saddr;
@@ -112,47 +105,48 @@ typedef struct kvAsyncContext {
/* Subscription callbacks */
struct {
kvCallbackList replies;
redisCallbackList replies;
struct dict *channels;
struct dict *patterns;
struct dict *schannels;
int pending_unsubs;
} sub;
/* Any configured RESP3 PUSH handler */
kvAsyncPushFn *push_cb;
} kvAsyncContext;
redisAsyncPushFn *push_cb;
} redisAsyncContext;
LIBKV_API kvAsyncContext *kvAsyncConnectWithOptions(const kvOptions *options);
LIBKV_API kvAsyncContext *kvAsyncConnect(const char *ip, int port);
LIBKV_API kvAsyncContext *kvAsyncConnectBind(const char *ip, int port, const char *source_addr);
LIBKV_API kvAsyncContext *kvAsyncConnectBindWithReuse(const char *ip, int port,
const char *source_addr);
LIBKV_API kvAsyncContext *kvAsyncConnectUnix(const char *path);
LIBKV_API int kvAsyncSetConnectCallback(kvAsyncContext *ac, kvConnectCallback *fn);
LIBKV_API int kvAsyncSetDisconnectCallback(kvAsyncContext *ac, kvDisconnectCallback *fn);
/* Functions that proxy to hiredis */
redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options);
redisAsyncContext *redisAsyncConnect(const char *ip, int port);
redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr);
redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port,
const char *source_addr);
redisAsyncContext *redisAsyncConnectUnix(const char *path);
int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn);
int redisAsyncSetConnectCallbackNC(redisAsyncContext *ac, redisConnectCallbackNC *fn);
int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn);
LIBKV_API kvAsyncPushFn *kvAsyncSetPushCallback(kvAsyncContext *ac, kvAsyncPushFn *fn);
LIBKV_API int kvAsyncSetTimeout(kvAsyncContext *ac, struct timeval tv);
LIBKV_API void kvAsyncDisconnect(kvAsyncContext *ac);
LIBKV_API void kvAsyncFree(kvAsyncContext *ac);
redisAsyncPushFn *redisAsyncSetPushCallback(redisAsyncContext *ac, redisAsyncPushFn *fn);
int redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv);
void redisAsyncDisconnect(redisAsyncContext *ac);
void redisAsyncFree(redisAsyncContext *ac);
/* Handle read/write events */
LIBKV_API void kvAsyncHandleRead(kvAsyncContext *ac);
LIBKV_API void kvAsyncHandleWrite(kvAsyncContext *ac);
LIBKV_API void kvAsyncHandleTimeout(kvAsyncContext *ac);
LIBKV_API void kvAsyncRead(kvAsyncContext *ac);
LIBKV_API void kvAsyncWrite(kvAsyncContext *ac);
void redisAsyncHandleRead(redisAsyncContext *ac);
void redisAsyncHandleWrite(redisAsyncContext *ac);
void redisAsyncHandleTimeout(redisAsyncContext *ac);
void redisAsyncRead(redisAsyncContext *ac);
void redisAsyncWrite(redisAsyncContext *ac);
/* Command functions for an async context. Write the command to the
* output buffer and register the provided callback. */
LIBKV_API int kvvAsyncCommand(kvAsyncContext *ac, kvCallbackFn *fn, void *privdata, const char *format, va_list ap);
LIBKV_API int kvAsyncCommand(kvAsyncContext *ac, kvCallbackFn *fn, void *privdata, const char *format, ...);
LIBKV_API int kvAsyncCommandArgv(kvAsyncContext *ac, kvCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen);
LIBKV_API int kvAsyncFormattedCommand(kvAsyncContext *ac, kvCallbackFn *fn, void *privdata, const char *cmd, size_t len);
int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap);
int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...);
int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen);
int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len);
#ifdef __cplusplus
}
#endif
#endif /* KV_ASYNC_H */
#endif
+75
View File
@@ -0,0 +1,75 @@
/*
* Copyright (c) 2009-2011, Redis Ltd.
* Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __HIREDIS_ASYNC_PRIVATE_H
#define __HIREDIS_ASYNC_PRIVATE_H
#define _EL_ADD_READ(ctx) \
do { \
refreshTimeout(ctx); \
if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \
} while (0)
#define _EL_DEL_READ(ctx) do { \
if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \
} while(0)
#define _EL_ADD_WRITE(ctx) \
do { \
refreshTimeout(ctx); \
if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \
} while (0)
#define _EL_DEL_WRITE(ctx) do { \
if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \
} while(0)
#define _EL_CLEANUP(ctx) do { \
if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \
ctx->ev.cleanup = NULL; \
} while(0)
static inline void refreshTimeout(redisAsyncContext *ctx) {
#define REDIS_TIMER_ISSET(tvp) \
(tvp && ((tvp)->tv_sec || (tvp)->tv_usec))
#define REDIS_EL_TIMER(ac, tvp) \
if ((ac)->ev.scheduleTimer && REDIS_TIMER_ISSET(tvp)) { \
(ac)->ev.scheduleTimer((ac)->ev.data, *(tvp)); \
}
if (ctx->c.flags & REDIS_CONNECTED) {
REDIS_EL_TIMER(ctx, ctx->c.command_timeout);
} else {
REDIS_EL_TIMER(ctx, ctx->c.connect_timeout);
}
}
void __redisAsyncDisconnect(redisAsyncContext *ac);
void redisProcessCallbacks(redisAsyncContext *ac);
#endif /* __HIREDIS_ASYNC_PRIVATE_H */
+50 -87
View File
@@ -5,7 +5,7 @@
* tables of power of two in size are used, collisions are handled by
* chaining. See the source code for more information... :)
*
* Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2006-2010, Redis Ltd.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -34,37 +34,25 @@
*/
#include "fmacros.h"
#include "dict.h"
#include "alloc.h"
#include <stdlib.h>
#include <assert.h>
#include <limits.h>
#include <stdlib.h>
#define UNUSED(x) (void)(x)
/* -------------------------- types ----------------------------------------- */
struct dictEntry {
void *key;
void *val;
struct dictEntry *next;
};
#include "dict.h"
/* -------------------------- private prototypes ---------------------------- */
static int _dictExpandIfNeeded(dict *ht);
static unsigned long _dictNextPower(unsigned long size);
static int _dictKeyIndex(dict *ht, const void *key);
static int _dictInit(dict *ht, dictType *type);
static int _dictInit(dict *ht, dictType *type, void *privDataPtr);
/* -------------------------- hash functions -------------------------------- */
/* Generic hash function (a popular one from Bernstein).
* I tested a few and this was the best. */
uint64_t dictGenHashFunction(const unsigned char *buf, int len) {
uint64_t hash = 5381;
static unsigned int dictGenHashFunction(const unsigned char *buf, int len) {
unsigned int hash = 5381;
while (len--)
hash = ((hash << 5) + hash) + (*buf++); /* hash * 33 + c */
@@ -73,7 +61,7 @@ uint64_t dictGenHashFunction(const unsigned char *buf, int len) {
/* ----------------------------- API implementation ------------------------- */
/* Reset a hashtable already initialized with ht_init().
/* Reset an hashtable already initialized with ht_init().
* NOTE: This function should only called by ht_destroy(). */
static void _dictReset(dict *ht) {
ht->table = NULL;
@@ -83,24 +71,25 @@ static void _dictReset(dict *ht) {
}
/* Create a new hash table */
dict *dictCreate(dictType *type) {
dict *ht = vk_malloc(sizeof(*ht));
static dict *dictCreate(dictType *type, void *privDataPtr) {
dict *ht = hi_malloc(sizeof(*ht));
if (ht == NULL)
return NULL;
_dictInit(ht, type);
_dictInit(ht,type,privDataPtr);
return ht;
}
/* Initialize the hash table */
static int _dictInit(dict *ht, dictType *type) {
static int _dictInit(dict *ht, dictType *type, void *privDataPtr) {
_dictReset(ht);
ht->type = type;
ht->privdata = privDataPtr;
return DICT_OK;
}
/* Expand or create the hashtable */
int dictExpand(dict *ht, unsigned long size) {
static int dictExpand(dict *ht, unsigned long size) {
dict n; /* the new hashtable */
unsigned long realsize = _dictNextPower(size), i;
@@ -109,26 +98,25 @@ int dictExpand(dict *ht, unsigned long size) {
if (ht->used > size)
return DICT_ERR;
_dictInit(&n, ht->type);
_dictInit(&n, ht->type, ht->privdata);
n.size = realsize;
n.sizemask = realsize - 1;
n.table = vk_calloc(realsize, sizeof(dictEntry *));
n.sizemask = realsize-1;
n.table = hi_calloc(realsize,sizeof(dictEntry*));
if (n.table == NULL)
return DICT_ERR;
/* Copy all the elements from the old to the new table:
* note that if the old hash table is empty ht->size is zero,
* so dictExpand just creates a hash table. */
* so dictExpand just creates an hash table. */
n.used = ht->used;
for (i = 0; i < ht->size && ht->used > 0; i++) {
dictEntry *he, *nextHe;
if (ht->table[i] == NULL)
continue;
if (ht->table[i] == NULL) continue;
/* For each hash entry on this slot... */
he = ht->table[i];
while (he) {
while(he) {
unsigned int h;
nextHe = he->next;
@@ -142,7 +130,7 @@ int dictExpand(dict *ht, unsigned long size) {
}
}
assert(ht->used == 0);
vk_free(ht->table);
hi_free(ht->table);
/* Remap the new hashtable in the old */
*ht = n;
@@ -150,7 +138,7 @@ int dictExpand(dict *ht, unsigned long size) {
}
/* Add an element to the target hash table */
int dictAdd(dict *ht, void *key, void *val) {
static int dictAdd(dict *ht, void *key, void *val) {
int index;
dictEntry *entry;
@@ -160,7 +148,7 @@ int dictAdd(dict *ht, void *key, void *val) {
return DICT_ERR;
/* Allocates the memory and stores key */
entry = vk_malloc(sizeof(*entry));
entry = hi_malloc(sizeof(*entry));
if (entry == NULL)
return DICT_ERR;
@@ -168,8 +156,8 @@ int dictAdd(dict *ht, void *key, void *val) {
ht->table[index] = entry;
/* Set the hash entry fields. */
dictSetKey(ht, entry, key);
dictSetVal(ht, entry, val);
dictSetHashKey(ht, entry, key);
dictSetHashVal(ht, entry, val);
ht->used++;
return DICT_OK;
}
@@ -178,7 +166,7 @@ int dictAdd(dict *ht, void *key, void *val) {
* Return 1 if the key was added from scratch, 0 if there was already an
* element with such key and dictReplace() just performed a value update
* operation. */
int dictReplace(dict *ht, void *key, void *val) {
static int dictReplace(dict *ht, void *key, void *val) {
dictEntry *entry, auxentry;
/* Try to add the element. If the key
@@ -197,13 +185,13 @@ int dictReplace(dict *ht, void *key, void *val) {
* you want to increment (set), and then decrement (free), and not the
* reverse. */
auxentry = *entry;
dictSetVal(ht, entry, val);
dictSetHashVal(ht, entry, val);
dictFreeEntryVal(ht, &auxentry);
return 0;
}
/* Search and remove an element */
int dictDelete(dict *ht, const void *key) {
static int dictDelete(dict *ht, const void *key) {
unsigned int h;
dictEntry *de, *prevde;
@@ -213,17 +201,17 @@ int dictDelete(dict *ht, const void *key) {
de = ht->table[h];
prevde = NULL;
while (de) {
if (dictCompareHashKeys(ht, key, de->key)) {
while(de) {
if (dictCompareHashKeys(ht,key,de->key)) {
/* Unlink the element from the list */
if (prevde)
prevde->next = de->next;
else
ht->table[h] = de->next;
dictFreeEntryKey(ht, de);
dictFreeEntryVal(ht, de);
vk_free(de);
dictFreeEntryKey(ht,de);
dictFreeEntryVal(ht,de);
hi_free(de);
ht->used--;
return DICT_OK;
}
@@ -241,41 +229,37 @@ static int _dictClear(dict *ht) {
for (i = 0; i < ht->size && ht->used > 0; i++) {
dictEntry *he, *nextHe;
if ((he = ht->table[i]) == NULL)
continue;
while (he) {
if ((he = ht->table[i]) == NULL) continue;
while(he) {
nextHe = he->next;
dictFreeEntryKey(ht, he);
dictFreeEntryVal(ht, he);
vk_free(he);
hi_free(he);
ht->used--;
he = nextHe;
}
}
/* Free the table and the allocated cache structure */
vk_free(ht->table);
hi_free(ht->table);
/* Re-initialize the table */
_dictReset(ht);
return DICT_OK; /* never fails */
}
/* Clear & Release the hash table */
void dictRelease(dict *ht) {
if (ht == NULL)
return;
static void dictRelease(dict *ht) {
_dictClear(ht);
vk_free(ht);
hi_free(ht);
}
dictEntry *dictFind(dict *ht, const void *key) {
static dictEntry *dictFind(dict *ht, const void *key) {
dictEntry *he;
unsigned int h;
if (ht->size == 0)
return NULL;
if (ht->size == 0) return NULL;
h = dictHashKey(ht, key) & ht->sizemask;
he = ht->table[h];
while (he) {
while(he) {
if (dictCompareHashKeys(ht, key, he->key))
return he;
he = he->next;
@@ -283,40 +267,19 @@ dictEntry *dictFind(dict *ht, const void *key) {
return NULL;
}
void dictSetKey(dict *d, dictEntry *de, void *key) {
if (d->type->keyDup)
de->key = d->type->keyDup(key);
else
de->key = key;
}
void dictSetVal(dict *d, dictEntry *de, void *val) {
UNUSED(d);
de->val = val;
}
void *dictGetKey(const dictEntry *de) {
return de->key;
}
void *dictGetVal(const dictEntry *de) {
return de->val;
}
void dictInitIterator(dictIterator *iter, dict *ht) {
static void dictInitIterator(dictIterator *iter, dict *ht) {
iter->ht = ht;
iter->index = -1;
iter->entry = NULL;
iter->nextEntry = NULL;
}
dictEntry *dictNext(dictIterator *iter) {
static dictEntry *dictNext(dictIterator *iter) {
while (1) {
if (iter->entry == NULL) {
iter->index++;
if (iter->index >=
(signed)iter->ht->size)
break;
(signed)iter->ht->size) break;
iter->entry = iter->ht->table[iter->index];
} else {
iter->entry = iter->nextEntry;
@@ -340,7 +303,7 @@ static int _dictExpandIfNeeded(dict *ht) {
if (ht->size == 0)
return dictExpand(ht, DICT_HT_INITIAL_SIZE);
if (ht->used == ht->size)
return dictExpand(ht, ht->size * 2);
return dictExpand(ht, ht->size*2);
return DICT_OK;
}
@@ -348,9 +311,8 @@ static int _dictExpandIfNeeded(dict *ht) {
static unsigned long _dictNextPower(unsigned long size) {
unsigned long i = DICT_HT_INITIAL_SIZE;
if (size >= LONG_MAX)
return LONG_MAX;
while (1) {
if (size >= LONG_MAX) return LONG_MAX;
while(1) {
if (i >= size)
return i;
i *= 2;
@@ -358,7 +320,7 @@ static unsigned long _dictNextPower(unsigned long size) {
}
/* Returns the index of a free slot that can be populated with
* a hash entry for the given 'key'.
* an hash entry for the given 'key'.
* If the key already exists, -1 is returned. */
static int _dictKeyIndex(dict *ht, const void *key) {
unsigned int h;
@@ -371,10 +333,11 @@ static int _dictKeyIndex(dict *ht, const void *key) {
h = dictHashKey(ht, key) & ht->sizemask;
/* Search if this slot does not already contain the given key */
he = ht->table[h];
while (he) {
while(he) {
if (dictCompareHashKeys(ht, key, he->key))
return -1;
he = he->next;
}
return h;
}
+51 -32
View File
@@ -5,7 +5,7 @@
* tables of power of two in size are used, collisions are handled by
* chaining. See the source code for more information... :)
*
* Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2006-2010, Redis Ltd.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -36,19 +36,25 @@
#ifndef __DICT_H
#define __DICT_H
#include <stdint.h>
#define DICT_OK 0
#define DICT_ERR 1
typedef struct dictEntry dictEntry; /* opaque */
/* Unused arguments generate annoying warnings... */
#define DICT_NOTUSED(V) ((void) V)
typedef struct dictEntry {
void *key;
void *val;
struct dictEntry *next;
} dictEntry;
typedef struct dictType {
uint64_t (*hashFunction)(const void *key);
void *(*keyDup)(const void *key);
int (*keyCompare)(const void *key1, const void *key2);
void (*keyDestructor)(void *key);
void (*valDestructor)(void *obj);
unsigned int (*hashFunction)(const void *key);
void *(*keyDup)(void *privdata, const void *key);
void *(*valDup)(void *privdata, const void *obj);
int (*keyCompare)(void *privdata, const void *key1, const void *key2);
void (*keyDestructor)(void *privdata, void *key);
void (*valDestructor)(void *privdata, void *obj);
} dictType;
typedef struct dict {
@@ -57,6 +63,7 @@ typedef struct dict {
unsigned long size;
unsigned long sizemask;
unsigned long used;
void *privdata;
} dict;
typedef struct dictIterator {
@@ -66,41 +73,53 @@ typedef struct dictIterator {
} dictIterator;
/* This is the initial size of every hash table */
#define DICT_HT_INITIAL_SIZE 4
#define DICT_HT_INITIAL_SIZE 4
/* ------------------------------- Macros ------------------------------------*/
#define dictFreeEntryVal(ht, entry) \
if ((ht)->type->valDestructor) \
(ht)->type->valDestructor(dictGetVal(entry))
if ((ht)->type->valDestructor) \
(ht)->type->valDestructor((ht)->privdata, (entry)->val)
#define dictSetHashVal(ht, entry, _val_) do { \
if ((ht)->type->valDup) \
entry->val = (ht)->type->valDup((ht)->privdata, _val_); \
else \
entry->val = (_val_); \
} while(0)
#define dictFreeEntryKey(ht, entry) \
if ((ht)->type->keyDestructor) \
(ht)->type->keyDestructor(dictGetKey(entry))
if ((ht)->type->keyDestructor) \
(ht)->type->keyDestructor((ht)->privdata, (entry)->key)
#define dictCompareHashKeys(ht, key1, key2) \
(((ht)->type->keyCompare) ? \
(ht)->type->keyCompare(key1, key2) : \
(key1) == (key2))
#define dictSetHashKey(ht, entry, _key_) do { \
if ((ht)->type->keyDup) \
entry->key = (ht)->type->keyDup((ht)->privdata, _key_); \
else \
entry->key = (_key_); \
} while(0)
#define dictCompareHashKeys(ht, key1, key2) \
(((ht)->type->keyCompare) ? \
(ht)->type->keyCompare((ht)->privdata, key1, key2) : \
(key1) == (key2))
#define dictHashKey(ht, key) (ht)->type->hashFunction(key)
#define dictGetEntryKey(he) ((he)->key)
#define dictGetEntryVal(he) ((he)->val)
#define dictSlots(ht) ((ht)->size)
#define dictSize(ht) ((ht)->used)
/* API */
uint64_t dictGenHashFunction(const unsigned char *buf, int len);
dict *dictCreate(dictType *type);
int dictExpand(dict *ht, unsigned long size);
int dictAdd(dict *ht, void *key, void *val);
int dictReplace(dict *ht, void *key, void *val);
int dictDelete(dict *ht, const void *key);
void dictRelease(dict *ht);
dictEntry *dictFind(dict *ht, const void *key);
void dictSetKey(dict *d, dictEntry *de, void *key);
void dictSetVal(dict *d, dictEntry *de, void *val);
void *dictGetKey(const dictEntry *de);
void *dictGetVal(const dictEntry *de);
void dictInitIterator(dictIterator *iter, dict *ht);
dictEntry *dictNext(dictIterator *iter);
static unsigned int dictGenHashFunction(const unsigned char *buf, int len);
static dict *dictCreate(dictType *type, void *privDataPtr);
static int dictExpand(dict *ht, unsigned long size);
static int dictAdd(dict *ht, void *key, void *val);
static int dictReplace(dict *ht, void *key, void *val);
static int dictDelete(dict *ht, const void *key);
static void dictRelease(dict *ht);
static dictEntry * dictFind(dict *ht, const void *key);
static void dictInitIterator(dictIterator *iter, dict *ht);
static dictEntry *dictNext(dictIterator *iter);
#endif /* __DICT_H */
+61
View File
@@ -0,0 +1,61 @@
INCLUDE(FindPkgConfig)
# Check for GLib
PKG_CHECK_MODULES(GLIB2 glib-2.0)
if (GLIB2_FOUND)
INCLUDE_DIRECTORIES(${GLIB2_INCLUDE_DIRS})
LINK_DIRECTORIES(${GLIB2_LIBRARY_DIRS})
ADD_EXECUTABLE(example-glib example-glib.c)
TARGET_LINK_LIBRARIES(example-glib hiredis ${GLIB2_LIBRARIES})
ENDIF(GLIB2_FOUND)
FIND_PATH(LIBEV ev.h
HINTS /usr/local /usr/opt/local
ENV LIBEV_INCLUDE_DIR)
if (LIBEV)
# Just compile and link with libev
ADD_EXECUTABLE(example-libev example-libev.c)
TARGET_LINK_LIBRARIES(example-libev hiredis ev)
ENDIF()
FIND_PATH(LIBEVENT event.h)
if (LIBEVENT)
ADD_EXECUTABLE(example-libevent example-libevent.c)
TARGET_LINK_LIBRARIES(example-libevent hiredis event)
ENDIF()
FIND_PATH(LIBHV hv/hv.h)
IF (LIBHV)
ADD_EXECUTABLE(example-libhv example-libhv.c)
TARGET_LINK_LIBRARIES(example-libhv hiredis hv)
ENDIF()
FIND_PATH(LIBUV uv.h)
IF (LIBUV)
ADD_EXECUTABLE(example-libuv example-libuv.c)
TARGET_LINK_LIBRARIES(example-libuv hiredis uv)
ENDIF()
FIND_PATH(LIBSDEVENT systemd/sd-event.h)
IF (LIBSDEVENT)
ADD_EXECUTABLE(example-libsdevent example-libsdevent.c)
TARGET_LINK_LIBRARIES(example-libsdevent hiredis systemd)
ENDIF()
IF (APPLE)
FIND_LIBRARY(CF CoreFoundation)
ADD_EXECUTABLE(example-macosx example-macosx.c)
TARGET_LINK_LIBRARIES(example-macosx hiredis ${CF})
ENDIF()
IF (ENABLE_SSL)
ADD_EXECUTABLE(example-ssl example-ssl.c)
TARGET_LINK_LIBRARIES(example-ssl hiredis hiredis_ssl)
ENDIF()
ADD_EXECUTABLE(example example.c)
TARGET_LINK_LIBRARIES(example hiredis)
ADD_EXECUTABLE(example-push example-push.c)
TARGET_LINK_LIBRARIES(example-push hiredis)
+62
View File
@@ -0,0 +1,62 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
#include <hiredis.h>
#include <async.h>
#include <adapters/ae.h>
/* Put event loop in the global scope, so it can be explicitly stopped */
static aeEventLoop *loop;
void getCallback(redisAsyncContext *c, void *r, void *privdata) {
redisReply *reply = r;
if (reply == NULL) return;
printf("argv[%s]: %s\n", (char*)privdata, reply->str);
/* Disconnect after receiving the reply to GET */
redisAsyncDisconnect(c);
}
void connectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
aeStop(loop);
return;
}
printf("Connected...\n");
}
void disconnectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
aeStop(loop);
return;
}
printf("Disconnected...\n");
aeStop(loop);
}
int main (int argc, char **argv) {
signal(SIGPIPE, SIG_IGN);
redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
if (c->err) {
/* Let *c leak for now... */
printf("Error: %s\n", c->errstr);
return 1;
}
loop = aeCreateEventLoop(64);
redisAeAttach(loop, c);
redisAsyncSetConnectCallback(c,connectCallback);
redisAsyncSetDisconnectCallback(c,disconnectCallback);
redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
aeMain(loop);
return 0;
}
+73
View File
@@ -0,0 +1,73 @@
#include <stdlib.h>
#include <hiredis.h>
#include <async.h>
#include <adapters/glib.h>
static GMainLoop *mainloop;
static void
connect_cb (const redisAsyncContext *ac G_GNUC_UNUSED,
int status)
{
if (status != REDIS_OK) {
g_printerr("Failed to connect: %s\n", ac->errstr);
g_main_loop_quit(mainloop);
} else {
g_printerr("Connected...\n");
}
}
static void
disconnect_cb (const redisAsyncContext *ac G_GNUC_UNUSED,
int status)
{
if (status != REDIS_OK) {
g_error("Failed to disconnect: %s", ac->errstr);
} else {
g_printerr("Disconnected...\n");
g_main_loop_quit(mainloop);
}
}
static void
command_cb(redisAsyncContext *ac,
gpointer r,
gpointer user_data G_GNUC_UNUSED)
{
redisReply *reply = r;
if (reply) {
g_print("REPLY: %s\n", reply->str);
}
redisAsyncDisconnect(ac);
}
gint
main (gint argc G_GNUC_UNUSED,
gchar *argv[] G_GNUC_UNUSED)
{
redisAsyncContext *ac;
GMainContext *context = NULL;
GSource *source;
ac = redisAsyncConnect("127.0.0.1", 6379);
if (ac->err) {
g_printerr("%s\n", ac->errstr);
exit(EXIT_FAILURE);
}
source = redis_source_new(ac);
mainloop = g_main_loop_new(context, FALSE);
g_source_attach(source, context);
redisAsyncSetConnectCallback(ac, connect_cb);
redisAsyncSetDisconnectCallback(ac, disconnect_cb);
redisAsyncCommand(ac, command_cb, NULL, "SET key 1234");
redisAsyncCommand(ac, command_cb, NULL, "GET key");
g_main_loop_run(mainloop);
return EXIT_SUCCESS;
}
+60
View File
@@ -0,0 +1,60 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
#include <hiredis.h>
#include <async.h>
#include <adapters/ivykis.h>
void getCallback(redisAsyncContext *c, void *r, void *privdata) {
redisReply *reply = r;
if (reply == NULL) return;
printf("argv[%s]: %s\n", (char*)privdata, reply->str);
/* Disconnect after receiving the reply to GET */
redisAsyncDisconnect(c);
}
void connectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
printf("Connected...\n");
}
void disconnectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
printf("Disconnected...\n");
}
int main (int argc, char **argv) {
#ifndef _WIN32
signal(SIGPIPE, SIG_IGN);
#endif
iv_init();
redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
if (c->err) {
/* Let *c leak for now... */
printf("Error: %s\n", c->errstr);
return 1;
}
redisIvykisAttach(c);
redisAsyncSetConnectCallback(c,connectCallback);
redisAsyncSetDisconnectCallback(c,disconnectCallback);
redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
iv_main();
iv_deinit();
return 0;
}
+54
View File
@@ -0,0 +1,54 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
#include <hiredis.h>
#include <async.h>
#include <adapters/libev.h>
void getCallback(redisAsyncContext *c, void *r, void *privdata) {
redisReply *reply = r;
if (reply == NULL) return;
printf("argv[%s]: %s\n", (char*)privdata, reply->str);
/* Disconnect after receiving the reply to GET */
redisAsyncDisconnect(c);
}
void connectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
printf("Connected...\n");
}
void disconnectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
printf("Disconnected...\n");
}
int main (int argc, char **argv) {
#ifndef _WIN32
signal(SIGPIPE, SIG_IGN);
#endif
redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
if (c->err) {
/* Let *c leak for now... */
printf("Error: %s\n", c->errstr);
return 1;
}
redisLibevAttach(EV_DEFAULT_ c);
redisAsyncSetConnectCallback(c,connectCallback);
redisAsyncSetDisconnectCallback(c,disconnectCallback);
redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
ev_loop(EV_DEFAULT_ 0);
return 0;
}
+90
View File
@@ -0,0 +1,90 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
#include <hiredis.h>
#include <hiredis_ssl.h>
#include <async.h>
#include <adapters/libevent.h>
void getCallback(redisAsyncContext *c, void *r, void *privdata) {
redisReply *reply = r;
if (reply == NULL) return;
printf("argv[%s]: %s\n", (char*)privdata, reply->str);
/* Disconnect after receiving the reply to GET */
redisAsyncDisconnect(c);
}
void connectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
printf("Connected...\n");
}
void disconnectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
printf("Disconnected...\n");
}
int main (int argc, char **argv) {
#ifndef _WIN32
signal(SIGPIPE, SIG_IGN);
#endif
struct event_base *base = event_base_new();
if (argc < 5) {
fprintf(stderr,
"Usage: %s <key> <host> <port> <cert> <certKey> [ca]\n", argv[0]);
exit(1);
}
const char *value = argv[1];
size_t nvalue = strlen(value);
const char *hostname = argv[2];
int port = atoi(argv[3]);
const char *cert = argv[4];
const char *certKey = argv[5];
const char *caCert = argc > 5 ? argv[6] : NULL;
redisSSLContext *ssl;
redisSSLContextError ssl_error = REDIS_SSL_CTX_NONE;
redisInitOpenSSL();
ssl = redisCreateSSLContext(caCert, NULL,
cert, certKey, NULL, &ssl_error);
if (!ssl) {
printf("Error: %s\n", redisSSLContextGetError(ssl_error));
return 1;
}
redisAsyncContext *c = redisAsyncConnect(hostname, port);
if (c->err) {
/* Let *c leak for now... */
printf("Error: %s\n", c->errstr);
return 1;
}
if (redisInitiateSSLWithContext(&c->c, ssl) != REDIS_OK) {
printf("SSL Error!\n");
exit(1);
}
redisLibeventAttach(c,base);
redisAsyncSetConnectCallback(c,connectCallback);
redisAsyncSetDisconnectCallback(c,disconnectCallback);
redisAsyncCommand(c, NULL, NULL, "SET key %b", value, nvalue);
redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
event_base_dispatch(base);
redisFreeSSLContext(ssl);
return 0;
}
+67
View File
@@ -0,0 +1,67 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
#include <hiredis.h>
#include <async.h>
#include <adapters/libevent.h>
void getCallback(redisAsyncContext *c, void *r, void *privdata) {
redisReply *reply = r;
if (reply == NULL) {
if (c->errstr) {
printf("errstr: %s\n", c->errstr);
}
return;
}
printf("argv[%s]: %s\n", (char*)privdata, reply->str);
/* Disconnect after receiving the reply to GET */
redisAsyncDisconnect(c);
}
void connectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
printf("Connected...\n");
}
void disconnectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
printf("Disconnected...\n");
}
int main (int argc, char **argv) {
#ifndef _WIN32
signal(SIGPIPE, SIG_IGN);
#endif
struct event_base *base = event_base_new();
redisOptions options = {0};
REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379);
struct timeval tv = {0};
tv.tv_sec = 1;
options.connect_timeout = &tv;
redisAsyncContext *c = redisAsyncConnectWithOptions(&options);
if (c->err) {
/* Let *c leak for now... */
printf("Error: %s\n", c->errstr);
return 1;
}
redisLibeventAttach(c,base);
redisAsyncSetConnectCallback(c,connectCallback);
redisAsyncSetDisconnectCallback(c,disconnectCallback);
redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
event_base_dispatch(base);
return 0;
}
+70
View File
@@ -0,0 +1,70 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
#include <hiredis.h>
#include <async.h>
#include <adapters/libhv.h>
void getCallback(redisAsyncContext *c, void *r, void *privdata) {
redisReply *reply = r;
if (reply == NULL) return;
printf("argv[%s]: %s\n", (char*)privdata, reply->str);
/* Disconnect after receiving the reply to GET */
redisAsyncDisconnect(c);
}
void debugCallback(redisAsyncContext *c, void *r, void *privdata) {
(void)privdata;
redisReply *reply = r;
if (reply == NULL) {
printf("`DEBUG SLEEP` error: %s\n", c->errstr ? c->errstr : "unknown error");
return;
}
redisAsyncDisconnect(c);
}
void connectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
printf("Connected...\n");
}
void disconnectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
printf("Disconnected...\n");
}
int main (int argc, char **argv) {
#ifndef _WIN32
signal(SIGPIPE, SIG_IGN);
#endif
redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
if (c->err) {
/* Let *c leak for now... */
printf("Error: %s\n", c->errstr);
return 1;
}
hloop_t* loop = hloop_new(HLOOP_FLAG_QUIT_WHEN_NO_ACTIVE_EVENTS);
redisLibhvAttach(c, loop);
redisAsyncSetTimeout(c, (struct timeval){.tv_sec = 0, .tv_usec = 500000});
redisAsyncSetConnectCallback(c,connectCallback);
redisAsyncSetDisconnectCallback(c,disconnectCallback);
redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
redisAsyncCommand(c, debugCallback, NULL, "DEBUG SLEEP %d", 1);
hloop_run(loop);
hloop_free(&loop);
return 0;
}
@@ -1,71 +1,69 @@
#define _XOPEN_SOURCE 600 /* Required by libsdevent (CLOCK_MONOTONIC) */
#include <kv/async.h>
#include <kv/kv.h>
#include <kv/adapters/libsdevent.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
void debugCallback(kvAsyncContext *c, void *r, void *privdata) {
#include <hiredis.h>
#include <async.h>
#include <adapters/libsdevent.h>
void debugCallback(redisAsyncContext *c, void *r, void *privdata) {
(void)privdata;
kvReply *reply = r;
redisReply *reply = r;
if (reply == NULL) {
/* The DEBUG SLEEP command will almost always fail, because we have set a 1 second timeout */
printf("`DEBUG SLEEP` error: %s\n", c->errstr ? c->errstr : "unknown error");
return;
}
/* Disconnect after receiving the reply of DEBUG SLEEP (which will not)*/
kvAsyncDisconnect(c);
redisAsyncDisconnect(c);
}
void getCallback(kvAsyncContext *c, void *r, void *privdata) {
kvReply *reply = r;
void getCallback(redisAsyncContext *c, void *r, void *privdata) {
redisReply *reply = r;
if (reply == NULL) {
printf("`GET key` error: %s\n", c->errstr ? c->errstr : "unknown error");
return;
}
printf("`GET key` result: argv[%s]: %s\n", (char *)privdata, reply->str);
printf("`GET key` result: argv[%s]: %s\n", (char*)privdata, reply->str);
/* start another request that demonstrate timeout */
kvAsyncCommand(c, debugCallback, NULL, "DEBUG SLEEP %f", 1.5);
redisAsyncCommand(c, debugCallback, NULL, "DEBUG SLEEP %f", 1.5);
}
void connectCallback(kvAsyncContext *c, int status) {
if (status != KV_OK) {
void connectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("connect error: %s\n", c->errstr);
return;
}
printf("Connected...\n");
}
void disconnectCallback(const kvAsyncContext *c, int status) {
if (status != KV_OK) {
void disconnectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("disconnect because of error: %s\n", c->errstr);
return;
}
printf("Disconnected...\n");
}
int main(int argc, char **argv) {
int main (int argc, char **argv) {
signal(SIGPIPE, SIG_IGN);
struct sd_event *event;
sd_event_default(&event);
kvAsyncContext *c = kvAsyncConnect("127.0.0.1", 6379);
redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
if (c->err) {
printf("Error: %s\n", c->errstr);
kvAsyncFree(c);
redisAsyncFree(c);
return 1;
}
kvLibsdeventAttach(c, event);
kvAsyncSetConnectCallback(c, connectCallback);
kvAsyncSetDisconnectCallback(c, disconnectCallback);
kvAsyncSetTimeout(c, (struct timeval){.tv_sec = 1, .tv_usec = 0});
redisLibsdeventAttach(c,event);
redisAsyncSetConnectCallback(c,connectCallback);
redisAsyncSetDisconnectCallback(c,disconnectCallback);
redisAsyncSetTimeout(c, (struct timeval){ .tv_sec = 1, .tv_usec = 0});
/*
In this demo, we first `set key`, then `get key` to demonstrate the basic usage of libsdevent adapter.
@@ -74,9 +72,8 @@ int main(int argc, char **argv) {
timeout error, which is shown in the `debugCallback`.
*/
kvAsyncCommand(
c, NULL, NULL, "SET key %b", argv[argc - 1], strlen(argv[argc - 1]));
kvAsyncCommand(c, getCallback, (char *)"end-1", "GET key");
redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
/* sd-event does not quit when there are no handlers registered. Manually exit after 1.5 seconds */
sd_event_source *s;
@@ -1,72 +1,70 @@
#define _XOPEN_SOURCE 600 /* Required by libuv (pthread_rwlock_t) */
#include <kv/async.h>
#include <kv/kv.h>
#include <kv/adapters/libuv.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
void debugCallback(kvAsyncContext *c, void *r, void *privdata) {
#include <hiredis.h>
#include <async.h>
#include <adapters/libuv.h>
void debugCallback(redisAsyncContext *c, void *r, void *privdata) {
(void)privdata; //unused
kvReply *reply = r;
redisReply *reply = r;
if (reply == NULL) {
/* The DEBUG SLEEP command will almost always fail, because we have set a 1 second timeout */
printf("`DEBUG SLEEP` error: %s\n", c->errstr ? c->errstr : "unknown error");
return;
}
/* Disconnect after receiving the reply of DEBUG SLEEP (which will not)*/
kvAsyncDisconnect(c);
redisAsyncDisconnect(c);
}
void getCallback(kvAsyncContext *c, void *r, void *privdata) {
kvReply *reply = r;
void getCallback(redisAsyncContext *c, void *r, void *privdata) {
redisReply *reply = r;
if (reply == NULL) {
printf("`GET key` error: %s\n", c->errstr ? c->errstr : "unknown error");
return;
}
printf("`GET key` result: argv[%s]: %s\n", (char *)privdata, reply->str);
printf("`GET key` result: argv[%s]: %s\n", (char*)privdata, reply->str);
/* start another request that demonstrate timeout */
kvAsyncCommand(c, debugCallback, NULL, "DEBUG SLEEP %f", 1.5);
redisAsyncCommand(c, debugCallback, NULL, "DEBUG SLEEP %f", 1.5);
}
void connectCallback(kvAsyncContext *c, int status) {
if (status != KV_OK) {
void connectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("connect error: %s\n", c->errstr);
return;
}
printf("Connected...\n");
}
void disconnectCallback(const kvAsyncContext *c, int status) {
if (status != KV_OK) {
void disconnectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("disconnect because of error: %s\n", c->errstr);
return;
}
printf("Disconnected...\n");
}
int main(int argc, char **argv) {
int main (int argc, char **argv) {
#ifndef _WIN32
signal(SIGPIPE, SIG_IGN);
#endif
uv_loop_t *loop = uv_default_loop();
uv_loop_t* loop = uv_default_loop();
kvAsyncContext *c = kvAsyncConnect("127.0.0.1", 6379);
redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
if (c->err) {
/* Let *c leak for now... */
printf("Error: %s\n", c->errstr);
return 1;
}
kvLibuvAttach(c, loop);
kvAsyncSetConnectCallback(c, connectCallback);
kvAsyncSetDisconnectCallback(c, disconnectCallback);
kvAsyncSetTimeout(c, (struct timeval){.tv_sec = 1, .tv_usec = 0});
redisLibuvAttach(c,loop);
redisAsyncSetConnectCallback(c,connectCallback);
redisAsyncSetDisconnectCallback(c,disconnectCallback);
redisAsyncSetTimeout(c, (struct timeval){ .tv_sec = 1, .tv_usec = 0});
/*
In this demo, we first `set key`, then `get key` to demonstrate the basic usage of libuv adapter.
@@ -75,9 +73,8 @@ int main(int argc, char **argv) {
timeout error, which is shown in the `debugCallback`.
*/
kvAsyncCommand(
c, NULL, NULL, "SET key %b", argv[argc - 1], strlen(argv[argc - 1]));
kvAsyncCommand(c, getCallback, (char *)"end-1", "GET key");
redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
uv_run(loop, UV_RUN_DEFAULT);
return 0;
+66
View File
@@ -0,0 +1,66 @@
//
// Created by Дмитрий Бахвалов on 13.07.15.
// Copyright (c) 2015 Dmitry Bakhvalov. All rights reserved.
//
#include <stdio.h>
#include <hiredis.h>
#include <async.h>
#include <adapters/macosx.h>
void getCallback(redisAsyncContext *c, void *r, void *privdata) {
redisReply *reply = r;
if (reply == NULL) return;
printf("argv[%s]: %s\n", (char*)privdata, reply->str);
/* Disconnect after receiving the reply to GET */
redisAsyncDisconnect(c);
}
void connectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
printf("Connected...\n");
}
void disconnectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
CFRunLoopStop(CFRunLoopGetCurrent());
printf("Disconnected...\n");
}
int main (int argc, char **argv) {
signal(SIGPIPE, SIG_IGN);
CFRunLoopRef loop = CFRunLoopGetCurrent();
if( !loop ) {
printf("Error: Cannot get current run loop\n");
return 1;
}
redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
if (c->err) {
/* Let *c leak for now... */
printf("Error: %s\n", c->errstr);
return 1;
}
redisMacOSAttach(c, loop);
redisAsyncSetConnectCallback(c,connectCallback);
redisAsyncSetDisconnectCallback(c,disconnectCallback);
redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
CFRunLoopRun();
return 0;
}
+62
View File
@@ -0,0 +1,62 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
#include <unistd.h>
#include <async.h>
#include <adapters/poll.h>
/* Put in the global scope, so that loop can be explicitly stopped */
static int exit_loop = 0;
void getCallback(redisAsyncContext *c, void *r, void *privdata) {
redisReply *reply = r;
if (reply == NULL) return;
printf("argv[%s]: %s\n", (char*)privdata, reply->str);
/* Disconnect after receiving the reply to GET */
redisAsyncDisconnect(c);
}
void connectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
exit_loop = 1;
return;
}
printf("Connected...\n");
}
void disconnectCallback(const redisAsyncContext *c, int status) {
exit_loop = 1;
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
printf("Disconnected...\n");
}
int main (int argc, char **argv) {
signal(SIGPIPE, SIG_IGN);
redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
if (c->err) {
/* Let *c leak for now... */
printf("Error: %s\n", c->errstr);
return 1;
}
redisPollAttach(c);
redisAsyncSetConnectCallback(c,connectCallback);
redisAsyncSetDisconnectCallback(c,disconnectCallback);
redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
while (!exit_loop)
{
redisPollTick(c, 0.1);
}
return 0;
}
@@ -27,29 +27,26 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <kv/kv.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <hiredis.h>
#define KEY_COUNT 5
#define panicAbort(fmt, ...) \
do { \
fprintf( \
stderr, \
"%s:%d:%s(): " fmt, __FILE__, __LINE__, __func__, __VA_ARGS__); \
exit(-1); \
#define panicAbort(fmt, ...) \
do { \
fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, __LINE__, __func__, __VA_ARGS__); \
exit(-1); \
} while (0)
static void assertReplyAndFree(kvContext *context, kvReply *reply, int type) {
static void assertReplyAndFree(redisContext *context, redisReply *reply, int type) {
if (reply == NULL)
panicAbort("NULL reply from server (error: %s)", context->errstr);
if (reply->type != type) {
if (reply->type == KV_REPLY_ERROR)
fprintf(stderr, "Server Error: %s\n", reply->str);
if (reply->type == REDIS_REPLY_ERROR)
fprintf(stderr, "Redis Error: %s\n", reply->str);
panicAbort("Expected reply type %d but got type %d", type, reply->type);
}
@@ -58,34 +55,35 @@ static void assertReplyAndFree(kvContext *context, kvReply *reply, int type) {
}
/* Switch to the RESP3 protocol and enable client tracking */
static void enableClientTracking(kvContext *c) {
kvReply *reply = kvCommand(c, "HELLO 3");
static void enableClientTracking(redisContext *c) {
redisReply *reply = redisCommand(c, "HELLO 3");
if (reply == NULL || c->err) {
panicAbort("NULL reply or server error (error: %s)", c->errstr);
}
if (reply->type != KV_REPLY_MAP) {
if (reply->type != REDIS_REPLY_MAP) {
fprintf(stderr, "Error: Can't send HELLO 3 command. Are you sure you're ");
fprintf(stderr, "connected to kv-server >= 6.0.0?\nServer error: %s\n",
reply->type == KV_REPLY_ERROR ? reply->str : "(unknown)");
fprintf(stderr, "connected to redis-server >= 6.0.0?\nRedis error: %s\n",
reply->type == REDIS_REPLY_ERROR ? reply->str : "(unknown)");
exit(-1);
}
freeReplyObject(reply);
/* Enable client tracking */
reply = kvCommand(c, "CLIENT TRACKING ON");
assertReplyAndFree(c, reply, KV_REPLY_STATUS);
reply = redisCommand(c, "CLIENT TRACKING ON");
assertReplyAndFree(c, reply, REDIS_REPLY_STATUS);
}
void pushReplyHandler(void *privdata, void *r) {
kvReply *reply = r;
redisReply *reply = r;
int *invalidations = privdata;
/* Sanity check on the invalidation reply */
if (reply->type != KV_REPLY_PUSH || reply->elements != 2 ||
reply->element[1]->type != KV_REPLY_ARRAY ||
reply->element[1]->element[0]->type != KV_REPLY_STRING) {
if (reply->type != REDIS_REPLY_PUSH || reply->elements != 2 ||
reply->element[1]->type != REDIS_REPLY_ARRAY ||
reply->element[1]->element[0]->type != REDIS_REPLY_STRING)
{
panicAbort("%s", "Can't parse PUSH message!");
}
@@ -99,7 +97,7 @@ void pushReplyHandler(void *privdata, void *r) {
}
/* We aren't actually freeing anything here, but it is included to show that we can
* have libkv call our data destructor when freeing the context */
* have hiredis call our data destructor when freeing the context */
void privdata_dtor(void *privdata) {
unsigned int *icount = privdata;
printf("privdata_dtor(): In context privdata dtor (invalidations: %u)\n", *icount);
@@ -107,29 +105,29 @@ void privdata_dtor(void *privdata) {
int main(int argc, char **argv) {
unsigned int j, invalidations = 0;
kvContext *c;
kvReply *reply;
redisContext *c;
redisReply *reply;
const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1";
int port = (argc > 2) ? atoi(argv[2]) : 6379;
kvOptions o = {0};
KV_OPTIONS_SET_TCP(&o, hostname, port);
redisOptions o = {0};
REDIS_OPTIONS_SET_TCP(&o, hostname, port);
/* Set our context privdata to the address of our invalidation counter. Each
* time our PUSH handler is called, libkv will pass the privdata for context.
* time our PUSH handler is called, hiredis will pass the privdata for context.
*
* This could also be done after we create the context like so:
*
* c->privdata = &invalidations;
* c->free_privdata = privdata_dtor;
*/
KV_OPTIONS_SET_PRIVDATA(&o, &invalidations, privdata_dtor);
REDIS_OPTIONS_SET_PRIVDATA(&o, &invalidations, privdata_dtor);
/* Set our custom PUSH message handler */
o.push_cb = pushReplyHandler;
c = kvConnectWithOptions(&o);
c = redisConnectWithOptions(&o);
if (c == NULL || c->err)
panicAbort("Connection error: %s", c ? c->errstr : "OOM");
@@ -139,23 +137,23 @@ int main(int argc, char **argv) {
/* Set some keys and then read them back. Once we do that, Redis will deliver
* invalidation push messages whenever the key is modified */
for (j = 0; j < KEY_COUNT; j++) {
reply = kvCommand(c, "SET key:%d initial:%d", j, j);
assertReplyAndFree(c, reply, KV_REPLY_STATUS);
reply = redisCommand(c, "SET key:%d initial:%d", j, j);
assertReplyAndFree(c, reply, REDIS_REPLY_STATUS);
reply = kvCommand(c, "GET key:%d", j);
assertReplyAndFree(c, reply, KV_REPLY_STRING);
reply = redisCommand(c, "GET key:%d", j);
assertReplyAndFree(c, reply, REDIS_REPLY_STRING);
}
/* Trigger invalidation messages by updating keys we just read */
for (j = 0; j < KEY_COUNT; j++) {
printf(" main(): SET key:%d update:%d\n", j, j);
reply = kvCommand(c, "SET key:%d update:%d", j, j);
assertReplyAndFree(c, reply, KV_REPLY_STATUS);
reply = redisCommand(c, "SET key:%d update:%d", j, j);
assertReplyAndFree(c, reply, REDIS_REPLY_STATUS);
printf(" main(): SET REPLY OK\n");
}
printf("\nTotal detected invalidations: %d, expected: %d\n", invalidations, KEY_COUNT);
/* PING server */
kvFree(c);
redisFree(c);
}
+46
View File
@@ -0,0 +1,46 @@
#include <iostream>
using namespace std;
#include <QCoreApplication>
#include <QTimer>
#include "example-qt.h"
void getCallback(redisAsyncContext *, void * r, void * privdata) {
redisReply * reply = static_cast<redisReply *>(r);
ExampleQt * ex = static_cast<ExampleQt *>(privdata);
if (reply == nullptr || ex == nullptr) return;
cout << "key: " << reply->str << endl;
ex->finish();
}
void ExampleQt::run() {
m_ctx = redisAsyncConnect("localhost", 6379);
if (m_ctx->err) {
cerr << "Error: " << m_ctx->errstr << endl;
redisAsyncFree(m_ctx);
emit finished();
}
m_adapter.setContext(m_ctx);
redisAsyncCommand(m_ctx, NULL, NULL, "SET key %s", m_value);
redisAsyncCommand(m_ctx, getCallback, this, "GET key");
}
int main (int argc, char **argv) {
QCoreApplication app(argc, argv);
ExampleQt example(argv[argc-1]);
QObject::connect(&example, SIGNAL(finished()), &app, SLOT(quit()));
QTimer::singleShot(0, &example, SLOT(run()));
return app.exec();
}

Some files were not shown because too many files have changed in this diff Show More