Redis 7.4.3 (#13963 )

Redis 7.4.3
Limiting output buffer for unauthenticated client (CVE-2025-21605)
2025-04-23 14:54:51 +03:00 · 2025-04-23 10:26:24 +03:00 · 2025-04-23 10:18:06 +03:00 · 2025-04-22 14:53:32 +03:00 · 2025-04-22 12:16:02 +03:00 · 2025-04-22 10:50:48 +03:00
43 changed files with 934 additions and 195 deletions
@@ -91,8 +91,8 @@ jobs:
      run: |
        apt-get update
        apt-get install -y gnupg2
-        echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial main" >> /etc/apt/sources.list
-        echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial universe" >> /etc/apt/sources.list
+        echo "deb http://archive.ubuntu.com/ubuntu/ xenial main" >> /etc/apt/sources.list
+        echo "deb http://archive.ubuntu.com/ubuntu/ xenial universe" >> /etc/apt/sources.list
        apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5
        apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32
        apt-get update
@@ -76,7 +76,6 @@ jobs:
    if: |
      (github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
      !contains(github.event.inputs.skipjobs, 'fortify')
-    container: ubuntu:lunar
    timeout-minutes: 14400
    steps:
    - name: prep
@@ -94,11 +93,10 @@ jobs:
        ref: ${{ env.GITHUB_HEAD_REF }}
    - name: make
      run: |
-        apt-get update && apt-get install -y make gcc-13
-        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100
+        apt-get update && apt-get install -y make gcc
        make CC=gcc REDIS_CFLAGS='-Werror -DREDIS_TEST -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3'
    - name: testprep
-      run: apt-get install -y tcl8.6 tclx procps
+      run: sudo apt-get install -y tcl8.6 tclx procps
    - name: test
      if: true && !contains(github.event.inputs.skiptests, 'redis')
      run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
@@ -875,7 +873,7 @@ jobs:
  build-macos:
    strategy:
      matrix:
-        os: [macos-12, macos-14]
+        os: [macos-13, macos-15]
    runs-on: ${{ matrix.os }}
    if: |
      (github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
@@ -902,7 +900,7 @@ jobs:
      run: make REDIS_CFLAGS='-Werror -DREDIS_TEST'

  test-freebsd:
-    runs-on: macos-12
+    runs-on: macos-13
    if: |
      (github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
      !contains(github.event.inputs.skipjobs, 'freebsd')
@@ -1075,8 +1073,8 @@ jobs:
      run: |
        apt-get update
        apt-get install -y gnupg2
-        echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial main" >> /etc/apt/sources.list
-        echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial universe" >> /etc/apt/sources.list
+        echo "deb http://archive.ubuntu.com/ubuntu/ xenial main" >> /etc/apt/sources.list
+        echo "deb http://archive.ubuntu.com/ubuntu/ xenial universe" >> /etc/apt/sources.list
        apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5
        apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32
        apt-get update
@@ -1123,8 +1121,8 @@ jobs:
      run: |
        apt-get update
        apt-get install -y gnupg2
-        echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial main" >> /etc/apt/sources.list
-        echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial universe" >> /etc/apt/sources.list
+        echo "deb http://archive.ubuntu.com/ubuntu/ xenial main" >> /etc/apt/sources.list
+        echo "deb http://archive.ubuntu.com/ubuntu/ xenial universe" >> /etc/apt/sources.list
        apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5
        apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32
        apt-get update
@@ -1177,8 +1175,8 @@ jobs:
      run: |
        apt-get update
        apt-get install -y gnupg2 
-        echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial main" >> /etc/apt/sources.list
-        echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial universe" >> /etc/apt/sources.list
+        echo "deb http://archive.ubuntu.com/ubuntu/ xenial main" >> /etc/apt/sources.list
+        echo "deb http://archive.ubuntu.com/ubuntu/ xenial universe" >> /etc/apt/sources.list
        apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5
        apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32
        apt-get update
@@ -27,7 +27,7 @@ jobs:
            --tags -slow
    - name: Archive redis log
      if: ${{ failure() }}
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
      with:
        name: test-external-redis-log
        path: external-redis.log
@@ -55,7 +55,7 @@ jobs:
            --tags -slow
    - name: Archive redis log
      if: ${{ failure() }}
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
      with:
        name: test-external-cluster-log
        path: external-redis-cluster.log
@@ -79,7 +79,7 @@ jobs:
            --tags "-slow -needs:debug"
      - name: Archive redis log
        if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
        with:
          name: test-external-redis-nodebug-log
          path: external-redis-nodebug.log
@@ -1,16 +1,221 @@
-Hello! This file is just a placeholder, since this is the "unstable" branch
-of Redis, the place where all the development happens.
+Redis Community Edition 7.4 release notes
+=========================================

-There is no release notes for this branch, it gets forked into another branch
-every time there is a partial feature freeze in order to eventually create
-a new stable release.
+--------------------------------------------------------------------------------
+Upgrade urgency levels:

-Usually "unstable" is stable enough for you to use it in development environments
-however you should never use it in production environments. It is possible
-to download the latest stable release here:
+LOW:      No need to upgrade unless there are new features you want to use.
+MODERATE: Program an upgrade of the server, but it's not urgent.
+HIGH:     There is a critical bug that may affect a subset of users. Upgrade!
+CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP.
+SECURITY: There are security fixes in the release.
+--------------------------------------------------------------------------------

-    https://download.redis.io/redis-stable.tar.gz
+================================================================================
+Redis 7.4.3    Released Wed 23 Apr 2025 12:00:00 IST
+================================================================================

-More information is available at https://redis.io
+Update urgency: `SECURITY`: There are security fixes in the release.
+
+### Security fixes
+
+- (CVE-2025-21605) An unauthenticated client can cause an unlimited growth of output buffers
+
+### Bug fixes
+
+- #13661 `FUNCTION FLUSH` - memory leak when using jemalloc
+- #13793 `WAITAOF` returns prematurely
+- #13853 `SLAVEOF` - crash when clients are blocked on lazy free
+- #13863 `RANDOMKEY` - infinite loop during client pause
+- #13877 ShardID inconsistency when both primary and replica support it
+
+
+================================================================================
+Redis Community Edition 7.4.2    Released Mon 6 Jan 2025 12:30:00 IDT
+================================================================================
+
+Upgrade urgency SECURITY: See security fixes below.
+
+### Security fixes
+
+- (CVE-2024-46981) Lua script commands may lead to remote code execution
+- (CVE-2024-51741) Denial-of-service due to malformed ACL selectors
+
+### Bug fixes
+
+- #13627 Crash on module memory defragmentation
+- #13338 Streams: `XINFO` lag field is wrong when tombstone is after the `last_id` of the consume group
+- #13473 Streams: `XTRIM` does not update the maximal tombstone, leading to an incorrect lag
+- #13470 `INFO` after `HDEL` show wrong number of hash keys with expiration
+- #13476 Fix a race condition in the `cache_memory` of `functionsLibCtx`
+- #13626 Memory leak on failed RDB loading
+- #13539 Hash: fix key ref for a hash that no longer has fields with expiration on `RENAME`/`MOVE`/`SWAPDB`/`RESTORE`
+- #13443 Cluster: crash when loading cluster config
+- #13422 Cluster: `CLUSTER SHARDS` returns empty array
+- #13465 Cluster: incompatibility with older node versions
+- #13608 Cluster: `SORT ... GET #`: incorrect error message
+
+
+================================================================================
+Redis Community Edition 7.4.1    Released Wed 02 Oct 2024 20:17:04 IDT
+================================================================================
+
+Upgrade urgency SECURITY: See security fixes below.
+
+Security fixes
+==============
+
+* (CVE-2024-31449) Lua library commands may lead to stack overflow and potential RCE.
+* (CVE-2024-31227) Potential Denial-of-service due to malformed ACL selectors.
+* (CVE-2024-31228) Potential Denial-of-service due to unbounded pattern matching.
+
+================================================================================
+Redis Community Edition 7.4.0 GA   Released Mon Jul 29 2024 9:00:00 IDT
+================================================================================
+
+This is the General Availability release of Redis Community Edition 7.4.
+
+Changes to new 7.4 features (compared to 7.4 RC2)
+=================================================
+* #13391,#13438 Hash - expiration of individual fields: RDB file format changes
+* #13372 Hash - expiration of individual fields: rename and fix counting of `expired_subkeys` metric
+* #13372 Hash - expiration of individual fields: rename `INFO` keyspace field to `subexpiry`
+
+Configuration parameters
+========================
+* #13400 Add hide-user-data-from-log - allows hiding user data from the log file
+
+Bug fixes
+=========
+* #13407 trigger Lua GC after `SCRIPT LOAD`
+* #13380 Fix possible crash due to OOM panic on invalid command
+* #13383 `FUNCTION FLUSH` - improve Lua GC behavior and fix thread race in ASYNC mode
+* #13408 `HEXPIRE`-like commands should emit HDEL keyspace notification if expire time is in the past
+
+
+================================================================================
+Redis Community Edition 7.4 RC2   Released Thu 27 Jun 2024 10:00:00 IST
+================================================================================
+
+Upgrade urgency LOW: This is the second Release Candidate for Redis Community Edition 7.4.
+
+Performance and resource utilization improvements
+=================================================
+* #13296 Optimize CPU cache efficiency
+
+Changes to new 7.4 new features (compared to 7.4 RC1)
+=====================================================
+* #13343 Hash - expiration of individual fields: when key does not exist - reply with an array (nonexisting code for each field)
+* #13329 Hash - expiration of individual fields: new keyspace event: `hexpired`
+
+Modules API - Potentially breaking changes to new 7.4 features (compared to 7.4 RC1)
+====================================================================================
+* #13326 Hash - expiration of individual fields: avoid lazy expire when called from a Modules API function
+
+
+================================================================================
+Redis Community Edition 7.4 RC1   Released Thu 6 Jun 2024 10:00:00 IST
+================================================================================
+
+Note: License changed - see LICENSE.txt
+
+Upgrade urgency LOW: This is the first Release Candidate for Redis Community Edition 7.4.
+
+Here is a comprehensive list of changes in this release compared to 7.2.5.
+
+New Features
+============
+* #13303 Hash - expiration of individual fields. 9 commands were introduced:
+  - `HEXPIRE` and `HPEXPIRE` set the remaining time to live for specific fields
+  - `HEXPIREAT` and `HPEXPIREAT` set the expiration time to a UNIX timestamp for specific fields
+  - `HPERSIST` removes the expiration for specific fields
+  - `HEXPIRETIME` and `HPEXPIRETIME` get the expiration time for specific fields
+  - `HTTL` and `HPTTL` get the remaining time to live for specific fields
+* #13117 `XREAD`: new id value `+` to start reading from the last message
+* #12765 `HSCAN`: new [NOVALUES] flag to report only field names
+* #12728 `SORT`, `SORT_RO`: allow `BY` and `GET` options in cluster mode when the pattern maps to the same slot as the key
+* #12299 `CLIENT KILL`: new optional filter: `MAXAGE maxage` - retain connections older than `maxage` seconds
+* #12971 Lua: expose `os.clock()` API for getting the elapsed time of Lua code execution
+* #13276 Allow `SPUBLISH` command within `MULTI ... EXEC` transactions on replica
+
+Bug fixes
+=========
+* #12898 `XREADGROUP`: fix entries-read inconsistency between master and replicas
+* #13042 `SORT ... STORE`: fix created lists to respect list compression and packing configs
+* #12817, #12905 Fix race condition issues between the main thread and module threads
+* #12577 Unsubscribe all clients from replica for shard channel if the master ownership changes
+* #12622 `WAITAOF` could timeout or hang if used after a module command that propagated effects only to replicas and not to AOF
+* #11734 `BITCOUNT` and `BITPOS` with nonexistent key and illegal arguments return an error, not 0
+* #12394 `BITCOUNT`: check for wrong argument before checking if key exists
+* #12961 Allow execution of read-only transactions when out of memory
+* #13274 Fix crash when a client performs ACL change that disconnects itself
+* #13311 Cluster: Fix crash due to unblocking client during slot migration
+
+Security improvements
+=====================
+* #13108 Lua: LRU eviction for scripts generated with `EVAL` *** BEHAVIOR CHANGE ***
+* #12961 Restrict the total request size of `MULTI ... EXEC` transactions
+* #12860 Redact ACL username information and mark '*-key-file-pass configs' as sensitive
+
+Performance and resource utilization improvements
+=================================================
+* #12838 Improve performance when many clients call `PUNSUBSCRIBE` / `SUNSUBSCRIBE` simultaneously
+* #12627 Reduce lag when waking `WAITAOF` clients and there is not much traffic
+* #12754 Optimize `KEYS` when pattern includes hashtag and implies a single slot
+* #11695 Reduce memory and improve performance by replacing cluster metadata with slot specific dictionaries
+* #13087 `SCRIPT FLUSH ASYNC` now does not block the main thread
+* #12996 Active memory defragmentation efficiency improvements
+* #12899 Improve performance of read/update operation during rehashing
+* #12536 `SCAN ... MATCH`: Improve performance when the pattern implies cluster slot
+* #12450 `ZRANGE ... LIMIT`: improved performance
+
+Other general improvements
+==========================
+* #13133 Lua: allocate VM code with jemalloc instead of libc and count it as used memory *** BEHAVIOR CHANGE ***
+* #12171 `ACL LOAD`: do not disconnect all clients *** BEHAVIOR CHANGE ***
+* #13020 Allow adjusting defrag configurations while active defragmentation is running
+* #12949 Increase the accuracy of avg_ttl (the average keyspace keys TTL)
+* #12977 Allow running `WAITAOF` in scripts
+* #12782 Implement TCP keep-alive across most Unix-like systems
+* #12707 Improved error codes when rejecting scripts in cluster mode
+* #12596 Support `XREAD ... BLOCK` in scripts; rejected only if it ends up blocking
+
+New metrics
+===========
+* #12849 `INFO`: `pubsub_clients` - number of clients in Pub/Sub mode
+* #12966 `INFO`: `watching_clients` - number of clients that are watching keys
+* #12966 `INFO`: `total_watched_keys` - number of watched keys
+* #12476 `INFO`: `client_query_buffer_limit_disconnections` - count client input buffer OOM events
+* #12476 `INFO`: `client_output_buffer_limit_disconnections` - count client output buffer OOM events
+* #12996 `INFO`: `allocator_muzzy` - memory returned to the OS but still shows as RSS until the OS reclaims it
+* #13108 `INFO`: `evicted_scripts` - number of evicted eval scripts. Users can check it to see if they are abusing EVAL
+* #12996 `MEMORY STATS`: `allocator.muzzy` - memory returned to the OS but still shows as RSS until the OS reclaims it
+* #12913 `INFO MEMORY` `mem_overhead_db_hashtable_rehashing` - memory resharding overhead (only the memory that will be released soon)
+* #12913 `MEMORY STATS': `overhead.db.hashtable.lut` - total overhead of dictionary buckets in databases
+* #12913 `MEMORY STATS': `overhead.db.hashtable.rehashing` - temporary memory overhead of database dictionaries currently being rehashed 
+* #12913 `MEMORY STATS': `db.dict.rehashing.count` - number of top level dictionaries currently being rehashed
+* #12966 `CLIENT LIST`: `watch` - number of keys each client is currently watching
+
+Modules API
+===========
+* #12985 New API calls: `RM_TryCalloc` and `RM_TryRealloc` - allow modules to handle memory allocation failures gracefully
+* #13069 New API call: `RM_ClusterKeySlot` - which slot a key will hash to
+* #13069 New API call: `RM_ClusterCanonicalKeyNameInSlot` - get a consistent key that will map to a slot 
+* #12486 New API call: `RM_AddACLCategory` - allow modules to declare new ACL categories
+
+Configuration parameters
+========================
+* #12178 New configuration parameters: `max-new-connections-per-cycle` and `max-new-tls-connections-per-cycle` to limit the number of new client connections per event-loop cycle
+* #7351 Rename some CPU configuration parameters for style alignment. Added alias to the old names to avoid breaking change
+
+CLI tools
+=========
+* #10609 redis-cli: new `-t <timeout>` argument: specify server connection timeout in seconds
+* #11315 redis-cli: new `-4` and `-6` flags to prefer IPV4 or IPV6 on DNS lookup
+* #12862 redis-cli: allows pressing up arrow to return any command (including sensitive commands which are still not persisted)
+* #12543 redis-cli: add reverse history search (like Ctrl+R in terminals)
+* #12826 redis-cli: add `--keystats` and `--keystats-samples` to combines `--memkeys` and `--bigkeys` with additional distribution data
+* #12735 redis-cli: fix: `--bigkeys` and `--memkeys` now work on cluster replicas
+* #9411 redis-benchmark: add support for binary strings
+* #12986 redis-benchmark: fix: pick random slot for a node to distribute operation across slots

-Happy hacking!
@@ -132,6 +132,7 @@ static int bit_tohex(lua_State *L)
  const char *hexdigits = "0123456789abcdef";
  char buf[8];
  int i;
+  if (n == INT32_MIN) n = INT32_MIN+1;
  if (n < 0) { n = -n; hexdigits = "0123456789ABCDEF"; }
  if (n > 8) n = 8;
  for (i = (int)n; --i >= 0; ) { buf[i] = hexdigits[b & 15]; b >>= 4; }
@@ -110,6 +110,9 @@ else
 ifeq ($(SANITIZER),undefined)
 	MALLOC=libc
 	CFLAGS+=-fsanitize=undefined -fno-sanitize-recover=all -fno-omit-frame-pointer
+	ifeq (clang,$(CLANG))
+		CFLAGS+=-fno-sanitize=function
+	endif
 	LDFLAGS+=-fsanitize=undefined
 else
 ifeq ($(SANITIZER),thread)
@@ -1061,6 +1061,7 @@ int ACLSetSelector(aclSelector *selector, const char* op, size_t oplen) {
        int flags = 0;
        size_t offset = 1;
        if (op[0] == '%') {
+            int perm_ok = 1;
            for (; offset < oplen; offset++) {
                if (toupper(op[offset]) == 'R' && !(flags & ACL_READ_PERMISSION)) {
                    flags |= ACL_READ_PERMISSION;
@@ -1070,10 +1071,14 @@ int ACLSetSelector(aclSelector *selector, const char* op, size_t oplen) {
                    offset++;
                    break;
                } else {
-                    errno = EINVAL;
-                    return C_ERR;
+                    perm_ok = 0;
+                    break;
                }
            }
+            if (!flags || !perm_ok) {
+                errno = EINVAL;
+                return C_ERR;
+            }
        } else {
            flags = ACL_ALL_PERMISSION;
        }
@@ -1048,35 +1048,34 @@ void flushAppendOnlyFile(int force) {
    mstime_t latency;

    if (sdslen(server.aof_buf) == 0) {
-        /* Check if we need to do fsync even the aof buffer is empty,
-         * because previously in AOF_FSYNC_EVERYSEC mode, fsync is
-         * called only when aof buffer is not empty, so if users
-         * stop write commands before fsync called in one second,
-         * the data in page cache cannot be flushed in time. */
-        if (server.aof_fsync == AOF_FSYNC_EVERYSEC &&
-            server.aof_last_incr_fsync_offset != server.aof_last_incr_size &&
-            server.mstime - server.aof_last_fsync >= 1000 &&
-            !(sync_in_progress = aofFsyncInProgress())) {
-            goto try_fsync;
-
-        /* Check if we need to do fsync even the aof buffer is empty,
-         * the reason is described in the previous AOF_FSYNC_EVERYSEC block,
-         * and AOF_FSYNC_ALWAYS is also checked here to handle a case where
-         * aof_fsync is changed from everysec to always. */
-        } else if (server.aof_fsync == AOF_FSYNC_ALWAYS &&
-                   server.aof_last_incr_fsync_offset != server.aof_last_incr_size)
-        {
-            goto try_fsync;
-        } else {
+        if (server.aof_last_incr_fsync_offset == server.aof_last_incr_size) {
            /* All data is fsync'd already: Update fsynced_reploff_pending just in case.
-             * This is needed to avoid a WAITAOF hang in case a module used RM_Call with the NO_AOF flag,
-             * in which case master_repl_offset will increase but fsynced_reploff_pending won't be updated
-             * (because there's no reason, from the AOF POV, to call fsync) and then WAITAOF may wait on
-             * the higher offset (which contains data that was only propagated to replicas, and not to AOF) */
-            if (!sync_in_progress && server.aof_fsync != AOF_FSYNC_NO)
+             * This is needed to avoid a WAITAOF hang in case a module used RM_Call
+             * with the NO_AOF flag, in which case master_repl_offset will increase but
+             * fsynced_reploff_pending won't be updated (because there's no reason, from
+             * the AOF POV, to call fsync) and then WAITAOF may wait on the higher offset
+             * (which contains data that was only propagated to replicas, and not to AOF) */
+            if (!aofFsyncInProgress())
                atomicSet(server.fsynced_reploff_pending, server.master_repl_offset);
-            return;
+        } else {
+            /* Check if we need to do fsync even the aof buffer is empty,
+             * because previously in AOF_FSYNC_EVERYSEC mode, fsync is
+             * called only when aof buffer is not empty, so if users
+             * stop write commands before fsync called in one second,
+             * the data in page cache cannot be flushed in time. */
+            if (server.aof_fsync == AOF_FSYNC_EVERYSEC &&
+                server.mstime - server.aof_last_fsync >= 1000 &&
+                !(sync_in_progress = aofFsyncInProgress()))
+                goto try_fsync;
+
+            /* Check if we need to do fsync even the aof buffer is empty,
+             * the reason is described in the previous AOF_FSYNC_EVERYSEC block,
+             * and AOF_FSYNC_ALWAYS is also checked here to handle a case where
+             * aof_fsync is changed from everysec to always. */
+            if (server.aof_fsync == AOF_FSYNC_ALWAYS)
+                goto try_fsync;
        }
+        return;
    }

    if (server.aof_fsync == AOF_FSYNC_EVERYSEC)
@@ -270,6 +270,7 @@ void disconnectAllBlockedClients(void) {

            if (c->bstate.btype == BLOCKED_LAZYFREE) {
                addReply(c, shared.ok); /* No reason lazy-free to fail */
+                updateStatsOnUnblock(c, 0, 0, 0);
                c->flags &= ~CLIENT_PENDING_COMMAND;
                unblockClient(c, 1);
            } else {
@@ -2,8 +2,13 @@
 * Copyright (c) 2009-Present, Redis Ltd.
 * All rights reserved.
 *
+ * Copyright (c) 2024-present, Valkey contributors.
+ * All rights reserved.
+ *
 * Licensed under your choice of the Redis Source Available License 2.0
 * (RSALv2) or the Server Side Public License v1 (SSPLv1).
+ *
+ * Portions of this file are available under BSD3 terms; see REDISCONTRIBUTIONS for more information.
 */

 /*
@@ -88,6 +93,7 @@ int auxTlsPortPresent(clusterNode *n);
 static void clusterBuildMessageHdr(clusterMsg *hdr, int type, size_t msglen);
 void freeClusterLink(clusterLink *link);
 int verifyClusterNodeId(const char *name, int length);
+static void updateShardId(clusterNode *node, const char *shard_id);

 int getNodeDefaultClientPort(clusterNode *n) {
    return server.tls_cluster ? n->tls_port : n->tcp_port;
@@ -198,12 +204,11 @@ int auxShardIdSetter(clusterNode *n, void *value, int length) {
        return C_ERR;
    }
    memcpy(n->shard_id, value, CLUSTER_NAMELEN);
-    /* if n already has replicas, make sure they all agree
-     * on the shard id */
+    /* if n already has replicas, make sure they all use
+     * the primary shard id */
    for (int i = 0; i < n->numslaves; i++) {
-        if (memcmp(n->slaves[i]->shard_id, n->shard_id, CLUSTER_NAMELEN) != 0) {
-            return C_ERR;
-        }
+        if (memcmp(n->slaves[i]->shard_id, n->shard_id, CLUSTER_NAMELEN) != 0)
+            updateShardId(n->slaves[i], n->shard_id);
    }
    clusterAddNodeToShard(value, n);
    return C_OK;
@@ -545,18 +550,12 @@ int clusterLoadConfig(char *filename) {
                clusterAddNode(master);
            }
            /* shard_id can be absent if we are loading a nodes.conf generated
-             * by an older version of Redis; we should follow the primary's
-             * shard_id in this case */
-            if (auxFieldHandlers[af_shard_id].isPresent(n) == 0) {
-                memcpy(n->shard_id, master->shard_id, CLUSTER_NAMELEN);
-                clusterAddNodeToShard(master->shard_id, n);
-            } else if (clusterGetNodesInMyShard(master) != NULL &&
-                       memcmp(master->shard_id, n->shard_id, CLUSTER_NAMELEN) != 0)
-            {
-                /* If the primary has been added to a shard, make sure this
-                 * node has the same persisted shard id as the primary. */
-                goto fmterr;
-            }
+             * by an older version of Redis; 
+             * ignore replica's shard_id in the file, only use the primary's.
+             * If replica precedes primary in file, it will be corrected
+             * later by the auxShardIdSetter */
+            memcpy(n->shard_id, master->shard_id, CLUSTER_NAMELEN);
+            clusterAddNodeToShard(master->shard_id, n);
            n->slaveof = master;
            clusterNodeAddSlave(master,n);
        } else if (auxFieldHandlers[af_shard_id].isPresent(n) == 0) {
@@ -634,6 +633,8 @@ int clusterLoadConfig(char *filename) {
    }
    /* Config sanity check */
    if (server.cluster->myself == NULL) goto fmterr;
+    if (!(myself->flags & (CLUSTER_NODE_MASTER | CLUSTER_NODE_SLAVE))) goto fmterr;
+    if (nodeIsSlave(myself) && myself->slaveof == NULL) goto fmterr;

    zfree(line);
    fclose(fp);
@@ -901,22 +902,33 @@ static void updateAnnouncedHumanNodename(clusterNode *node, char *new) {
    clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG);
 }

+static void assignShardIdToNode(clusterNode *node, const char *shard_id, int flag) {
+    clusterRemoveNodeFromShard(node);
+    memcpy(node->shard_id, shard_id, CLUSTER_NAMELEN);
+    clusterAddNodeToShard(shard_id, node);
+    clusterDoBeforeSleep(flag); 
+}

 static void updateShardId(clusterNode *node, const char *shard_id) {
    if (shard_id && memcmp(node->shard_id, shard_id, CLUSTER_NAMELEN) != 0) {
-        clusterRemoveNodeFromShard(node);
-        memcpy(node->shard_id, shard_id, CLUSTER_NAMELEN);
-        clusterAddNodeToShard(shard_id, node);
-        clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG);
-    }
-    if (shard_id && myself != node && myself->slaveof == node) {
-        if (memcmp(myself->shard_id, shard_id, CLUSTER_NAMELEN) != 0) {
-            /* shard-id can diverge right after a rolling upgrade
-             * from pre-7.2 releases */
-            clusterRemoveNodeFromShard(myself);
-            memcpy(myself->shard_id, shard_id, CLUSTER_NAMELEN);
-            clusterAddNodeToShard(shard_id, myself);
-            clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|CLUSTER_TODO_FSYNC_CONFIG);
+        /* We always make our best effort to keep the shard-id consistent
+         * between the master and its replicas:
+         *
+         * 1. When updating the master's shard-id, we simultaneously update the
+         *    shard-id of all its replicas to ensure consistency.
+         * 2. When updating replica's shard-id, if it differs from its master's shard-id,
+         *    we discard this replica's shard-id and continue using master's shard-id.
+         *    This applies even if the master does not support shard-id, in which
+         *    case we rely on the master's randomly generated shard-id. */
+        if (node->slaveof == NULL) {
+            assignShardIdToNode(node, shard_id, CLUSTER_TODO_SAVE_CONFIG);
+            for (int i = 0; i < clusterNodeNumSlaves(node); i++) {
+                clusterNode *slavenode = clusterNodeGetSlave(node, i);
+                if (memcmp(slavenode->shard_id, shard_id, CLUSTER_NAMELEN) != 0)
+                    assignShardIdToNode(slavenode, shard_id, CLUSTER_TODO_SAVE_CONFIG|CLUSTER_TODO_FSYNC_CONFIG);
+            }
+        } else if (memcmp(node->slaveof->shard_id, shard_id, CLUSTER_NAMELEN) == 0) {
+            assignShardIdToNode(node, shard_id, CLUSTER_TODO_SAVE_CONFIG);
        }
    }
 }
@@ -1624,6 +1636,22 @@ void clusterRemoveNodeFromShard(clusterNode *node) {
    sdsfree(s);
 }

+static clusterNode *clusterGetMasterFromShard(list *nodes) {
+    clusterNode *n = NULL;
+    listIter li;
+    listNode *ln;
+    listRewind(nodes,&li);
+    while ((ln = listNext(&li)) != NULL) {
+        clusterNode *node = listNodeValue(ln);
+        if (!nodeFailed(node)) {
+            n = node;
+            break;
+        }
+    }
+    if (!n) return NULL;
+    return clusterNodeGetMaster(n);
+}
+
 /* -----------------------------------------------------------------------------
 * CLUSTER config epoch handling
 * -------------------------------------------------------------------------- */
@@ -2577,9 +2605,6 @@ uint32_t writePingExt(clusterMsg *hdr, int gossipcount)  {
    extensions++;

    if (hdr != NULL) {
-        if (extensions != 0) {
-            hdr->mflags[0] |= CLUSTERMSG_FLAG0_EXT_DATA;
-        }
        hdr->extensions = htons(extensions);
    }

@@ -2769,6 +2794,9 @@ int clusterProcessPacket(clusterLink *link) {
    }

    sender = getNodeFromLinkAndMsg(link, hdr);
+    if (sender && (hdr->mflags[0] & CLUSTERMSG_FLAG0_EXT_DATA)) {
+        sender->flags |= CLUSTER_NODE_EXTENSIONS_SUPPORTED;
+    }

    /* Update the last time we saw any data from this node. We
     * use this in order to avoid detecting a timeout from a node that
@@ -3534,6 +3562,8 @@ static void clusterBuildMessageHdr(clusterMsg *hdr, int type, size_t msglen) {
    /* Set the message flags. */
    if (clusterNodeIsMaster(myself) && server.cluster->mf_end)
        hdr->mflags[0] |= CLUSTERMSG_FLAG0_PAUSED;
+    hdr->mflags[0] |= CLUSTERMSG_FLAG0_EXT_DATA; /* Always make other nodes know that
+                                                  * this node supports extension data. */

    hdr->totlen = htonl(msglen);
 }
@@ -3612,7 +3642,9 @@ void clusterSendPing(clusterLink *link, int type) {
     * to put inside the packet. */
    estlen = sizeof(clusterMsg) - sizeof(union clusterMsgData);
    estlen += (sizeof(clusterMsgDataGossip)*(wanted + pfail_wanted));
-    estlen += writePingExt(NULL, 0);
+    if (link->node && nodeSupportsExtensions(link->node)) {
+        estlen += writePingExt(NULL, 0);
+    }
    /* Note: clusterBuildMessageHdr() expects the buffer to be always at least
     * sizeof(clusterMsg) or more. */
    if (estlen < (int)sizeof(clusterMsg)) estlen = sizeof(clusterMsg);
@@ -3682,7 +3714,9 @@ void clusterSendPing(clusterLink *link, int type) {

    /* Compute the actual total length and send! */
    uint32_t totlen = 0;
-    totlen += writePingExt(hdr, gossipcount);
+    if (link->node && nodeSupportsExtensions(link->node)) {
+        totlen += writePingExt(hdr, gossipcount);
+    }
    totlen += sizeof(clusterMsg)-sizeof(union clusterMsgData);
    totlen += (sizeof(clusterMsgDataGossip)*gossipcount);
    serverAssert(gossipcount < USHRT_MAX);
@@ -5649,14 +5683,13 @@ void addNodeDetailsToShardReply(client *c, clusterNode *node) {
 /* Add the shard reply of a single shard based off the given primary node. */
 void addShardReplyForClusterShards(client *c, list *nodes) {
    serverAssert(listLength(nodes) > 0);
-    clusterNode *n = listNodeValue(listFirst(nodes));
+
    addReplyMapLen(c, 2);
    addReplyBulkCString(c, "slots");

    /* Use slot_info_pairs from the primary only */
-    n = clusterNodeGetMaster(n);
-
-    if (n->slot_info_pairs != NULL) {
+    clusterNode *n = clusterGetMasterFromShard(nodes);
+    if (n && n->slot_info_pairs != NULL) {
        serverAssert((n->slot_info_pairs_count % 2) == 0);
        addReplyArrayLen(c, n->slot_info_pairs_count);
        for (int i = 0; i < n->slot_info_pairs_count; i++)
@@ -1,3 +1,16 @@
+/*
+ * Copyright (c) 2009-Present, Redis Ltd.
+ * All rights reserved.
+ *
+ * Copyright (c) 2024-present, Valkey contributors.
+ * All rights reserved.
+ *
+ * Licensed under your choice of the Redis Source Available License 2.0
+ * (RSALv2) or the Server Side Public License v1 (SSPLv1).
+ *
+ * Portions of this file are available under BSD3 terms; see REDISCONTRIBUTIONS for more information.
+ */
+
 #ifndef CLUSTER_LEGACY_H
 #define CLUSTER_LEGACY_H

@@ -51,6 +64,7 @@ typedef struct clusterLink {
 #define CLUSTER_NODE_MEET 128     /* Send a MEET message to this node */
 #define CLUSTER_NODE_MIGRATE_TO 256 /* Master eligible for replica migration. */
 #define CLUSTER_NODE_NOFAILOVER 512 /* Slave will not try to failover. */
+#define CLUSTER_NODE_EXTENSIONS_SUPPORTED 1024 /* This node supports extensions. */
 #define CLUSTER_NODE_NULL_NAME "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"

 #define nodeIsSlave(n) ((n)->flags & CLUSTER_NODE_SLAVE)
@@ -59,6 +73,7 @@ typedef struct clusterLink {
 #define nodeTimedOut(n) ((n)->flags & CLUSTER_NODE_PFAIL)
 #define nodeFailed(n) ((n)->flags & CLUSTER_NODE_FAIL)
 #define nodeCantFailover(n) ((n)->flags & CLUSTER_NODE_NOFAILOVER)
+#define nodeSupportsExtensions(n) ((n)->flags & CLUSTER_NODE_EXTENSIONS_SUPPORTED)

 /* This structure represent elements of node->fail_reports. */
 typedef struct clusterNodeFailReport {
@@ -348,7 +348,7 @@ robj *dbRandomKey(redisDb *db) {
        key = dictGetKey(de);
        keyobj = createStringObject(key,sdslen(key));
        if (dbFindExpires(db, key)) {
-            if (allvolatile && server.masterhost && --maxtries == 0) {
+            if (allvolatile && (server.masterhost || isPausedActions(PAUSE_ACTION_EXPIRE)) && --maxtries == 0) {
                /* If the DB is composed only of keys with an expire set,
                 * it could happen that all the keys are already logically
                 * expired in the slave, so the function cannot stop because
@@ -702,7 +702,7 @@ void flushallSyncBgDone(uint64_t client_id) {
    client *c = lookupClientByID(client_id);

    /* Verify that client still exists */
-    if (!c) return;
+    if (!(c && c->flags & CLIENT_BLOCKED)) return;

    /* Update current_client (Called functions might rely on it) */
    client *old_client = server.current_client;
@@ -394,8 +394,7 @@ long scanLaterList(robj *ob, unsigned long *cursor, long long endtime) {
    quicklistNode *node;
    long iterations = 0;
    int bookmark_failed = 0;
-    if (ob->type != OBJ_LIST || ob->encoding != OBJ_ENCODING_QUICKLIST)
-        return 0;
+    serverAssert(ob->type == OBJ_LIST && ob->encoding == OBJ_ENCODING_QUICKLIST);

    if (*cursor == 0) {
        /* if cursor is 0, we start new iteration */
@@ -444,8 +443,7 @@ void scanLaterZsetCallback(void *privdata, const dictEntry *_de) {
 }

 void scanLaterZset(robj *ob, unsigned long *cursor) {
-    if (ob->type != OBJ_ZSET || ob->encoding != OBJ_ENCODING_SKIPLIST)
-        return;
+    serverAssert(ob->type == OBJ_ZSET && ob->encoding == OBJ_ENCODING_SKIPLIST);
    zset *zs = (zset*)ob->ptr;
    dict *d = zs->dict;
    scanLaterZsetData data = {zs};
@@ -461,8 +459,7 @@ void scanCallbackCountScanned(void *privdata, const dictEntry *de) {
 }

 void scanLaterSet(robj *ob, unsigned long *cursor) {
-    if (ob->type != OBJ_SET || ob->encoding != OBJ_ENCODING_HT)
-        return;
+    serverAssert(ob->type == OBJ_SET && ob->encoding == OBJ_ENCODING_HT);
    dict *d = ob->ptr;
    dictDefragFunctions defragfns = {
        .defragAlloc = activeDefragAlloc,
@@ -472,8 +469,7 @@ void scanLaterSet(robj *ob, unsigned long *cursor) {
 }

 void scanLaterHash(robj *ob, unsigned long *cursor) {
-    if (ob->type != OBJ_HASH || ob->encoding != OBJ_ENCODING_HT)
-        return;
+    serverAssert(ob->type == OBJ_HASH && ob->encoding == OBJ_ENCODING_HT);
    dict *d = ob->ptr;
    dictDefragFunctions defragfns = {
        .defragAlloc = activeDefragAlloc,
@@ -568,10 +564,7 @@ int scanLaterStreamListpacks(robj *ob, unsigned long *cursor, long long endtime)
    static unsigned char last[sizeof(streamID)];
    raxIterator ri;
    long iterations = 0;
-    if (ob->type != OBJ_STREAM || ob->encoding != OBJ_ENCODING_STREAM) {
-        *cursor = 0;
-        return 0;
-    }
+    serverAssert(ob->type == OBJ_STREAM && ob->encoding == OBJ_ENCODING_STREAM);

    stream *s = ob->ptr;
    raxStart(&ri,s->rax);
@@ -718,8 +711,9 @@ void defragStream(redisDb *db, dictEntry *kde) {
 void defragModule(redisDb *db, dictEntry *kde) {
    robj *obj = dictGetVal(kde);
    serverAssert(obj->type == OBJ_MODULE);
-
-    if (!moduleDefragValue(dictGetKey(kde), obj, db->id))
+    robj keyobj;
+    initStaticStringObject(keyobj, dictGetKey(kde));
+    if (!moduleDefragValue(&keyobj, obj, db->id))
        defragLater(db, kde);
 }

@@ -918,20 +912,22 @@ void defragOtherGlobals(void) {
 int defragLaterItem(dictEntry *de, unsigned long *cursor, long long endtime, int dbid) {
    if (de) {
        robj *ob = dictGetVal(de);
-        if (ob->type == OBJ_LIST) {
+        if (ob->type == OBJ_LIST && ob->encoding == OBJ_ENCODING_QUICKLIST) {
            return scanLaterList(ob, cursor, endtime);
-        } else if (ob->type == OBJ_SET) {
+        } else if (ob->type == OBJ_SET && ob->encoding == OBJ_ENCODING_HT) {
            scanLaterSet(ob, cursor);
-        } else if (ob->type == OBJ_ZSET) {
+        } else if (ob->type == OBJ_ZSET && ob->encoding == OBJ_ENCODING_SKIPLIST) {
            scanLaterZset(ob, cursor);
-        } else if (ob->type == OBJ_HASH) {
+        } else if (ob->type == OBJ_HASH && ob->encoding == OBJ_ENCODING_HT) {
            scanLaterHash(ob, cursor);
-        } else if (ob->type == OBJ_STREAM) {
+        } else if (ob->type == OBJ_STREAM && ob->encoding == OBJ_ENCODING_STREAM) {
            return scanLaterStreamListpacks(ob, cursor, endtime);
        } else if (ob->type == OBJ_MODULE) {
-            return moduleLateDefrag(dictGetKey(de), ob, cursor, endtime, dbid);
+            robj keyobj;
+            initStaticStringObject(keyobj, dictGetKey(de));
+            return moduleLateDefrag(&keyobj, ob, cursor, endtime, dbid);
        } else {
-            *cursor = 0; /* object type may have changed since we schedule it for later */
+            *cursor = 0; /* object type/encoding may have changed since we schedule it for later */
        }
    } else {
        *cursor = 0; /* object may have been deleted already */
@@ -259,12 +259,17 @@ void scriptingInit(int setup) {
 void freeLuaScriptsSync(dict *lua_scripts, list *lua_scripts_lru_list, lua_State *lua) {
    dictRelease(lua_scripts);
    listRelease(lua_scripts_lru_list);
-    lua_close(lua);

 #if defined(USE_JEMALLOC)
    /* When lua is closed, destroy the previously used private tcache. */
    void *ud = (global_State*)G(lua)->ud;
    unsigned int lua_tcache = (unsigned int)(uintptr_t)ud;
+#endif
+
+    lua_gc(lua, LUA_GCCOLLECT, 0);
+    lua_close(lua);
+
+#if defined(USE_JEMALLOC)
    je_mallctl("tcache.destroy", NULL, NULL, (void *)&lua_tcache, sizeof(unsigned int));
 #endif
 }
@@ -23,6 +23,9 @@
 #include <lua.h>
 #include <lauxlib.h>
 #include <lualib.h>
+#if defined(USE_JEMALLOC)
+#include <lstate.h>
+#endif

 #define LUA_ENGINE_NAME "LUA"
 #define REGISTRY_ENGINE_CTX_NAME "__ENGINE_CTX__"
@@ -189,8 +192,19 @@ static void luaEngineFreeFunction(void *engine_ctx, void *compiled_function) {

 static void luaEngineFreeCtx(void *engine_ctx) {
    luaEngineCtx *lua_engine_ctx = engine_ctx;
+#if defined(USE_JEMALLOC)
+    /* When lua is closed, destroy the previously used private tcache. */
+    void *ud = (global_State*)G(lua_engine_ctx->lua)->ud;
+    unsigned int lua_tcache = (unsigned int)(uintptr_t)ud;
+#endif
+
+    lua_gc(lua_engine_ctx->lua, LUA_GCCOLLECT, 0);
    lua_close(lua_engine_ctx->lua);
    zfree(lua_engine_ctx);
+
+#if defined(USE_JEMALLOC)
+    je_mallctl("tcache.destroy", NULL, NULL, (void *)&lua_tcache, sizeof(unsigned int));
+#endif
 }

 static void luaRegisterFunctionArgsInitialize(registerFunctionArgs *register_f_args,
@@ -171,7 +171,7 @@ void functionsLibCtxClear(functionsLibCtx *lib_ctx) {
        stats->n_lib = 0;
    }
    dictReleaseIterator(iter);
-    curr_functions_lib_ctx->cache_memory = 0;
+    lib_ctx->cache_memory = 0;
 }

 void functionsLibCtxClearCurrent(int async) {
@@ -124,7 +124,9 @@ static void cumulativeKeyCountAdd(kvstore *kvs, int didx, long delta) {

    dict *d = kvstoreGetDict(kvs, didx);
    size_t dsize = dictSize(d);
-    int non_empty_dicts_delta = dsize == 1? 1 : dsize == 0? -1 : 0;
+    /* Increment if dsize is 1 and delta is positive (first element inserted, dict becomes non-empty).
+     * Decrement if dsize is 0 (dict becomes empty). */
+    int non_empty_dicts_delta = (dsize == 1 && delta > 0) ? 1 : (dsize == 0) ? -1 : 0;
    kvs->non_empty_dicts += non_empty_dicts_delta;

    /* BIT does not need to be calculated when there's only one dict. */
@@ -1026,6 +1028,31 @@ int kvstoreTest(int argc, char **argv, int flags) {
        kvstoreRelease(kvs);
    }

+    TEST("Verify non-empty dict count is correctly updated") {
+        kvstore *kvs = kvstoreCreate(&KvstoreDictTestType, 2, KVSTORE_ALLOCATE_DICTS_ON_DEMAND);
+        for (int idx = 0; idx < 4; idx++) {
+            for (i = 0; i < 16; i++) {
+                de = kvstoreDictAddRaw(kvs, idx, stringFromInt(i), NULL);
+                assert(de != NULL);
+                /* When the first element is inserted, the number of non-empty dictionaries is increased by 1. */
+                if (i == 0) assert(kvstoreNumNonEmptyDicts(kvs) == idx + 1);
+            }
+        }
+
+        /* Step by step, clear all dictionaries and ensure non-empty dict count is updated */
+        for (int idx = 0; idx < 4; idx++) {
+            kvs_di = kvstoreGetDictSafeIterator(kvs, idx);
+            while((de = kvstoreDictIteratorNext(kvs_di)) != NULL) {
+                key = dictGetKey(de);
+                assert(kvstoreDictDelete(kvs, idx, key) == DICT_OK);
+                /* When the dictionary is emptied, the number of non-empty dictionaries is reduced by 1. */
+                if (kvstoreDictSize(kvs, idx) == 0) assert(kvstoreNumNonEmptyDicts(kvs) == 3 - idx);
+            }
+            kvstoreReleaseDictIterator(kvs_di);
+        }
+        kvstoreRelease(kvs);
+    }
+
    kvstoreRelease(kvs1);
    kvstoreRelease(kvs2);
    return 0;
@@ -12364,7 +12364,7 @@ int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loa
    }

    if (post_load_err) {
-        moduleUnload(ctx.module->name, NULL);
+        serverAssert(moduleUnload(ctx.module->name, NULL, 1) == C_OK);
        moduleFreeContext(&ctx);
        return C_ERR;
    }
@@ -12380,14 +12380,17 @@ int moduleLoad(const char *path, void **module_argv, int module_argc, int is_loa

 /* Unload the module registered with the specified name. On success
 * C_OK is returned, otherwise C_ERR is returned and errmsg is set
- * with an appropriate message. */
-int moduleUnload(sds name, const char **errmsg) {
+ * with an appropriate message.
+ * Only forcefully unload this module, passing forced_unload != 0, 
+ * if it is certain that it has not yet been in use (e.g., immediate
+ * unload on failed load). */
+int moduleUnload(sds name, const char **errmsg, int forced_unload) {
    struct RedisModule *module = dictFetchValue(modules,name);

    if (module == NULL) {
        *errmsg = "no such module with that name";
        return C_ERR;
-    } else if (listLength(module->types)) {
+    } else if (listLength(module->types) && !forced_unload) {
        *errmsg = "the module exports one or more module-side data "
                  "types, can't unload";
        return C_ERR;
@@ -13170,7 +13173,7 @@ NULL

    } else if (!strcasecmp(subcmd,"unload") && c->argc == 3) {
        const char *errmsg = NULL;
-        if (moduleUnload(c->argv[2]->ptr, &errmsg) == C_OK)
+        if (moduleUnload(c->argv[2]->ptr, &errmsg, 0) == C_OK)
            addReply(c,shared.ok);
        else {
            if (errmsg == NULL) errmsg = "operation not possible.";
@@ -3926,6 +3926,11 @@ int checkClientOutputBufferLimits(client *c) {
    int soft = 0, hard = 0, class;
    unsigned long used_mem = getClientOutputBufferMemoryUsage(c);

+    /* For unauthenticated clients the output buffer is limited to prevent
+     * them from abusing it by not reading the replies */
+    if (used_mem > 1024 && authRequired(c))
+        return 1;
+
    class = getClientType(c);
    /* For the purpose of output buffer limiting, masters are handled
     * like normal clients. */
@@ -2332,6 +2332,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error)
                rdbReportCorruptRDB("invalid expireAt time: %llu",
                                    (unsigned long long) expireAt);
                decrRefCount(o);
+                if (dupSearchDict != NULL) dictRelease(dupSearchDict);
                return NULL;
            }

@@ -64,7 +64,7 @@ lua_State *createLuaState(void) {
    size_t sz = sizeof(unsigned int);
    int err = je_mallctl("tcache.create", (void *)&tcache, &sz, NULL, 0);
    if (err) {
-        serverLog(LL_WARNING, "Failed creating the lua jemalloc tcache.");
+        serverLog(LL_WARNING, "Failed creating the lua jemalloc tcache (err=%d).", err);
        exit(1);
    }

@@ -79,7 +79,7 @@ void luaEnvInit(void) {
    size_t sz = sizeof(unsigned int);
    int err = je_mallctl("arenas.create", (void *)&arena, &sz, NULL, 0);
    if (err) {
-        serverLog(LL_WARNING, "Failed creating the lua jemalloc arena.");
+        serverLog(LL_WARNING, "Failed creating the lua jemalloc arena (err=%d).", err);
        exit(1);
    }
    server.lua_arena = arena;
@@ -2510,7 +2510,7 @@ void moduleInitModulesSystem(void);
 void moduleInitModulesSystemLast(void);
 void modulesCron(void);
 int moduleLoad(const char *path, void **argv, int argc, int is_loadex);
-int moduleUnload(sds name, const char **errmsg);
+int moduleUnload(sds name, const char **errmsg, int forced_unload);
 void moduleLoadFromQueue(void);
 int moduleGetCommandKeysViaAPI(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
 int moduleGetCommandChannelsViaAPI(struct redisCommand *cmd, robj **argv, int argc, getKeysResult *result);
@@ -241,8 +241,12 @@ void sortCommandGeneric(client *c, int readonly) {
        } else if (!strcasecmp(c->argv[j]->ptr,"get") && leftargs >= 1) {
            /* If GET is specified with a real pattern, we can't accept it in cluster mode,
             * unless we can make sure the keys formed by the pattern are in the same slot 
-             * as the key to sort. */
-            if (server.cluster_enabled && patternHashSlot(c->argv[j+1]->ptr, sdslen(c->argv[j+1]->ptr)) != getKeySlot(c->argv[1]->ptr)) {
+             * as the key to sort. The pattern # represents the key itself, so just skip
+             * pattern slot check. */
+            if (server.cluster_enabled &&
+                strcmp(c->argv[j+1]->ptr, "#") &&
+                patternHashSlot(c->argv[j+1]->ptr, sdslen(c->argv[j+1]->ptr)) != getKeySlot(c->argv[1]->ptr))
+            {
                addReplyError(c, "GET option of SORT denied in Cluster mode when "
                              "keys formed by the pattern may be in different slots.");
                syntax_error++;
@@ -1134,6 +1134,20 @@ int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cm
        dictEntry *de = dbFind(c->db, key->ptr);
        serverAssert(de != NULL);
        lpt->key = dictGetKey(de);
+    } else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) {
+        listpackEx *lpt = o->ptr;
+
+        /* If the hash previously had HFEs but later no longer does, the key ref
+         * (lpt->key) in the hash might become outdated after a MOVE/COPY/RENAME/RESTORE
+         * operation. These commands maintain the key ref only if HFEs are present.
+         * That is, we can only be sure that key ref is valid as long as it is not
+         * "trash". (TODO: dbFind() can be avoided. Instead need to extend the
+         * lookupKey*() to return dictEntry). */
+        if (lpt->meta.trash) {
+            dictEntry *de = dbFind(c->db, key->ptr);
+            serverAssert(de != NULL);
+            lpt->key = dictGetKey(de);
+        }
    } else if (o->encoding == OBJ_ENCODING_HT) {
        /* Take care dict has HFE metadata */
        if (!isDictWithMetaHFE(ht)) {
@@ -1151,6 +1165,18 @@ int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cm
            m->key = dictGetKey(de); /* reference key in keyspace */
            m->hfe = ebCreate();     /* Allocate HFE DS */
            m->expireMeta.trash = 1; /* mark as trash (as long it wasn't ebAdd()) */
+        } else {
+            dictExpireMetadata *m = (dictExpireMetadata *) dictMetadata(ht);
+            /* If the hash previously had HFEs but later no longer does, the key ref
+             * (m->key) in the hash might become outdated after a MOVE/COPY/RENAME/RESTORE
+             * operation. These commands maintain the key ref only if HFEs are present.
+             * That is, we can only be sure that key ref is valid as long as it is not
+             * "trash". */
+            if (m->expireMeta.trash) {
+                dictEntry *de = dbFind(db, key->ptr);
+                serverAssert(de != NULL);
+                m->key = dictGetKey(de); /* reference key in keyspace */
+            }
        }
    }

@@ -1985,6 +2011,21 @@ uint64_t hashTypeRemoveFromExpires(ebuckets *hexpires, robj *o) {
    return expireTime;
 }

+int hashTypeIsFieldsWithExpire(robj *o) {
+    if (o->encoding == OBJ_ENCODING_LISTPACK) {
+        return 0;
+    } else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) {
+        return EB_EXPIRE_TIME_INVALID != listpackExGetMinExpire(o);
+    } else { /* o->encoding == OBJ_ENCODING_HT */
+        dict *d = o->ptr;
+        /* If dict doesn't holds HFE metadata */
+        if (!isDictWithMetaHFE(d))
+            return 0;
+        dictExpireMetadata *meta = (dictExpireMetadata *) dictMetadata(d);
+        return ebGetTotalItems(meta->hfe, &hashFieldExpireBucketsType) != 0;
+    }
+}
+
 /* Add hash to global HFE DS and update key for notifications.
 *
 * key         - must be the same key instance that is persisted in db->dict
@@ -2315,6 +2356,14 @@ void hdelCommand(client *c) {
    if ((o = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL ||
        checkType(c,o,OBJ_HASH)) return;

+    /* Hash field expiration is optimized to avoid frequent update global HFE DS for
+     * each field deletion. Eventually active-expiration will run and update or remove
+     * the hash from global HFE DS gracefully. Nevertheless, statistic "subexpiry"
+     * might reflect wrong number of hashes with HFE to the user if it is the last
+     * field with expiration. The following logic checks if this is indeed the last
+     * field with expiration and removes it from global HFE DS. */
+    int isHFE = hashTypeIsFieldsWithExpire(o);
+
    for (j = 2; j < c->argc; j++) {
        if (hashTypeDelete(o,c->argv[j]->ptr,1)) {
            deleted++;
@@ -2328,9 +2377,13 @@ void hdelCommand(client *c) {
    if (deleted) {
        signalModifiedKey(c,c->db,c->argv[1]);
        notifyKeyspaceEvent(NOTIFY_HASH,"hdel",c->argv[1],c->db->id);
-        if (keyremoved)
-            notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],
-                                c->db->id);
+        if (keyremoved) {
+            notifyKeyspaceEvent(NOTIFY_GENERIC, "del", c->argv[1], c->db->id);
+        } else {
+            if (isHFE && (hashTypeIsFieldsWithExpire(o) == 0)) /* is it last HFE */
+                ebRemove(&c->db->hexpires, &hashExpireBucketsType, o);
+        }
+
        server.dirty += deleted;
    }
    addReplyLongLong(c,deleted);
@@ -1405,11 +1405,6 @@ int streamRangeHasTombstones(stream *s, streamID *start, streamID *end) {
        return 0;
    }

-    if (streamCompareID(&s->first_id,&s->max_deleted_entry_id) > 0) {
-        /* The latest tombstone is before the first entry. */
-        return 0;
-    }
-
    if (start) {
        start_id = *start;
    } else {
@@ -1446,6 +1441,17 @@ void streamReplyWithCGLag(client *c, stream *s, streamCG *cg) {
        /* The lag of a newly-initialized stream is 0. */
        lag = 0;
        valid = 1;
+    } else if (!s->length) { /* All entries deleted, now empty. */
+        lag = 0;
+        valid = 1;
+    } else if (streamCompareID(&cg->last_id,&s->first_id) < 0 &&
+               streamCompareID(&s->max_deleted_entry_id,&s->first_id) < 0)
+    {
+        /* When both the consumer group's last_id and the maximum tombstone are behind
+         * the stream's first entry, the consumer group's lag will always be equal to
+         * the number of remainin entries in the stream. */
+        lag = s->length;
+        valid = 1;
    } else if (cg->entries_read != SCG_INVALID_ENTRIES_READ && !streamRangeHasTombstones(s,&cg->last_id,NULL)) {
        /* No fragmentation ahead means that the group's logical reads counter
         * is valid for performing the lag calculation. */
@@ -1502,6 +1508,10 @@ long long streamEstimateDistanceFromFirstEverEntry(stream *s, streamID *id) {
        return s->entries_added;
    }

+    /* There are fragmentations between the `id` and the stream's last-generated-id. */
+    if (!streamIDEqZero(id) && streamCompareID(id,&s->max_deleted_entry_id) < 0)
+        return SCG_INVALID_ENTRIES_READ;
+
    int cmp_last = streamCompareID(id,&s->last_id);
    if (cmp_last == 0) {
        /* Return the exact counter of the last entry in the stream. */
@@ -1684,10 +1694,10 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
    while(streamIteratorGetID(&si,&id,&numfields)) {
        /* Update the group last_id if needed. */
        if (group && streamCompareID(&id,&group->last_id) > 0) {
-            if (group->entries_read != SCG_INVALID_ENTRIES_READ && !streamRangeHasTombstones(s,&id,NULL)) {
-                /* A valid counter and no future tombstones mean we can 
-                 * increment the read counter to keep tracking the group's
-                 * progress. */
+            if (group->entries_read != SCG_INVALID_ENTRIES_READ && !streamRangeHasTombstones(s,&group->last_id,NULL)) {
+                /* A valid counter and no tombstones between the group's last-delivered-id
+                 * and the stream's last-generated-id mean we can increment the read counter
+                 * to keep tracking the group's progress. */
                group->entries_read++;
            } else if (s->entries_added) {
                /* The group's counter may be invalid, so we try to obtain it. */
@@ -2289,14 +2299,13 @@ void xreadCommand(client *c) {
                                "just return an empty result set.");
                goto cleanup;
            }
-            if (o) {
+            if (o && ((stream *)o->ptr)->length) {
                stream *s = o->ptr;
-                ids[id_idx] = s->last_id;
-                if (streamDecrID(&ids[id_idx]) != C_OK) {
-                    /* shouldn't happen */
-                    addReplyError(c,"the stream last element ID is 0-0");
-                    goto cleanup;
-                }
+                /* We need to get the last valid ID.
+                 * It is impossible to use s->last_id because
+                 * entry with s->last_id may have been removed. */
+                streamLastValidID(s, &ids[id_idx]);
+                streamDecrID(&ids[id_idx]);
            } else {
                ids[id_idx].ms = 0;
                ids[id_idx].seq = 0;
@@ -55,8 +55,11 @@

 /* Glob-style pattern matching. */
 static int stringmatchlen_impl(const char *pattern, int patternLen,
-        const char *string, int stringLen, int nocase, int *skipLongerMatches)
+        const char *string, int stringLen, int nocase, int *skipLongerMatches, int nesting)
 {
+    /* Protection against abusive patterns. */
+    if (nesting > 1000) return 0;
+
    while(patternLen && stringLen) {
        switch(pattern[0]) {
        case '*':
@@ -68,7 +71,7 @@ static int stringmatchlen_impl(const char *pattern, int patternLen,
                return 1; /* match */
            while(stringLen) {
                if (stringmatchlen_impl(pattern+1, patternLen-1,
-                            string, stringLen, nocase, skipLongerMatches))
+                            string, stringLen, nocase, skipLongerMatches, nesting+1))
                    return 1; /* match */
                if (*skipLongerMatches)
                    return 0; /* no match */
@@ -190,7 +193,7 @@ static int stringmatchlen_impl(const char *pattern, int patternLen,
 int stringmatchlen(const char *pattern, int patternLen,
        const char *string, int stringLen, int nocase) {
    int skipLongerMatches = 0;
-    return stringmatchlen_impl(pattern,patternLen,string,stringLen,nocase,&skipLongerMatches);
+    return stringmatchlen_impl(pattern,patternLen,string,stringLen,nocase,&skipLongerMatches,0);
 }

 int stringmatch(const char *pattern, const char *string, int nocase) {
@@ -1,2 +1,2 @@
-#define REDIS_VERSION "255.255.255"
-#define REDIS_VERSION_NUM 0x00ffffff
+#define REDIS_VERSION "7.4.3"
+#define REDIS_VERSION_NUM 0x00070403
@@ -124,6 +124,10 @@ test "Verify health as fail for killed node" {
    }
 }

+test "Verify that other nodes can correctly output the new master's slots" {
+    assert_not_equal {} [dict get [get_node_info_from_shard [R 4 CLUSTER MYID] 8 "shard"] slots]
+}
+
 set primary_id 4
 set replica_id 0

@@ -312,3 +312,12 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)

    return REDISMODULE_OK;
 }
+
+int RedisModule_OnUnload(RedisModuleCtx *ctx) {
+    REDISMODULE_NOT_USED(ctx);
+    if (datatype) {
+        RedisModule_Free(datatype);
+        datatype = NULL;
+    }
+    return REDISMODULE_OK;
+}
@@ -141,13 +141,14 @@ size_t FragFreeEffort(RedisModuleString *key, const void *value) {
 }

 int FragDefrag(RedisModuleDefragCtx *ctx, RedisModuleString *key, void **value) {
-    REDISMODULE_NOT_USED(key);
    unsigned long i = 0;
    int steps = 0;

    int dbid = RedisModule_GetDbIdFromDefragCtx(ctx);
    RedisModule_Assert(dbid != -1);

+    RedisModule_Log(NULL, "notice", "Defrag key: %s", RedisModule_StringPtrLen(key, NULL));
+
    /* Attempt to get cursor, validate it's what we're exepcting */
    if (RedisModule_DefragCursorGet(ctx, &i) == REDISMODULE_OK) {
        if (i > 0) datatype_resumes++;
@@ -116,6 +116,32 @@ start_server {tags {"acl external:skip"}} {
        assert_match "*NOPERM*key*" $err
    }

+    test {Validate read and write permissions format - empty permission} {
+        catch {r ACL SETUSER key-permission-RW %~} err
+        set err
+    } {ERR Error in ACL SETUSER modifier '%~': Syntax error}
+
+    test {Validate read and write permissions format - empty selector} {
+        catch {r ACL SETUSER key-permission-RW %} err
+        set err
+    } {ERR Error in ACL SETUSER modifier '%': Syntax error}
+
+    test {Validate read and write permissions format - empty pattern} {
+        # Empty pattern results with R/W access to no key
+        r ACL SETUSER key-permission-RW on nopass %RW~ +@all
+        $r2 auth key-permission-RW password
+        catch {$r2 SET x 5} err
+        set err
+    } {NOPERM No permissions to access a key}
+
+    test {Validate read and write permissions format - no pattern} {
+        # No pattern results with R/W access to no key (currently we accept this syntax error)
+        r ACL SETUSER key-permission-RW on nopass %RW +@all
+        $r2 auth key-permission-RW password
+        catch {$r2 SET x 5} err
+        set err
+    } {NOPERM No permissions to access a key}
+
    test {Test separate read and write permissions on different selectors are not additive} {
        r ACL SETUSER key-permission-RW-selector on nopass "(%R~read* +@all)" "(%W~write* +@all)"
        $r2 auth key-permission-RW-selector password
@@ -45,6 +45,24 @@ start_server {tags {"auth external:skip"} overrides {requirepass foobar}} {
        assert_match {*unauthenticated bulk length*} $e
        $rr close
    }
+
+    test {For unauthenticated clients output buffer is limited} {
+        set rr [redis [srv "host"] [srv "port"] 1 $::tls]
+        $rr SET x 5
+        catch {[$rr read]} e
+        assert_match {*NOAUTH Authentication required*} $e
+
+        # Fill the output buffer in a loop without reading it and make
+        # sure the client disconnected.
+        # Considering the socket eat some of the replies, we are testing
+        # that such client can't consume more than few MB's.
+        catch {
+            for {set j 0} {$j < 1000000} {incr j} {
+                    $rr SET x 5
+            }
+        } e
+        assert_match {I/O error reading reply} $e
+    }
 }

 start_server {tags {"auth_binary_password external:skip"}} {
@@ -1,3 +1,16 @@
+#
+# Copyright (c) 2009-Present, Redis Ltd.
+# All rights reserved.
+#
+# Copyright (c) 2024-present, Valkey contributors.
+# All rights reserved.
+#
+# Licensed under your choice of the Redis Source Available License 2.0
+# (RSALv2) or the Server Side Public License v1 (SSPLv1).
+#
+# Portions of this file are available under BSD3 terms; see REDISCONTRIBUTIONS for more information.
+#
+
 proc get_slot_field {slot_output shard_id node_id attrib_id} {
    return [lindex [lindex [lindex $slot_output $shard_id] $node_id] $attrib_id]
 }
@@ -116,10 +129,11 @@ test "Verify the nodes configured with prefer hostname only show hostname for ne
    # Have everyone forget node 6 and isolate it from the cluster.
    isolate_node 6

-    # Set hostnames for the masters, now that the node is isolated
-    R 0 config set cluster-announce-hostname "shard-1.com"
-    R 1 config set cluster-announce-hostname "shard-2.com"
-    R 2 config set cluster-announce-hostname "shard-3.com"
+    set primaries 3
+    for {set j 0} {$j < $primaries} {incr j} {
+        # Set hostnames for the masters, now that the node is isolated
+        R $j config set cluster-announce-hostname "shard-$j.com"
+    }

    # Prevent Node 0 and Node 6 from properly meeting,
    # they'll hang in the handshake phase. This allows us to 
@@ -149,9 +163,17 @@ test "Verify the nodes configured with prefer hostname only show hostname for ne
    } else {
        fail "Node did not learn about the 2 shards it can talk to"
    }
-    set slot_result [R 6 CLUSTER SLOTS]
-    assert_equal [lindex [get_slot_field $slot_result 0 2 3] 1] "shard-2.com"
-    assert_equal [lindex [get_slot_field $slot_result 1 2 3] 1] "shard-3.com"
+    wait_for_condition 50 100 {
+        [lindex [get_slot_field [R 6 CLUSTER SLOTS] 0 2 3] 1] eq "shard-1.com"
+    } else {
+        fail "hostname for shard-1 didn't reach node 6"
+    }
+
+    wait_for_condition 50 100 {
+        [lindex [get_slot_field [R 6 CLUSTER SLOTS] 1 2 3] 1] eq "shard-2.com"
+    } else {
+        fail "hostname for shard-2 didn't reach node 6"
+    }

    # Also make sure we know about the isolated master, we 
    # just can't reach it.
@@ -170,10 +192,14 @@ test "Verify the nodes configured with prefer hostname only show hostname for ne
    } else {
        fail "Node did not learn about the 2 shards it can talk to"
    }
-    set slot_result [R 6 CLUSTER SLOTS]
-    assert_equal [lindex [get_slot_field $slot_result 0 2 3] 1] "shard-1.com"
-    assert_equal [lindex [get_slot_field $slot_result 1 2 3] 1] "shard-2.com"
-    assert_equal [lindex [get_slot_field $slot_result 2 2 3] 1] "shard-3.com"
+
+    for {set j 0} {$j < $primaries} {incr j} {
+        wait_for_condition 50 100 {
+            [lindex [get_slot_field [R 6 CLUSTER SLOTS] $j 2 3] 1] eq "shard-$j.com"
+        } else {
+            fail "hostname information for shard-$j didn't reach node 6"
+        }
+    }
 }

 test "Test restart will keep hostname information" {
@@ -1,3 +1,16 @@
+#
+# Copyright (c) 2009-Present, Redis Ltd.
+# All rights reserved.
+#
+# Copyright (c) 2024-present, Valkey contributors.
+# All rights reserved.
+#
+# Licensed under your choice of the Redis Source Available License 2.0
+# (RSALv2) or the Server Side Public License v1 (SSPLv1).
+#
+# Portions of this file are available under BSD3 terms; see REDISCONTRIBUTIONS for more information.
+#
+
 proc cmdstat {cmd} {
    return [cmdrstat $cmd r]
 }
@@ -386,10 +399,10 @@ start_server {tags {"info" "external:skip"}} {
            r config set client-output-buffer-limit "normal 10 0 0"
            r set key [string repeat a 100000] ;# to trigger output buffer limit check this needs to be big
            catch {r get key}
+            r config set client-output-buffer-limit $org_outbuf_limit
            set info [r info stats]
            assert_equal [getInfoProperty $info client_output_buffer_limit_disconnections] {1}
-            r config set client-output-buffer-limit $org_outbuf_limit
-        } {OK} {logreqres:skip} ;# same as obuf-limits.tcl, skip logreqres
+        } {} {logreqres:skip} ;# same as obuf-limits.tcl, skip logreqres

        test {clients: pubsub clients} {
            set info [r info clients]
@@ -509,6 +509,12 @@ foreach {type large} [array get largevalue] {
        r KEYS "a*a*a*a*a*a*a*a*a*a*a*a*a*a*a*a*a*a*a*a*b"
    } {}

+    test {Regression for pattern matching very long nested loops} {
+        r flushdb
+        r SET [string repeat "a" 50000] 1
+        r KEYS [string repeat "*?" 50000]
+    } {}
+
    test {Coverage: basic SWAPDB test and unhappy path} {
       r flushall
       r select 0
@@ -174,4 +174,18 @@ start_server {tags {"lazyfree"}} {
        assert_equal [s lazyfreed_objects] 2
        $rd close
    }
+
+    test "Unblocks client blocked on lazyfree via REPLICAOF command" {
+        set rd [redis_deferring_client]
+
+        populate 50000 ;# Just to make flushdb async slower
+        $rd flushdb
+        wait_for_blocked_client
+        # Test that slaveof command unblocks clients without assertion failure
+        r slaveof 127.0.0.1 0
+        assert_equal [$rd read] {OK}
+        $rd close
+        r ping
+        r slaveof no one
+    } {OK} {external:skip}
 }
@@ -1,6 +1,11 @@
 set testmodule [file normalize tests/modules/datatype.so]

 start_server {tags {"modules"}} {
+    test {DataType: test loadex with invalid config} {
+        catch { r module loadex $testmodule CONFIG invalid_config 1 } e
+        assert_match {*ERR Error loading the extension*} $e
+    }
+
    r module load $testmodule

    test {DataType: Test module is sane, GET/SET work.} {
@@ -359,6 +359,26 @@ start_server {tags {"pause network"}} {
        } {bar2}
    }

+    test "Test the randomkey command will not cause the server to get into an infinite loop during the client pause write" {
+        r flushall
+
+        r multi
+        r set key value px 3
+        r client pause 10000 write
+        r exec
+        
+        after 5
+
+        wait_for_condition 50 100 {
+            [r randomkey] == "key"
+        } else {
+            fail "execute randomkey failed, caused by the infinite loop"
+        }
+
+        r client unpause
+        assert_equal [r randomkey] {}
+    }
+
    # Make sure we unpause at the end
    r client unpause
 }
@@ -691,6 +691,12 @@ start_server {tags {"scripting"}} {
        set e
    } {ERR *Attempt to modify a readonly table*}

+    test {lua bit.tohex bug} {
+        set res [run_script {return bit.tohex(65535, -2147483648)} 0]
+        r ping
+        set res
+    } {0000FFFF}
+
    test {Test an example script DECR_IF_GT} {
        set decr_if_gt {
            local current
@@ -1878,6 +1884,35 @@ start_server {tags {"scripting needs:debug"}} {
 }

 start_server {tags {"scripting"}} {
+    test "Test script flush will not leak memory - script:$is_eval" {
+        r flushall
+        r script flush
+        r function flush
+
+        # This is a best-effort test to check we don't leak some resources on
+        # script flush and function flush commands. For lua vm, we create a
+        # jemalloc thread cache. On each script flush command, thread cache is
+        # destroyed and we create a new one. In this test, running script flush
+        # many times to verify there is no increase in the memory usage while
+        # re-creating some of the resources for lua vm.
+        set used_memory [s used_memory]
+        set allocator_allocated [s allocator_allocated]
+
+        r multi
+        for {set j 1} {$j <= 500} {incr j} {
+            if {$is_eval} {
+                r SCRIPT FLUSH
+            } else {
+                r FUNCTION FLUSH
+            }
+        }
+        r exec
+
+        # Verify used memory is not (much) higher.
+        assert_lessthan [s used_memory] [expr $used_memory*1.5]
+        assert_lessthan [s allocator_allocated] [expr $allocator_allocated*1.5]
+    }
+
    test "Verify Lua performs GC correctly after script loading" {
        set dummy_script "--[string repeat x 10]\nreturn "
        set n 50000
@@ -73,7 +73,7 @@ start_server {
    set result [create_random_dataset 16 lpush]
    test "SORT GET #" {
        assert_equal [lsort -integer $result] [r sort tosort GET #]
-    } {} {cluster:skip}
+    }

 foreach command {SORT SORT_RO} {
    test "$command GET <const>" {
@@ -393,6 +393,11 @@ start_cluster 1 0 {tags {"external:skip cluster sort"}} {
        r sort "{a}mylist" by "{a}by*" get "{a}get*"
    } {30 200 100}

+    test "sort get # in cluster mode" {
+        assert_equal [r sort "{a}mylist" by "{a}by*" get # ] {3 1 2}
+        r sort "{a}mylist" by "{a}by*" get "{a}get*" get #
+    } {30 3 200 1 100 2}
+
    test "sort_ro by in cluster mode" {
        catch {r sort_ro "{a}mylist" by by*} e
        assert_match {ERR BY option of SORT denied in Cluster mode when *} $e
@@ -404,4 +409,9 @@ start_cluster 1 0 {tags {"external:skip cluster sort"}} {
        assert_match {ERR GET option of SORT denied in Cluster mode when *} $e
        r sort_ro "{a}mylist" by "{a}by*" get "{a}get*"
    } {30 200 100}
+
+    test "sort_ro get # in cluster mode" {
+        assert_equal [r sort_ro "{a}mylist" by "{a}by*" get # ] {3 1 2}
+        r sort_ro "{a}mylist" by "{a}by*" get "{a}get*" get #
+    } {30 3 200 1 100 2}
 }
@@ -199,12 +199,12 @@ start_server {tags {"external:skip needs:debug"}} {
            set hash_sizes {1 15 16 17 31 32 33 40}
            foreach h $hash_sizes {
                for {set i 1} {$i <= $h} {incr i} {
-                    # random expiration time
+                    # Random expiration time (Take care expired not after "mix$h")
                    r hset hrand$h f$i v$i
-                    r hpexpire hrand$h [expr {50 + int(rand() * 50)}] FIELDS 1 f$i
+                    r hpexpire hrand$h [expr {70 + int(rand() * 30)}] FIELDS 1 f$i
                    assert_equal 1 [r HEXISTS hrand$h f$i]

-                    # same expiration time
+                    # Same expiration time (Take care expired not after "mix$h")
                    r hset same$h f$i v$i
                    r hpexpire same$h 100 FIELDS 1 f$i
                    assert_equal 1 [r HEXISTS same$h f$i]
@@ -286,10 +286,9 @@ start_server {tags {"external:skip needs:debug"}} {
        test "HEXPIRETIME - returns TTL in Unix timestamp ($type)" {
            r del myhash
            r HSET myhash field1 value1
-            r HPEXPIRE myhash 1000 NX FIELDS 1 field1
-
            set lo [expr {[clock seconds] + 1}]
            set hi [expr {[clock seconds] + 2}]
+            r HPEXPIRE myhash 1000 NX FIELDS 1 field1
            assert_range [r HEXPIRETIME myhash FIELDS 1 field1] $lo $hi
            assert_range [r HPEXPIRETIME myhash FIELDS 1 field1] [expr $lo*1000] [expr $hi*1000]
        }
@@ -592,6 +591,20 @@ start_server {tags {"external:skip needs:debug"}} {
            wait_for_condition 30 10 { [r exists myhash2] == 0 } else { fail "`myhash2` should be expired" }
        }

+        test "Test RENAME hash that had HFEs but not during the rename ($type)" {
+            r del h1
+            r hset h1 f1 v1 f2 v2
+            r hpexpire h1 1 FIELDS 1 f1
+            after 20
+            r rename h1 h1_renamed
+            assert_equal [r exists h1] 0
+            assert_equal [r exists h1_renamed] 1
+            assert_equal [r hgetall h1_renamed] {f2 v2}
+            r hpexpire h1_renamed 1 FIELDS 1 f2
+            # Only active expire will delete the key
+            wait_for_condition 30 10 { [r exists h1_renamed] == 0 } else { fail "`h1_renamed` should be expired" }
+        }
+
        test "MOVE to another DB hash with fields to be expired ($type)" {
            r select 9
            r flushall
@@ -635,6 +648,20 @@ start_server {tags {"external:skip needs:debug"}} {

        } {} {singledb:skip}

+        test "Test COPY hash that had HFEs but not during the copy ($type)" {
+            r del h1
+            r hset h1 f1 v1 f2 v2
+            r hpexpire h1 1 FIELDS 1 f1
+            after 20
+            r COPY h1 h1_copy
+            assert_equal [r exists h1] 1
+            assert_equal [r exists h1_copy] 1
+            assert_equal [r hgetall h1_copy] {f2 v2}
+            r hpexpire h1_copy 1 FIELDS 1 f2
+            # Only active expire will delete the key
+            wait_for_condition 30 10 { [r exists h1_copy] == 0 } else { fail "`h1_copy` should be expired" }
+        }
+
        test "Test SWAPDB hash-fields to be expired ($type)" {
            r select 9
            r flushall
@@ -656,6 +683,29 @@ start_server {tags {"external:skip needs:debug"}} {
            wait_for_condition 20 10 { [r exists myhash] == 0 } else { fail "'myhash' should be expired" }
        } {} {singledb:skip}

+        test "Test SWAPDB hash that had HFEs but not during the swap ($type)" {
+            r select 9
+            r flushall
+            r hset myhash f1 v1 f2 v2
+            r hpexpire myhash 1 NX FIELDS 1 f1
+            after 10
+
+            r swapdb 9 10
+
+            # Verify the key and its field doesn't exist in the source DB
+            assert_equal [r exists myhash] 0
+            assert_equal [r dbsize] 0
+
+            # Verify the key and its field exists in the target DB
+            r select 10
+            assert_equal [r hgetall myhash] {f2 v2}
+            assert_equal [r dbsize] 1
+            r hpexpire myhash 1 NX FIELDS 1 f2
+
+            # Eventually the field will be expired and the key will be deleted
+            wait_for_condition 20 10 { [r exists myhash] == 0 } else { fail "'myhash' should be expired" }
+        } {} {singledb:skip}
+
        test "HMGET - returns empty entries if fields or hash expired ($type)" {
            r debug set-active-expire 0
            r del h1 h2
@@ -724,6 +774,20 @@ start_server {tags {"external:skip needs:debug"}} {
            assert_equal [r hexpiretime myhash FIELDS 3 a b c] {2524600800 2524600801 -1}
        }

+        test {RESTORE hash that had in the past HFEs but not during the dump} {
+            r config set sanitize-dump-payload yes
+            r del myhash
+            r hmset myhash a 1 b 2 c 3
+            r hpexpire myhash 1 fields 1 a
+            after 10
+            set encoded [r dump myhash]
+            r del myhash
+            r restore myhash 0 $encoded
+            assert_equal [lsort [r hgetall myhash]] "2 3 b c"
+            r hpexpire myhash 1 fields 2 b c
+            wait_for_condition 30 10 { [r exists myhash] == 0 } else { fail "`myhash` should be expired" }
+        }
+
        test {DUMP / RESTORE are able to serialize / unserialize a hash with TTL 0 for all fields} {
            r config set sanitize-dump-payload yes
            r del myhash
@@ -786,6 +850,36 @@ start_server {tags {"external:skip needs:debug"}} {
        }
    }

+    test "Statistics - Hashes with HFEs ($type)" {
+        r config resetstat
+        r flushall
+
+        # hash1: 5 fields, 3 with TTL. subexpiry incr +1
+        r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
+        r hpexpire myhash 150 FIELDS 3 f1 f2 f3
+        assert_match  [get_stat_subexpiry r] 1
+
+        # hash2: 5 fields, 3 with TTL. subexpiry incr +1
+        r hset myhash2 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
+        assert_match  [get_stat_subexpiry r] 1
+        r hpexpire myhash2 100 FIELDS 3 f1 f2 f3
+        assert_match  [get_stat_subexpiry r] 2
+
+        # hash3: 2 fields, 1 with TTL. HDEL field with TTL. subexpiry decr -1
+        r hset myhash3 f1 v1 f2 v2
+        r hpexpire myhash3 100 FIELDS 1 f2
+        assert_match  [get_stat_subexpiry r] 3
+        r hdel myhash3 f2
+        assert_match  [get_stat_subexpiry r] 2
+
+        # Expired fields of hash1 and hash2. subexpiry decr -2
+        wait_for_condition 50 50 {
+                [get_stat_subexpiry r] == 0
+        } else {
+                fail "Hash field expiry statistics failed"
+        }
+    }
+
    r config set hash-max-listpack-entries 512
 }

@@ -899,25 +993,6 @@ start_server {tags {"external:skip needs:debug"}} {

        r config set hash-max-listpack-value 64
    }
-
-    test "Statistics - Hashes with HFEs" {
-        r config resetstat
-        r del myhash
-        r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
-        r hpexpire myhash 100 FIELDS 3 f1 f2 f3
-
-        assert_match  [get_stat_subexpiry r] 1
-        r hset myhash2 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
-        assert_match  [get_stat_subexpiry r] 1
-        r hpexpire myhash2 100 FIELDS 3 f1 f2 f3
-        assert_match  [get_stat_subexpiry r] 2
-
-        wait_for_condition 50 50 {
-                [get_stat_subexpiry r] == 0
-        } else {
-                fail "Hash field expiry statistics failed"
-        }
-    }
 }

 start_server {tags {"external:skip needs:debug"}} {
@@ -932,6 +1007,7 @@ start_server {tags {"external:skip needs:debug"}} {
            start_server {overrides {appendonly {yes} appendfsync always} tags {external:skip}} {

                set aof [get_last_incr_aof_path r]
+                r debug set-active-expire 0 ;# Prevent fields from being expired during data preparation

                # Time is in the past so it should propagate HDELs to replica
                # and delete the fields
@@ -958,6 +1034,7 @@ start_server {tags {"external:skip needs:debug"}} {
                r hpexpireat h2 [expr [clock seconds]*1000+100000] LT FIELDS 1 f3
                r hexpireat h2 [expr [clock seconds]+10] NX FIELDS 1 f4

+                r debug set-active-expire 1
                wait_for_condition 50 100 {
                    [r hlen h2] eq 2
                } else {
@@ -986,7 +1063,7 @@ start_server {tags {"external:skip needs:debug"}} {
                    {hdel h2 f2}
                }
            }
-        }
+        } {} {needs:debug}

        test "Lazy Expire - fields are lazy deleted and propagated to replicas ($type)" {
            start_server {overrides {appendonly {yes} appendfsync always} tags {external:skip}} {
@@ -1239,6 +1239,74 @@ start_server {
        assert_equal [dict get $group lag] 2
    }

+    test {Consumer Group Lag with XDELs and tombstone after the last_id of consume group} {
+        r DEL x
+        r XGROUP CREATE x g1 $ MKSTREAM
+        r XADD x 1-0 data a
+        r XREADGROUP GROUP g1 alice STREAMS x > ;# Read one entry
+        r XADD x 2-0 data c
+        r XADD x 3-0 data d
+        r XDEL x 2-0
+
+        # Now the latest tombstone(2-0) is before the first entry(3-0), but there is still
+        # a tombstone(2-0) after the last_id(1-0) of the consume group.
+        set reply [r XINFO STREAM x FULL]
+        set group [lindex [dict get $reply groups] 0]
+        assert_equal [dict get $group entries-read] 1
+        assert_equal [dict get $group lag] {}
+
+        r XDEL x 1-0
+        # Although there is a tombstone(2-0) after the consumer group's last_id(1-0), all
+        # entries before the maximal tombstone have been deleted. This means that both the
+        # last_id and the largest tombstone are behind the first entry. Therefore, tombstones
+        # no longer affect the lag, which now reflects the remaining entries in the stream.
+        set reply [r XINFO STREAM x FULL]
+        set group [lindex [dict get $reply groups] 0]
+        assert_equal [dict get $group entries-read] 1
+        assert_equal [dict get $group lag] 1
+
+        # Now there is a tombstone(2-0) after the last_id of the consume group, so after consuming
+        # entry(3-0), the group's counter will be invalid.
+        r XREADGROUP GROUP g1 alice STREAMS x > 
+        set reply [r XINFO STREAM x FULL]
+        set group [lindex [dict get $reply groups] 0]
+        assert_equal [dict get $group entries-read] 3
+        assert_equal [dict get $group lag] 0
+    }
+
+    test {Consumer group lag with XTRIM} {
+        r DEL x
+        r XGROUP CREATE x mygroup $ MKSTREAM
+        r XADD x 1-0 data a
+        r XADD x 2-0 data b
+        r XADD x 3-0 data c
+        r XADD x 4-0 data d
+        r XADD x 5-0 data e
+        r XREADGROUP GROUP mygroup alice COUNT 1 STREAMS x >
+
+        set reply [r XINFO STREAM x FULL]
+        set group [lindex [dict get $reply groups] 0]
+        assert_equal [dict get $group entries-read] 1
+        assert_equal [dict get $group lag] 4
+
+        # Although XTRIM doesn't update the `max-deleted-entry-id`, it always updates the
+        # position of the first entry. When trimming causes the first entry to be behind
+        # the consumer group's last_id, the consumer group's lag will always be equal to
+        # the number of remainin entries in the stream.
+        r XTRIM x MAXLEN 1
+        set reply [r XINFO STREAM x FULL]
+        set group [lindex [dict get $reply groups] 0]
+        assert_equal [dict get $reply max-deleted-entry-id] "0-0"
+        assert_equal [dict get $group entries-read] 1
+        assert_equal [dict get $group lag] 1
+
+        # When all the entries were deleted, the lag is always 0.
+        r XTRIM x MAXLEN 0
+        set reply [r XINFO STREAM x FULL]
+        set group [lindex [dict get $reply groups] 0]
+        assert_equal [dict get $group lag] 0
+    }
+
    test {Loading from legacy (Redis <= v6.2.x, rdb_ver < 10) persistence} {
        # The payload was DUMPed from a v5 instance after:
        # XADD x 1-0 data a
@@ -435,6 +435,17 @@ start_server {

        # verify nil is still received when reading last entry
        assert_equal [r XREAD STREAMS lestream +] {}
+
+        # case when stream created empty
+
+        # make sure the stream is not initialized
+        r DEL lestream
+
+        # create empty stream with XGROUP CREATE
+        r XGROUP CREATE lestream legroup $ MKSTREAM
+
+        # verify nil is received when reading last entry
+        assert_equal [r XREAD STREAMS lestream +] {}
    }

    test {XREAD last element blocking from empty stream} {
@@ -510,6 +521,22 @@ start_server {
        assert_equal $res {{lestream {{3-0 {k3 v3}}}}}
    }

+    test "XREAD: read last element after XDEL (issue #13628)" {
+        # Should return actual last element after XDEL of current last element
+
+        # Add 2 entries to a stream and delete last one
+        r DEL stream
+        r XADD stream 1-0 f 1
+        r XADD stream 2-0 f 2
+        r XDEL stream 2-0
+
+        # Read last entry
+        set res [r XREAD STREAMS stream +]
+
+        # Verify the last entry was read
+        assert_equal $res {{stream {{1-0 {f 1}}}}}
+    }
+
    test "XREAD: XADD + DEL should not awake client" {
        set rd [redis_deferring_client]
        r del s1