Redis 7.4.2

1. Ubuntu Lunar reached End of Life on January 25, 2024, so upgrade the
ubuntu version to plucky in action `test-ubuntu-jemalloc-fortify` to
pass the daily CI
2. The macOS-12 environment is deprecated so upgrade macos-12 to
macos-13 in daily CI

---------

Co-authored-by: debing.sun <debing.sun@redis.com>

.github/workflows/daily.yml

@@ -76,7 +76,6 @@ jobs:
     if: |
       (github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
       !contains(github.event.inputs.skipjobs, 'fortify')
-    container: ubuntu:lunar
     timeout-minutes: 14400
     steps:
     - name: prep
@@ -94,11 +93,10 @@ jobs:
         ref: ${{ env.GITHUB_HEAD_REF }}
     - name: make
       run: |
-        apt-get update && apt-get install -y make gcc-13
-        update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100
+        apt-get update && apt-get install -y make gcc
         make CC=gcc REDIS_CFLAGS='-Werror -DREDIS_TEST -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3'
     - name: testprep
-      run: apt-get install -y tcl8.6 tclx procps
+      run: sudo apt-get install -y tcl8.6 tclx procps
     - name: test
       if: true && !contains(github.event.inputs.skiptests, 'redis')
       run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
@@ -875,7 +873,7 @@ jobs:
   build-macos:
     strategy:
       matrix:
-        os: [macos-12, macos-14]
+        os: [macos-13, macos-15]
     runs-on: ${{ matrix.os }}
     if: |
       (github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
@@ -902,7 +900,7 @@ jobs:
       run: make REDIS_CFLAGS='-Werror -DREDIS_TEST'
 
   test-freebsd:
-    runs-on: macos-12
+    runs-on: macos-13
     if: |
       (github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
       !contains(github.event.inputs.skipjobs, 'freebsd')

00-RELEASENOTES

@@ -1,16 +1,203 @@
-Hello! This file is just a placeholder, since this is the "unstable" branch
-of Redis, the place where all the development happens.
+Redis Community Edition 7.4 release notes
+=========================================
 
-There is no release notes for this branch, it gets forked into another branch
-every time there is a partial feature freeze in order to eventually create
-a new stable release.
+--------------------------------------------------------------------------------
+Upgrade urgency levels:
 
-Usually "unstable" is stable enough for you to use it in development environments
-however you should never use it in production environments. It is possible
-to download the latest stable release here:
+LOW:      No need to upgrade unless there are new features you want to use.
+MODERATE: Program an upgrade of the server, but it's not urgent.
+HIGH:     There is a critical bug that may affect a subset of users. Upgrade!
+CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP.
+SECURITY: There are security fixes in the release.
+--------------------------------------------------------------------------------
 
-    https://download.redis.io/redis-stable.tar.gz
 
-More information is available at https://redis.io
+================================================================================
+Redis Community Edition 7.4.2    Released Mon 6 Jan 2025 12:30:00 IDT
+================================================================================
+
+Upgrade urgency SECURITY: See security fixes below.
+
+### Security fixes
+
+- (CVE-2024-46981) Lua script commands may lead to remote code execution
+- (CVE-2024-51741) Denial-of-service due to malformed ACL selectors
+
+### Bug fixes
+
+- #13627 Crash on module memory defragmentation
+- #13338 Streams: `XINFO` lag field is wrong when tombstone is after the `last_id` of the consume group
+- #13473 Streams: `XTRIM` does not update the maximal tombstone, leading to an incorrect lag
+- #13470 `INFO` after `HDEL` show wrong number of hash keys with expiration
+- #13476 Fix a race condition in the `cache_memory` of `functionsLibCtx`
+- #13626 Memory leak on failed RDB loading
+- #13539 Hash: fix key ref for a hash that no longer has fields with expiration on `RENAME`/`MOVE`/`SWAPDB`/`RESTORE`
+- #13443 Cluster: crash when loading cluster config
+- #13422 Cluster: `CLUSTER SHARDS` returns empty array
+- #13465 Cluster: incompatibility with older node versions
+- #13608 Cluster: `SORT ... GET #`: incorrect error message
+
+
+================================================================================
+Redis Community Edition 7.4.1    Released Wed 02 Oct 2024 20:17:04 IDT
+================================================================================
+
+Upgrade urgency SECURITY: See security fixes below.
+
+Security fixes
+==============
+
+* (CVE-2024-31449) Lua library commands may lead to stack overflow and potential RCE.
+* (CVE-2024-31227) Potential Denial-of-service due to malformed ACL selectors.
+* (CVE-2024-31228) Potential Denial-of-service due to unbounded pattern matching.
+
+================================================================================
+Redis Community Edition 7.4.0 GA   Released Mon Jul 29 2024 9:00:00 IDT
+================================================================================
+
+This is the General Availability release of Redis Community Edition 7.4.
+
+Changes to new 7.4 features (compared to 7.4 RC2)
+=================================================
+* #13391,#13438 Hash - expiration of individual fields: RDB file format changes
+* #13372 Hash - expiration of individual fields: rename and fix counting of `expired_subkeys` metric
+* #13372 Hash - expiration of individual fields: rename `INFO` keyspace field to `subexpiry`
+
+Configuration parameters
+========================
+* #13400 Add hide-user-data-from-log - allows hiding user data from the log file
+
+Bug fixes
+=========
+* #13407 trigger Lua GC after `SCRIPT LOAD`
+* #13380 Fix possible crash due to OOM panic on invalid command
+* #13383 `FUNCTION FLUSH` - improve Lua GC behavior and fix thread race in ASYNC mode
+* #13408 `HEXPIRE`-like commands should emit HDEL keyspace notification if expire time is in the past
+
+
+================================================================================
+Redis Community Edition 7.4 RC2   Released Thu 27 Jun 2024 10:00:00 IST
+================================================================================
+
+Upgrade urgency LOW: This is the second Release Candidate for Redis Community Edition 7.4.
+
+Performance and resource utilization improvements
+=================================================
+* #13296 Optimize CPU cache efficiency
+
+Changes to new 7.4 new features (compared to 7.4 RC1)
+=====================================================
+* #13343 Hash - expiration of individual fields: when key does not exist - reply with an array (nonexisting code for each field)
+* #13329 Hash - expiration of individual fields: new keyspace event: `hexpired`
+
+Modules API - Potentially breaking changes to new 7.4 features (compared to 7.4 RC1)
+====================================================================================
+* #13326 Hash - expiration of individual fields: avoid lazy expire when called from a Modules API function
+
+
+================================================================================
+Redis Community Edition 7.4 RC1   Released Thu 6 Jun 2024 10:00:00 IST
+================================================================================
+
+Note: License changed - see LICENSE.txt
+
+Upgrade urgency LOW: This is the first Release Candidate for Redis Community Edition 7.4.
+
+Here is a comprehensive list of changes in this release compared to 7.2.5.
+
+New Features
+============
+* #13303 Hash - expiration of individual fields. 9 commands were introduced:
+  - `HEXPIRE` and `HPEXPIRE` set the remaining time to live for specific fields
+  - `HEXPIREAT` and `HPEXPIREAT` set the expiration time to a UNIX timestamp for specific fields
+  - `HPERSIST` removes the expiration for specific fields
+  - `HEXPIRETIME` and `HPEXPIRETIME` get the expiration time for specific fields
+  - `HTTL` and `HPTTL` get the remaining time to live for specific fields
+* #13117 `XREAD`: new id value `+` to start reading from the last message
+* #12765 `HSCAN`: new [NOVALUES] flag to report only field names
+* #12728 `SORT`, `SORT_RO`: allow `BY` and `GET` options in cluster mode when the pattern maps to the same slot as the key
+* #12299 `CLIENT KILL`: new optional filter: `MAXAGE maxage` - retain connections older than `maxage` seconds
+* #12971 Lua: expose `os.clock()` API for getting the elapsed time of Lua code execution
+* #13276 Allow `SPUBLISH` command within `MULTI ... EXEC` transactions on replica
+
+Bug fixes
+=========
+* #12898 `XREADGROUP`: fix entries-read inconsistency between master and replicas
+* #13042 `SORT ... STORE`: fix created lists to respect list compression and packing configs
+* #12817, #12905 Fix race condition issues between the main thread and module threads
+* #12577 Unsubscribe all clients from replica for shard channel if the master ownership changes
+* #12622 `WAITAOF` could timeout or hang if used after a module command that propagated effects only to replicas and not to AOF
+* #11734 `BITCOUNT` and `BITPOS` with nonexistent key and illegal arguments return an error, not 0
+* #12394 `BITCOUNT`: check for wrong argument before checking if key exists
+* #12961 Allow execution of read-only transactions when out of memory
+* #13274 Fix crash when a client performs ACL change that disconnects itself
+* #13311 Cluster: Fix crash due to unblocking client during slot migration
+
+Security improvements
+=====================
+* #13108 Lua: LRU eviction for scripts generated with `EVAL` *** BEHAVIOR CHANGE ***
+* #12961 Restrict the total request size of `MULTI ... EXEC` transactions
+* #12860 Redact ACL username information and mark '*-key-file-pass configs' as sensitive
+
+Performance and resource utilization improvements
+=================================================
+* #12838 Improve performance when many clients call `PUNSUBSCRIBE` / `SUNSUBSCRIBE` simultaneously
+* #12627 Reduce lag when waking `WAITAOF` clients and there is not much traffic
+* #12754 Optimize `KEYS` when pattern includes hashtag and implies a single slot
+* #11695 Reduce memory and improve performance by replacing cluster metadata with slot specific dictionaries
+* #13087 `SCRIPT FLUSH ASYNC` now does not block the main thread
+* #12996 Active memory defragmentation efficiency improvements
+* #12899 Improve performance of read/update operation during rehashing
+* #12536 `SCAN ... MATCH`: Improve performance when the pattern implies cluster slot
+* #12450 `ZRANGE ... LIMIT`: improved performance
+
+Other general improvements
+==========================
+* #13133 Lua: allocate VM code with jemalloc instead of libc and count it as used memory *** BEHAVIOR CHANGE ***
+* #12171 `ACL LOAD`: do not disconnect all clients *** BEHAVIOR CHANGE ***
+* #13020 Allow adjusting defrag configurations while active defragmentation is running
+* #12949 Increase the accuracy of avg_ttl (the average keyspace keys TTL)
+* #12977 Allow running `WAITAOF` in scripts
+* #12782 Implement TCP keep-alive across most Unix-like systems
+* #12707 Improved error codes when rejecting scripts in cluster mode
+* #12596 Support `XREAD ... BLOCK` in scripts; rejected only if it ends up blocking
+
+New metrics
+===========
+* #12849 `INFO`: `pubsub_clients` - number of clients in Pub/Sub mode
+* #12966 `INFO`: `watching_clients` - number of clients that are watching keys
+* #12966 `INFO`: `total_watched_keys` - number of watched keys
+* #12476 `INFO`: `client_query_buffer_limit_disconnections` - count client input buffer OOM events
+* #12476 `INFO`: `client_output_buffer_limit_disconnections` - count client output buffer OOM events
+* #12996 `INFO`: `allocator_muzzy` - memory returned to the OS but still shows as RSS until the OS reclaims it
+* #13108 `INFO`: `evicted_scripts` - number of evicted eval scripts. Users can check it to see if they are abusing EVAL
+* #12996 `MEMORY STATS`: `allocator.muzzy` - memory returned to the OS but still shows as RSS until the OS reclaims it
+* #12913 `INFO MEMORY` `mem_overhead_db_hashtable_rehashing` - memory resharding overhead (only the memory that will be released soon)
+* #12913 `MEMORY STATS': `overhead.db.hashtable.lut` - total overhead of dictionary buckets in databases
+* #12913 `MEMORY STATS': `overhead.db.hashtable.rehashing` - temporary memory overhead of database dictionaries currently being rehashed 
+* #12913 `MEMORY STATS': `db.dict.rehashing.count` - number of top level dictionaries currently being rehashed
+* #12966 `CLIENT LIST`: `watch` - number of keys each client is currently watching
+
+Modules API
+===========
+* #12985 New API calls: `RM_TryCalloc` and `RM_TryRealloc` - allow modules to handle memory allocation failures gracefully
+* #13069 New API call: `RM_ClusterKeySlot` - which slot a key will hash to
+* #13069 New API call: `RM_ClusterCanonicalKeyNameInSlot` - get a consistent key that will map to a slot 
+* #12486 New API call: `RM_AddACLCategory` - allow modules to declare new ACL categories
+
+Configuration parameters
+========================
+* #12178 New configuration parameters: `max-new-connections-per-cycle` and `max-new-tls-connections-per-cycle` to limit the number of new client connections per event-loop cycle
+* #7351 Rename some CPU configuration parameters for style alignment. Added alias to the old names to avoid breaking change
+
+CLI tools
+=========
+* #10609 redis-cli: new `-t <timeout>` argument: specify server connection timeout in seconds
+* #11315 redis-cli: new `-4` and `-6` flags to prefer IPV4 or IPV6 on DNS lookup
+* #12862 redis-cli: allows pressing up arrow to return any command (including sensitive commands which are still not persisted)
+* #12543 redis-cli: add reverse history search (like Ctrl+R in terminals)
+* #12826 redis-cli: add `--keystats` and `--keystats-samples` to combines `--memkeys` and `--bigkeys` with additional distribution data
+* #12735 redis-cli: fix: `--bigkeys` and `--memkeys` now work on cluster replicas
+* #9411 redis-benchmark: add support for binary strings
+* #12986 redis-benchmark: fix: pick random slot for a node to distribute operation across slots
 
-Happy hacking!

deps/lua/src/lua_bit.c

@@ -132,6 +132,7 @@ static int bit_tohex(lua_State *L)
   const char *hexdigits = "0123456789abcdef";
   char buf[8];
   int i;
+  if (n == INT32_MIN) n = INT32_MIN+1;
   if (n < 0) { n = -n; hexdigits = "0123456789ABCDEF"; }
   if (n > 8) n = 8;
   for (i = (int)n; --i >= 0; ) { buf[i] = hexdigits[b & 15]; b >>= 4; }

src/acl.c

@@ -1061,6 +1061,7 @@ int ACLSetSelector(aclSelector *selector, const char* op, size_t oplen) {
         int flags = 0;
         size_t offset = 1;
         if (op[0] == '%') {
+            int perm_ok = 1;
             for (; offset < oplen; offset++) {
                 if (toupper(op[offset]) == 'R' && !(flags & ACL_READ_PERMISSION)) {
                     flags |= ACL_READ_PERMISSION;
@@ -1070,10 +1071,14 @@ int ACLSetSelector(aclSelector *selector, const char* op, size_t oplen) {
                     offset++;
                     break;
                 } else {
-                    errno = EINVAL;
-                    return C_ERR;
+                    perm_ok = 0;
+                    break;
                 }
             }
+            if (!flags || !perm_ok) {
+                errno = EINVAL;
+                return C_ERR;
+            }
         } else {
             flags = ACL_ALL_PERMISSION;
         }

src/cluster_legacy.c

@@ -2,8 +2,13 @@
  * Copyright (c) 2009-Present, Redis Ltd.
  * All rights reserved.
  *
+ * Copyright (c) 2024-present, Valkey contributors.
+ * All rights reserved.
+ *
  * Licensed under your choice of the Redis Source Available License 2.0
  * (RSALv2) or the Server Side Public License v1 (SSPLv1).
+ *
+ * Portions of this file are available under BSD3 terms; see REDISCONTRIBUTIONS for more information.
  */
 
 /*
@@ -634,6 +639,8 @@ int clusterLoadConfig(char *filename) {
     }
     /* Config sanity check */
     if (server.cluster->myself == NULL) goto fmterr;
+    if (!(myself->flags & (CLUSTER_NODE_MASTER | CLUSTER_NODE_SLAVE))) goto fmterr;
+    if (nodeIsSlave(myself) && myself->slaveof == NULL) goto fmterr;
 
     zfree(line);
     fclose(fp);
@@ -1624,6 +1631,22 @@ void clusterRemoveNodeFromShard(clusterNode *node) {
     sdsfree(s);
 }
 
+static clusterNode *clusterGetMasterFromShard(list *nodes) {
+    clusterNode *n = NULL;
+    listIter li;
+    listNode *ln;
+    listRewind(nodes,&li);
+    while ((ln = listNext(&li)) != NULL) {
+        clusterNode *node = listNodeValue(ln);
+        if (!nodeFailed(node)) {
+            n = node;
+            break;
+        }
+    }
+    if (!n) return NULL;
+    return clusterNodeGetMaster(n);
+}
+
 /* -----------------------------------------------------------------------------
  * CLUSTER config epoch handling
  * -------------------------------------------------------------------------- */
@@ -2577,9 +2600,6 @@ uint32_t writePingExt(clusterMsg *hdr, int gossipcount)  {
     extensions++;
 
     if (hdr != NULL) {
-        if (extensions != 0) {
-            hdr->mflags[0] |= CLUSTERMSG_FLAG0_EXT_DATA;
-        }
         hdr->extensions = htons(extensions);
     }
 
@@ -2769,6 +2789,9 @@ int clusterProcessPacket(clusterLink *link) {
     }
 
     sender = getNodeFromLinkAndMsg(link, hdr);
+    if (sender && (hdr->mflags[0] & CLUSTERMSG_FLAG0_EXT_DATA)) {
+        sender->flags |= CLUSTER_NODE_EXTENSIONS_SUPPORTED;
+    }
 
     /* Update the last time we saw any data from this node. We
      * use this in order to avoid detecting a timeout from a node that
@@ -3534,6 +3557,8 @@ static void clusterBuildMessageHdr(clusterMsg *hdr, int type, size_t msglen) {
     /* Set the message flags. */
     if (clusterNodeIsMaster(myself) && server.cluster->mf_end)
         hdr->mflags[0] |= CLUSTERMSG_FLAG0_PAUSED;
+    hdr->mflags[0] |= CLUSTERMSG_FLAG0_EXT_DATA; /* Always make other nodes know that
+                                                  * this node supports extension data. */
 
     hdr->totlen = htonl(msglen);
 }
@@ -3612,7 +3637,9 @@ void clusterSendPing(clusterLink *link, int type) {
      * to put inside the packet. */
     estlen = sizeof(clusterMsg) - sizeof(union clusterMsgData);
     estlen += (sizeof(clusterMsgDataGossip)*(wanted + pfail_wanted));
-    estlen += writePingExt(NULL, 0);
+    if (link->node && nodeSupportsExtensions(link->node)) {
+        estlen += writePingExt(NULL, 0);
+    }
     /* Note: clusterBuildMessageHdr() expects the buffer to be always at least
      * sizeof(clusterMsg) or more. */
     if (estlen < (int)sizeof(clusterMsg)) estlen = sizeof(clusterMsg);
@@ -3682,7 +3709,9 @@ void clusterSendPing(clusterLink *link, int type) {
 
     /* Compute the actual total length and send! */
     uint32_t totlen = 0;
-    totlen += writePingExt(hdr, gossipcount);
+    if (link->node && nodeSupportsExtensions(link->node)) {
+        totlen += writePingExt(hdr, gossipcount);
+    }
     totlen += sizeof(clusterMsg)-sizeof(union clusterMsgData);
     totlen += (sizeof(clusterMsgDataGossip)*gossipcount);
     serverAssert(gossipcount < USHRT_MAX);
@@ -5649,14 +5678,13 @@ void addNodeDetailsToShardReply(client *c, clusterNode *node) {
 /* Add the shard reply of a single shard based off the given primary node. */
 void addShardReplyForClusterShards(client *c, list *nodes) {
     serverAssert(listLength(nodes) > 0);
-    clusterNode *n = listNodeValue(listFirst(nodes));
+
     addReplyMapLen(c, 2);
     addReplyBulkCString(c, "slots");
 
     /* Use slot_info_pairs from the primary only */
-    n = clusterNodeGetMaster(n);
-
-    if (n->slot_info_pairs != NULL) {
+    clusterNode *n = clusterGetMasterFromShard(nodes);
+    if (n && n->slot_info_pairs != NULL) {
         serverAssert((n->slot_info_pairs_count % 2) == 0);
         addReplyArrayLen(c, n->slot_info_pairs_count);
         for (int i = 0; i < n->slot_info_pairs_count; i++)

src/cluster_legacy.h

@@ -1,3 +1,16 @@
+/*
+ * Copyright (c) 2009-Present, Redis Ltd.
+ * All rights reserved.
+ *
+ * Copyright (c) 2024-present, Valkey contributors.
+ * All rights reserved.
+ *
+ * Licensed under your choice of the Redis Source Available License 2.0
+ * (RSALv2) or the Server Side Public License v1 (SSPLv1).
+ *
+ * Portions of this file are available under BSD3 terms; see REDISCONTRIBUTIONS for more information.
+ */
+
 #ifndef CLUSTER_LEGACY_H
 #define CLUSTER_LEGACY_H
 
@@ -51,6 +64,7 @@ typedef struct clusterLink {
 #define CLUSTER_NODE_MEET 128     /* Send a MEET message to this node */
 #define CLUSTER_NODE_MIGRATE_TO 256 /* Master eligible for replica migration. */
 #define CLUSTER_NODE_NOFAILOVER 512 /* Slave will not try to failover. */
+#define CLUSTER_NODE_EXTENSIONS_SUPPORTED 1024 /* This node supports extensions. */
 #define CLUSTER_NODE_NULL_NAME "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"
 
 #define nodeIsSlave(n) ((n)->flags & CLUSTER_NODE_SLAVE)
@@ -59,6 +73,7 @@ typedef struct clusterLink {
 #define nodeTimedOut(n) ((n)->flags & CLUSTER_NODE_PFAIL)
 #define nodeFailed(n) ((n)->flags & CLUSTER_NODE_FAIL)
 #define nodeCantFailover(n) ((n)->flags & CLUSTER_NODE_NOFAILOVER)
+#define nodeSupportsExtensions(n) ((n)->flags & CLUSTER_NODE_EXTENSIONS_SUPPORTED)
 
 /* This structure represent elements of node->fail_reports. */
 typedef struct clusterNodeFailReport {

src/defrag.c

@@ -718,8 +718,9 @@ void defragStream(redisDb *db, dictEntry *kde) {
 void defragModule(redisDb *db, dictEntry *kde) {
     robj *obj = dictGetVal(kde);
     serverAssert(obj->type == OBJ_MODULE);
-
-    if (!moduleDefragValue(dictGetKey(kde), obj, db->id))
+    robj keyobj;
+    initStaticStringObject(keyobj, dictGetKey(kde));
+    if (!moduleDefragValue(&keyobj, obj, db->id))
         defragLater(db, kde);
 }
 
@@ -929,7 +930,9 @@ int defragLaterItem(dictEntry *de, unsigned long *cursor, long long endtime, int
         } else if (ob->type == OBJ_STREAM) {
             return scanLaterStreamListpacks(ob, cursor, endtime);
         } else if (ob->type == OBJ_MODULE) {
-            return moduleLateDefrag(dictGetKey(de), ob, cursor, endtime, dbid);
+            robj keyobj;
+            initStaticStringObject(keyobj, dictGetKey(de));
+            return moduleLateDefrag(&keyobj, ob, cursor, endtime, dbid);
         } else {
             *cursor = 0; /* object type may have changed since we schedule it for later */
         }

src/eval.c

@@ -259,6 +259,7 @@ void scriptingInit(int setup) {
 void freeLuaScriptsSync(dict *lua_scripts, list *lua_scripts_lru_list, lua_State *lua) {
     dictRelease(lua_scripts);
     listRelease(lua_scripts_lru_list);
+    lua_gc(lua, LUA_GCCOLLECT, 0);
     lua_close(lua);
 
 #if defined(USE_JEMALLOC)

src/function_lua.c

@@ -189,6 +189,7 @@ static void luaEngineFreeFunction(void *engine_ctx, void *compiled_function) {
 
 static void luaEngineFreeCtx(void *engine_ctx) {
     luaEngineCtx *lua_engine_ctx = engine_ctx;
+    lua_gc(lua_engine_ctx->lua, LUA_GCCOLLECT, 0);
     lua_close(lua_engine_ctx->lua);
     zfree(lua_engine_ctx);
 }

src/functions.c

@@ -171,7 +171,7 @@ void functionsLibCtxClear(functionsLibCtx *lib_ctx) {
         stats->n_lib = 0;
     }
     dictReleaseIterator(iter);
-    curr_functions_lib_ctx->cache_memory = 0;
+    lib_ctx->cache_memory = 0;
 }
 
 void functionsLibCtxClearCurrent(int async) {

src/kvstore.c

@@ -124,7 +124,9 @@ static void cumulativeKeyCountAdd(kvstore *kvs, int didx, long delta) {
 
     dict *d = kvstoreGetDict(kvs, didx);
     size_t dsize = dictSize(d);
-    int non_empty_dicts_delta = dsize == 1? 1 : dsize == 0? -1 : 0;
+    /* Increment if dsize is 1 and delta is positive (first element inserted, dict becomes non-empty).
+     * Decrement if dsize is 0 (dict becomes empty). */
+    int non_empty_dicts_delta = (dsize == 1 && delta > 0) ? 1 : (dsize == 0) ? -1 : 0;
     kvs->non_empty_dicts += non_empty_dicts_delta;
 
     /* BIT does not need to be calculated when there's only one dict. */
@@ -1026,6 +1028,31 @@ int kvstoreTest(int argc, char **argv, int flags) {
         kvstoreRelease(kvs);
     }
 
+    TEST("Verify non-empty dict count is correctly updated") {
+        kvstore *kvs = kvstoreCreate(&KvstoreDictTestType, 2, KVSTORE_ALLOCATE_DICTS_ON_DEMAND);
+        for (int idx = 0; idx < 4; idx++) {
+            for (i = 0; i < 16; i++) {
+                de = kvstoreDictAddRaw(kvs, idx, stringFromInt(i), NULL);
+                assert(de != NULL);
+                /* When the first element is inserted, the number of non-empty dictionaries is increased by 1. */
+                if (i == 0) assert(kvstoreNumNonEmptyDicts(kvs) == idx + 1);
+            }
+        }
+
+        /* Step by step, clear all dictionaries and ensure non-empty dict count is updated */
+        for (int idx = 0; idx < 4; idx++) {
+            kvs_di = kvstoreGetDictSafeIterator(kvs, idx);
+            while((de = kvstoreDictIteratorNext(kvs_di)) != NULL) {
+                key = dictGetKey(de);
+                assert(kvstoreDictDelete(kvs, idx, key) == DICT_OK);
+                /* When the dictionary is emptied, the number of non-empty dictionaries is reduced by 1. */
+                if (kvstoreDictSize(kvs, idx) == 0) assert(kvstoreNumNonEmptyDicts(kvs) == 3 - idx);
+            }
+            kvstoreReleaseDictIterator(kvs_di);
+        }
+        kvstoreRelease(kvs);
+    }
+
     kvstoreRelease(kvs1);
     kvstoreRelease(kvs2);
     return 0;

src/rdb.c

@@ -2332,6 +2332,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error)
                 rdbReportCorruptRDB("invalid expireAt time: %llu",
                                     (unsigned long long) expireAt);
                 decrRefCount(o);
+                if (dupSearchDict != NULL) dictRelease(dupSearchDict);
                 return NULL;
             }
 

src/sort.c

@@ -241,8 +241,12 @@ void sortCommandGeneric(client *c, int readonly) {
         } else if (!strcasecmp(c->argv[j]->ptr,"get") && leftargs >= 1) {
             /* If GET is specified with a real pattern, we can't accept it in cluster mode,
              * unless we can make sure the keys formed by the pattern are in the same slot 
-             * as the key to sort. */
-            if (server.cluster_enabled && patternHashSlot(c->argv[j+1]->ptr, sdslen(c->argv[j+1]->ptr)) != getKeySlot(c->argv[1]->ptr)) {
+             * as the key to sort. The pattern # represents the key itself, so just skip
+             * pattern slot check. */
+            if (server.cluster_enabled &&
+                strcmp(c->argv[j+1]->ptr, "#") &&
+                patternHashSlot(c->argv[j+1]->ptr, sdslen(c->argv[j+1]->ptr)) != getKeySlot(c->argv[1]->ptr))
+            {
                 addReplyError(c, "GET option of SORT denied in Cluster mode when "
                               "keys formed by the pattern may be in different slots.");
                 syntax_error++;

src/t_hash.c

@@ -1134,6 +1134,20 @@ int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cm
         dictEntry *de = dbFind(c->db, key->ptr);
         serverAssert(de != NULL);
         lpt->key = dictGetKey(de);
+    } else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) {
+        listpackEx *lpt = o->ptr;
+
+        /* If the hash previously had HFEs but later no longer does, the key ref
+         * (lpt->key) in the hash might become outdated after a MOVE/COPY/RENAME/RESTORE
+         * operation. These commands maintain the key ref only if HFEs are present.
+         * That is, we can only be sure that key ref is valid as long as it is not
+         * "trash". (TODO: dbFind() can be avoided. Instead need to extend the
+         * lookupKey*() to return dictEntry). */
+        if (lpt->meta.trash) {
+            dictEntry *de = dbFind(c->db, key->ptr);
+            serverAssert(de != NULL);
+            lpt->key = dictGetKey(de);
+        }
     } else if (o->encoding == OBJ_ENCODING_HT) {
         /* Take care dict has HFE metadata */
         if (!isDictWithMetaHFE(ht)) {
@@ -1151,6 +1165,18 @@ int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cm
             m->key = dictGetKey(de); /* reference key in keyspace */
             m->hfe = ebCreate();     /* Allocate HFE DS */
             m->expireMeta.trash = 1; /* mark as trash (as long it wasn't ebAdd()) */
+        } else {
+            dictExpireMetadata *m = (dictExpireMetadata *) dictMetadata(ht);
+            /* If the hash previously had HFEs but later no longer does, the key ref
+             * (m->key) in the hash might become outdated after a MOVE/COPY/RENAME/RESTORE
+             * operation. These commands maintain the key ref only if HFEs are present.
+             * That is, we can only be sure that key ref is valid as long as it is not
+             * "trash". */
+            if (m->expireMeta.trash) {
+                dictEntry *de = dbFind(db, key->ptr);
+                serverAssert(de != NULL);
+                m->key = dictGetKey(de); /* reference key in keyspace */
+            }
         }
     }
 
@@ -1985,6 +2011,21 @@ uint64_t hashTypeRemoveFromExpires(ebuckets *hexpires, robj *o) {
     return expireTime;
 }
 
+int hashTypeIsFieldsWithExpire(robj *o) {
+    if (o->encoding == OBJ_ENCODING_LISTPACK) {
+        return 0;
+    } else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) {
+        return EB_EXPIRE_TIME_INVALID != listpackExGetMinExpire(o);
+    } else { /* o->encoding == OBJ_ENCODING_HT */
+        dict *d = o->ptr;
+        /* If dict doesn't holds HFE metadata */
+        if (!isDictWithMetaHFE(d))
+            return 0;
+        dictExpireMetadata *meta = (dictExpireMetadata *) dictMetadata(d);
+        return ebGetTotalItems(meta->hfe, &hashFieldExpireBucketsType) != 0;
+    }
+}
+
 /* Add hash to global HFE DS and update key for notifications.
  *
  * key         - must be the same key instance that is persisted in db->dict
@@ -2315,6 +2356,14 @@ void hdelCommand(client *c) {
     if ((o = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL ||
         checkType(c,o,OBJ_HASH)) return;
 
+    /* Hash field expiration is optimized to avoid frequent update global HFE DS for
+     * each field deletion. Eventually active-expiration will run and update or remove
+     * the hash from global HFE DS gracefully. Nevertheless, statistic "subexpiry"
+     * might reflect wrong number of hashes with HFE to the user if it is the last
+     * field with expiration. The following logic checks if this is indeed the last
+     * field with expiration and removes it from global HFE DS. */
+    int isHFE = hashTypeIsFieldsWithExpire(o);
+
     for (j = 2; j < c->argc; j++) {
         if (hashTypeDelete(o,c->argv[j]->ptr,1)) {
             deleted++;
@@ -2328,9 +2377,13 @@ void hdelCommand(client *c) {
     if (deleted) {
         signalModifiedKey(c,c->db,c->argv[1]);
         notifyKeyspaceEvent(NOTIFY_HASH,"hdel",c->argv[1],c->db->id);
-        if (keyremoved)
-            notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],
-                                c->db->id);
+        if (keyremoved) {
+            notifyKeyspaceEvent(NOTIFY_GENERIC, "del", c->argv[1], c->db->id);
+        } else {
+            if (isHFE && (hashTypeIsFieldsWithExpire(o) == 0)) /* is it last HFE */
+                ebRemove(&c->db->hexpires, &hashExpireBucketsType, o);
+        }
+
         server.dirty += deleted;
     }
     addReplyLongLong(c,deleted);

src/t_stream.c

@@ -1405,11 +1405,6 @@ int streamRangeHasTombstones(stream *s, streamID *start, streamID *end) {
         return 0;
     }
 
-    if (streamCompareID(&s->first_id,&s->max_deleted_entry_id) > 0) {
-        /* The latest tombstone is before the first entry. */
-        return 0;
-    }
-
     if (start) {
         start_id = *start;
     } else {
@@ -1446,6 +1441,17 @@ void streamReplyWithCGLag(client *c, stream *s, streamCG *cg) {
         /* The lag of a newly-initialized stream is 0. */
         lag = 0;
         valid = 1;
+    } else if (!s->length) { /* All entries deleted, now empty. */
+        lag = 0;
+        valid = 1;
+    } else if (streamCompareID(&cg->last_id,&s->first_id) < 0 &&
+               streamCompareID(&s->max_deleted_entry_id,&s->first_id) < 0)
+    {
+        /* When both the consumer group's last_id and the maximum tombstone are behind
+         * the stream's first entry, the consumer group's lag will always be equal to
+         * the number of remainin entries in the stream. */
+        lag = s->length;
+        valid = 1;
     } else if (cg->entries_read != SCG_INVALID_ENTRIES_READ && !streamRangeHasTombstones(s,&cg->last_id,NULL)) {
         /* No fragmentation ahead means that the group's logical reads counter
          * is valid for performing the lag calculation. */
@@ -1502,6 +1508,10 @@ long long streamEstimateDistanceFromFirstEverEntry(stream *s, streamID *id) {
         return s->entries_added;
     }
 
+    /* There are fragmentations between the `id` and the stream's last-generated-id. */
+    if (!streamIDEqZero(id) && streamCompareID(id,&s->max_deleted_entry_id) < 0)
+        return SCG_INVALID_ENTRIES_READ;
+
     int cmp_last = streamCompareID(id,&s->last_id);
     if (cmp_last == 0) {
         /* Return the exact counter of the last entry in the stream. */
@@ -1684,10 +1694,10 @@ size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end
     while(streamIteratorGetID(&si,&id,&numfields)) {
         /* Update the group last_id if needed. */
         if (group && streamCompareID(&id,&group->last_id) > 0) {
-            if (group->entries_read != SCG_INVALID_ENTRIES_READ && !streamRangeHasTombstones(s,&id,NULL)) {
-                /* A valid counter and no future tombstones mean we can 
-                 * increment the read counter to keep tracking the group's
-                 * progress. */
+            if (group->entries_read != SCG_INVALID_ENTRIES_READ && !streamRangeHasTombstones(s,&group->last_id,NULL)) {
+                /* A valid counter and no tombstones between the group's last-delivered-id
+                 * and the stream's last-generated-id mean we can increment the read counter
+                 * to keep tracking the group's progress. */
                 group->entries_read++;
             } else if (s->entries_added) {
                 /* The group's counter may be invalid, so we try to obtain it. */

src/util.c

@@ -55,8 +55,11 @@
 
 /* Glob-style pattern matching. */
 static int stringmatchlen_impl(const char *pattern, int patternLen,
-        const char *string, int stringLen, int nocase, int *skipLongerMatches)
+        const char *string, int stringLen, int nocase, int *skipLongerMatches, int nesting)
 {
+    /* Protection against abusive patterns. */
+    if (nesting > 1000) return 0;
+
     while(patternLen && stringLen) {
         switch(pattern[0]) {
         case '*':
@@ -68,7 +71,7 @@ static int stringmatchlen_impl(const char *pattern, int patternLen,
                 return 1; /* match */
             while(stringLen) {
                 if (stringmatchlen_impl(pattern+1, patternLen-1,
-                            string, stringLen, nocase, skipLongerMatches))
+                            string, stringLen, nocase, skipLongerMatches, nesting+1))
                     return 1; /* match */
                 if (*skipLongerMatches)
                     return 0; /* no match */
@@ -190,7 +193,7 @@ static int stringmatchlen_impl(const char *pattern, int patternLen,
 int stringmatchlen(const char *pattern, int patternLen,
         const char *string, int stringLen, int nocase) {
     int skipLongerMatches = 0;
-    return stringmatchlen_impl(pattern,patternLen,string,stringLen,nocase,&skipLongerMatches);
+    return stringmatchlen_impl(pattern,patternLen,string,stringLen,nocase,&skipLongerMatches,0);
 }
 
 int stringmatch(const char *pattern, const char *string, int nocase) {

src/version.h

@@ -1,2 +1,2 @@
-#define REDIS_VERSION "255.255.255"
-#define REDIS_VERSION_NUM 0x00ffffff
+#define REDIS_VERSION "7.4.2"
+#define REDIS_VERSION_NUM 0x00070402

tests/cluster/tests/28-cluster-shards.tcl

@@ -124,6 +124,10 @@ test "Verify health as fail for killed node" {
     }
 }
 
+test "Verify that other nodes can correctly output the new master's slots" {
+    assert_not_equal {} [dict get [get_node_info_from_shard [R 4 CLUSTER MYID] 8 "shard"] slots]
+}
+
 set primary_id 4
 set replica_id 0
 

tests/modules/defragtest.c

@@ -141,13 +141,14 @@ size_t FragFreeEffort(RedisModuleString *key, const void *value) {
 }
 
 int FragDefrag(RedisModuleDefragCtx *ctx, RedisModuleString *key, void **value) {
-    REDISMODULE_NOT_USED(key);
     unsigned long i = 0;
     int steps = 0;
 
     int dbid = RedisModule_GetDbIdFromDefragCtx(ctx);
     RedisModule_Assert(dbid != -1);
 
+    RedisModule_Log(NULL, "notice", "Defrag key: %s", RedisModule_StringPtrLen(key, NULL));
+
     /* Attempt to get cursor, validate it's what we're exepcting */
     if (RedisModule_DefragCursorGet(ctx, &i) == REDISMODULE_OK) {
         if (i > 0) datatype_resumes++;

tests/unit/acl-v2.tcl

@@ -116,6 +116,32 @@ start_server {tags {"acl external:skip"}} {
         assert_match "*NOPERM*key*" $err
     }
 
+    test {Validate read and write permissions format - empty permission} {
+        catch {r ACL SETUSER key-permission-RW %~} err
+        set err
+    } {ERR Error in ACL SETUSER modifier '%~': Syntax error}
+
+    test {Validate read and write permissions format - empty selector} {
+        catch {r ACL SETUSER key-permission-RW %} err
+        set err
+    } {ERR Error in ACL SETUSER modifier '%': Syntax error}
+
+    test {Validate read and write permissions format - empty pattern} {
+        # Empty pattern results with R/W access to no key
+        r ACL SETUSER key-permission-RW on nopass %RW~ +@all
+        $r2 auth key-permission-RW password
+        catch {$r2 SET x 5} err
+        set err
+    } {NOPERM No permissions to access a key}
+
+    test {Validate read and write permissions format - no pattern} {
+        # No pattern results with R/W access to no key (currently we accept this syntax error)
+        r ACL SETUSER key-permission-RW on nopass %RW +@all
+        $r2 auth key-permission-RW password
+        catch {$r2 SET x 5} err
+        set err
+    } {NOPERM No permissions to access a key}
+
     test {Test separate read and write permissions on different selectors are not additive} {
         r ACL SETUSER key-permission-RW-selector on nopass "(%R~read* +@all)" "(%W~write* +@all)"
         $r2 auth key-permission-RW-selector password

tests/unit/cluster/hostnames.tcl

@@ -1,3 +1,16 @@
+#
+# Copyright (c) 2009-Present, Redis Ltd.
+# All rights reserved.
+#
+# Copyright (c) 2024-present, Valkey contributors.
+# All rights reserved.
+#
+# Licensed under your choice of the Redis Source Available License 2.0
+# (RSALv2) or the Server Side Public License v1 (SSPLv1).
+#
+# Portions of this file are available under BSD3 terms; see REDISCONTRIBUTIONS for more information.
+#
+
 proc get_slot_field {slot_output shard_id node_id attrib_id} {
     return [lindex [lindex [lindex $slot_output $shard_id] $node_id] $attrib_id]
 }
@@ -116,10 +129,11 @@ test "Verify the nodes configured with prefer hostname only show hostname for ne
     # Have everyone forget node 6 and isolate it from the cluster.
     isolate_node 6
 
-    # Set hostnames for the masters, now that the node is isolated
-    R 0 config set cluster-announce-hostname "shard-1.com"
-    R 1 config set cluster-announce-hostname "shard-2.com"
-    R 2 config set cluster-announce-hostname "shard-3.com"
+    set primaries 3
+    for {set j 0} {$j < $primaries} {incr j} {
+        # Set hostnames for the masters, now that the node is isolated
+        R $j config set cluster-announce-hostname "shard-$j.com"
+    }
 
     # Prevent Node 0 and Node 6 from properly meeting,
     # they'll hang in the handshake phase. This allows us to 
@@ -149,9 +163,17 @@ test "Verify the nodes configured with prefer hostname only show hostname for ne
     } else {
         fail "Node did not learn about the 2 shards it can talk to"
     }
-    set slot_result [R 6 CLUSTER SLOTS]
-    assert_equal [lindex [get_slot_field $slot_result 0 2 3] 1] "shard-2.com"
-    assert_equal [lindex [get_slot_field $slot_result 1 2 3] 1] "shard-3.com"
+    wait_for_condition 50 100 {
+        [lindex [get_slot_field [R 6 CLUSTER SLOTS] 0 2 3] 1] eq "shard-1.com"
+    } else {
+        fail "hostname for shard-1 didn't reach node 6"
+    }
+
+    wait_for_condition 50 100 {
+        [lindex [get_slot_field [R 6 CLUSTER SLOTS] 1 2 3] 1] eq "shard-2.com"
+    } else {
+        fail "hostname for shard-2 didn't reach node 6"
+    }
 
     # Also make sure we know about the isolated master, we 
     # just can't reach it.
@@ -170,10 +192,14 @@ test "Verify the nodes configured with prefer hostname only show hostname for ne
     } else {
         fail "Node did not learn about the 2 shards it can talk to"
     }
-    set slot_result [R 6 CLUSTER SLOTS]
-    assert_equal [lindex [get_slot_field $slot_result 0 2 3] 1] "shard-1.com"
-    assert_equal [lindex [get_slot_field $slot_result 1 2 3] 1] "shard-2.com"
-    assert_equal [lindex [get_slot_field $slot_result 2 2 3] 1] "shard-3.com"
+
+    for {set j 0} {$j < $primaries} {incr j} {
+        wait_for_condition 50 100 {
+            [lindex [get_slot_field [R 6 CLUSTER SLOTS] $j 2 3] 1] eq "shard-$j.com"
+        } else {
+            fail "hostname information for shard-$j didn't reach node 6"
+        }
+    }
 }
 
 test "Test restart will keep hostname information" {

tests/unit/keyspace.tcl

@@ -509,6 +509,12 @@ foreach {type large} [array get largevalue] {
         r KEYS "a*a*a*a*a*a*a*a*a*a*a*a*a*a*a*a*a*a*a*a*b"
     } {}
 
+    test {Regression for pattern matching very long nested loops} {
+        r flushdb
+        r SET [string repeat "a" 50000] 1
+        r KEYS [string repeat "*?" 50000]
+    } {}
+
     test {Coverage: basic SWAPDB test and unhappy path} {
        r flushall
        r select 0

tests/unit/scripting.tcl

@@ -691,6 +691,12 @@ start_server {tags {"scripting"}} {
         set e
     } {ERR *Attempt to modify a readonly table*}
 
+    test {lua bit.tohex bug} {
+        set res [run_script {return bit.tohex(65535, -2147483648)} 0]
+        r ping
+        set res
+    } {0000FFFF}
+
     test {Test an example script DECR_IF_GT} {
         set decr_if_gt {
             local current

tests/unit/sort.tcl

@@ -73,7 +73,7 @@ start_server {
     set result [create_random_dataset 16 lpush]
     test "SORT GET #" {
         assert_equal [lsort -integer $result] [r sort tosort GET #]
-    } {} {cluster:skip}
+    }
 
 foreach command {SORT SORT_RO} {
     test "$command GET <const>" {
@@ -393,6 +393,11 @@ start_cluster 1 0 {tags {"external:skip cluster sort"}} {
         r sort "{a}mylist" by "{a}by*" get "{a}get*"
     } {30 200 100}
 
+    test "sort get # in cluster mode" {
+        assert_equal [r sort "{a}mylist" by "{a}by*" get # ] {3 1 2}
+        r sort "{a}mylist" by "{a}by*" get "{a}get*" get #
+    } {30 3 200 1 100 2}
+
     test "sort_ro by in cluster mode" {
         catch {r sort_ro "{a}mylist" by by*} e
         assert_match {ERR BY option of SORT denied in Cluster mode when *} $e
@@ -404,4 +409,9 @@ start_cluster 1 0 {tags {"external:skip cluster sort"}} {
         assert_match {ERR GET option of SORT denied in Cluster mode when *} $e
         r sort_ro "{a}mylist" by "{a}by*" get "{a}get*"
     } {30 200 100}
+
+    test "sort_ro get # in cluster mode" {
+        assert_equal [r sort_ro "{a}mylist" by "{a}by*" get # ] {3 1 2}
+        r sort_ro "{a}mylist" by "{a}by*" get "{a}get*" get #
+    } {30 3 200 1 100 2}
 }

tests/unit/type/hash-field-expire.tcl

@@ -199,12 +199,12 @@ start_server {tags {"external:skip needs:debug"}} {
             set hash_sizes {1 15 16 17 31 32 33 40}
             foreach h $hash_sizes {
                 for {set i 1} {$i <= $h} {incr i} {
-                    # random expiration time
+                    # Random expiration time (Take care expired not after "mix$h")
                     r hset hrand$h f$i v$i
-                    r hpexpire hrand$h [expr {50 + int(rand() * 50)}] FIELDS 1 f$i
+                    r hpexpire hrand$h [expr {70 + int(rand() * 30)}] FIELDS 1 f$i
                     assert_equal 1 [r HEXISTS hrand$h f$i]
 
-                    # same expiration time
+                    # Same expiration time (Take care expired not after "mix$h")
                     r hset same$h f$i v$i
                     r hpexpire same$h 100 FIELDS 1 f$i
                     assert_equal 1 [r HEXISTS same$h f$i]
@@ -286,10 +286,9 @@ start_server {tags {"external:skip needs:debug"}} {
         test "HEXPIRETIME - returns TTL in Unix timestamp ($type)" {
             r del myhash
             r HSET myhash field1 value1
-            r HPEXPIRE myhash 1000 NX FIELDS 1 field1
-
             set lo [expr {[clock seconds] + 1}]
             set hi [expr {[clock seconds] + 2}]
+            r HPEXPIRE myhash 1000 NX FIELDS 1 field1
             assert_range [r HEXPIRETIME myhash FIELDS 1 field1] $lo $hi
             assert_range [r HPEXPIRETIME myhash FIELDS 1 field1] [expr $lo*1000] [expr $hi*1000]
         }
@@ -592,6 +591,20 @@ start_server {tags {"external:skip needs:debug"}} {
             wait_for_condition 30 10 { [r exists myhash2] == 0 } else { fail "`myhash2` should be expired" }
         }
 
+        test "Test RENAME hash that had HFEs but not during the rename ($type)" {
+            r del h1
+            r hset h1 f1 v1 f2 v2
+            r hpexpire h1 1 FIELDS 1 f1
+            after 20
+            r rename h1 h1_renamed
+            assert_equal [r exists h1] 0
+            assert_equal [r exists h1_renamed] 1
+            assert_equal [r hgetall h1_renamed] {f2 v2}
+            r hpexpire h1_renamed 1 FIELDS 1 f2
+            # Only active expire will delete the key
+            wait_for_condition 30 10 { [r exists h1_renamed] == 0 } else { fail "`h1_renamed` should be expired" }
+        }
+
         test "MOVE to another DB hash with fields to be expired ($type)" {
             r select 9
             r flushall
@@ -635,6 +648,20 @@ start_server {tags {"external:skip needs:debug"}} {
 
         } {} {singledb:skip}
 
+        test "Test COPY hash that had HFEs but not during the copy ($type)" {
+            r del h1
+            r hset h1 f1 v1 f2 v2
+            r hpexpire h1 1 FIELDS 1 f1
+            after 20
+            r COPY h1 h1_copy
+            assert_equal [r exists h1] 1
+            assert_equal [r exists h1_copy] 1
+            assert_equal [r hgetall h1_copy] {f2 v2}
+            r hpexpire h1_copy 1 FIELDS 1 f2
+            # Only active expire will delete the key
+            wait_for_condition 30 10 { [r exists h1_copy] == 0 } else { fail "`h1_copy` should be expired" }
+        }
+
         test "Test SWAPDB hash-fields to be expired ($type)" {
             r select 9
             r flushall
@@ -656,6 +683,29 @@ start_server {tags {"external:skip needs:debug"}} {
             wait_for_condition 20 10 { [r exists myhash] == 0 } else { fail "'myhash' should be expired" }
         } {} {singledb:skip}
 
+        test "Test SWAPDB hash that had HFEs but not during the swap ($type)" {
+            r select 9
+            r flushall
+            r hset myhash f1 v1 f2 v2
+            r hpexpire myhash 1 NX FIELDS 1 f1
+            after 10
+
+            r swapdb 9 10
+
+            # Verify the key and its field doesn't exist in the source DB
+            assert_equal [r exists myhash] 0
+            assert_equal [r dbsize] 0
+
+            # Verify the key and its field exists in the target DB
+            r select 10
+            assert_equal [r hgetall myhash] {f2 v2}
+            assert_equal [r dbsize] 1
+            r hpexpire myhash 1 NX FIELDS 1 f2
+
+            # Eventually the field will be expired and the key will be deleted
+            wait_for_condition 20 10 { [r exists myhash] == 0 } else { fail "'myhash' should be expired" }
+        } {} {singledb:skip}
+
         test "HMGET - returns empty entries if fields or hash expired ($type)" {
             r debug set-active-expire 0
             r del h1 h2
@@ -724,6 +774,20 @@ start_server {tags {"external:skip needs:debug"}} {
             assert_equal [r hexpiretime myhash FIELDS 3 a b c] {2524600800 2524600801 -1}
         }
 
+        test {RESTORE hash that had in the past HFEs but not during the dump} {
+            r config set sanitize-dump-payload yes
+            r del myhash
+            r hmset myhash a 1 b 2 c 3
+            r hpexpire myhash 1 fields 1 a
+            after 10
+            set encoded [r dump myhash]
+            r del myhash
+            r restore myhash 0 $encoded
+            assert_equal [lsort [r hgetall myhash]] "2 3 b c"
+            r hpexpire myhash 1 fields 2 b c
+            wait_for_condition 30 10 { [r exists myhash] == 0 } else { fail "`myhash` should be expired" }
+        }
+
         test {DUMP / RESTORE are able to serialize / unserialize a hash with TTL 0 for all fields} {
             r config set sanitize-dump-payload yes
             r del myhash
@@ -786,6 +850,36 @@ start_server {tags {"external:skip needs:debug"}} {
         }
     }
 
+    test "Statistics - Hashes with HFEs ($type)" {
+        r config resetstat
+        r flushall
+
+        # hash1: 5 fields, 3 with TTL. subexpiry incr +1
+        r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
+        r hpexpire myhash 150 FIELDS 3 f1 f2 f3
+        assert_match  [get_stat_subexpiry r] 1
+
+        # hash2: 5 fields, 3 with TTL. subexpiry incr +1
+        r hset myhash2 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
+        assert_match  [get_stat_subexpiry r] 1
+        r hpexpire myhash2 100 FIELDS 3 f1 f2 f3
+        assert_match  [get_stat_subexpiry r] 2
+
+        # hash3: 2 fields, 1 with TTL. HDEL field with TTL. subexpiry decr -1
+        r hset myhash3 f1 v1 f2 v2
+        r hpexpire myhash3 100 FIELDS 1 f2
+        assert_match  [get_stat_subexpiry r] 3
+        r hdel myhash3 f2
+        assert_match  [get_stat_subexpiry r] 2
+
+        # Expired fields of hash1 and hash2. subexpiry decr -2
+        wait_for_condition 50 50 {
+                [get_stat_subexpiry r] == 0
+        } else {
+                fail "Hash field expiry statistics failed"
+        }
+    }
+
     r config set hash-max-listpack-entries 512
 }
 
@@ -899,25 +993,6 @@ start_server {tags {"external:skip needs:debug"}} {
 
         r config set hash-max-listpack-value 64
     }
-
-    test "Statistics - Hashes with HFEs" {
-        r config resetstat
-        r del myhash
-        r hset myhash f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
-        r hpexpire myhash 100 FIELDS 3 f1 f2 f3
-
-        assert_match  [get_stat_subexpiry r] 1
-        r hset myhash2 f1 v1 f2 v2 f3 v3 f4 v4 f5 v5
-        assert_match  [get_stat_subexpiry r] 1
-        r hpexpire myhash2 100 FIELDS 3 f1 f2 f3
-        assert_match  [get_stat_subexpiry r] 2
-
-        wait_for_condition 50 50 {
-                [get_stat_subexpiry r] == 0
-        } else {
-                fail "Hash field expiry statistics failed"
-        }
-    }
 }
 
 start_server {tags {"external:skip needs:debug"}} {
@@ -932,6 +1007,7 @@ start_server {tags {"external:skip needs:debug"}} {
             start_server {overrides {appendonly {yes} appendfsync always} tags {external:skip}} {
 
                 set aof [get_last_incr_aof_path r]
+                r debug set-active-expire 0 ;# Prevent fields from being expired during data preparation
 
                 # Time is in the past so it should propagate HDELs to replica
                 # and delete the fields
@@ -958,6 +1034,7 @@ start_server {tags {"external:skip needs:debug"}} {
                 r hpexpireat h2 [expr [clock seconds]*1000+100000] LT FIELDS 1 f3
                 r hexpireat h2 [expr [clock seconds]+10] NX FIELDS 1 f4
 
+                r debug set-active-expire 1
                 wait_for_condition 50 100 {
                     [r hlen h2] eq 2
                 } else {
@@ -986,7 +1063,7 @@ start_server {tags {"external:skip needs:debug"}} {
                     {hdel h2 f2}
                 }
             }
-        }
+        } {} {needs:debug}
 
         test "Lazy Expire - fields are lazy deleted and propagated to replicas ($type)" {
             start_server {overrides {appendonly {yes} appendfsync always} tags {external:skip}} {

tests/unit/type/stream-cgroups.tcl

@@ -1239,6 +1239,74 @@ start_server {
         assert_equal [dict get $group lag] 2
     }
 
+    test {Consumer Group Lag with XDELs and tombstone after the last_id of consume group} {
+        r DEL x
+        r XGROUP CREATE x g1 $ MKSTREAM
+        r XADD x 1-0 data a
+        r XREADGROUP GROUP g1 alice STREAMS x > ;# Read one entry
+        r XADD x 2-0 data c
+        r XADD x 3-0 data d
+        r XDEL x 2-0
+
+        # Now the latest tombstone(2-0) is before the first entry(3-0), but there is still
+        # a tombstone(2-0) after the last_id(1-0) of the consume group.
+        set reply [r XINFO STREAM x FULL]
+        set group [lindex [dict get $reply groups] 0]
+        assert_equal [dict get $group entries-read] 1
+        assert_equal [dict get $group lag] {}
+
+        r XDEL x 1-0
+        # Although there is a tombstone(2-0) after the consumer group's last_id(1-0), all
+        # entries before the maximal tombstone have been deleted. This means that both the
+        # last_id and the largest tombstone are behind the first entry. Therefore, tombstones
+        # no longer affect the lag, which now reflects the remaining entries in the stream.
+        set reply [r XINFO STREAM x FULL]
+        set group [lindex [dict get $reply groups] 0]
+        assert_equal [dict get $group entries-read] 1
+        assert_equal [dict get $group lag] 1
+
+        # Now there is a tombstone(2-0) after the last_id of the consume group, so after consuming
+        # entry(3-0), the group's counter will be invalid.
+        r XREADGROUP GROUP g1 alice STREAMS x > 
+        set reply [r XINFO STREAM x FULL]
+        set group [lindex [dict get $reply groups] 0]
+        assert_equal [dict get $group entries-read] 3
+        assert_equal [dict get $group lag] 0
+    }
+
+    test {Consumer group lag with XTRIM} {
+        r DEL x
+        r XGROUP CREATE x mygroup $ MKSTREAM
+        r XADD x 1-0 data a
+        r XADD x 2-0 data b
+        r XADD x 3-0 data c
+        r XADD x 4-0 data d
+        r XADD x 5-0 data e
+        r XREADGROUP GROUP mygroup alice COUNT 1 STREAMS x >
+
+        set reply [r XINFO STREAM x FULL]
+        set group [lindex [dict get $reply groups] 0]
+        assert_equal [dict get $group entries-read] 1
+        assert_equal [dict get $group lag] 4
+
+        # Although XTRIM doesn't update the `max-deleted-entry-id`, it always updates the
+        # position of the first entry. When trimming causes the first entry to be behind
+        # the consumer group's last_id, the consumer group's lag will always be equal to
+        # the number of remainin entries in the stream.
+        r XTRIM x MAXLEN 1
+        set reply [r XINFO STREAM x FULL]
+        set group [lindex [dict get $reply groups] 0]
+        assert_equal [dict get $reply max-deleted-entry-id] "0-0"
+        assert_equal [dict get $group entries-read] 1
+        assert_equal [dict get $group lag] 1
+
+        # When all the entries were deleted, the lag is always 0.
+        r XTRIM x MAXLEN 0
+        set reply [r XINFO STREAM x FULL]
+        set group [lindex [dict get $reply groups] 0]
+        assert_equal [dict get $group lag] 0
+    }
+
     test {Loading from legacy (Redis <= v6.2.x, rdb_ver < 10) persistence} {
         # The payload was DUMPed from a v5 instance after:
         # XADD x 1-0 data a