Commit 80458d04 authored by YaacovHazan's avatar YaacovHazan
Browse files

Merge remote-tracking branch 'upstream/unstable' into HEAD

parents 5fe3e74a 99c40ab5
...@@ -60,10 +60,13 @@ typedef long long ustime_t; ...@@ -60,10 +60,13 @@ typedef long long ustime_t;
#define REDISMODULE_OPEN_KEY_NOEXPIRE (1<<19) #define REDISMODULE_OPEN_KEY_NOEXPIRE (1<<19)
/* Avoid any effects from fetching the key */ /* Avoid any effects from fetching the key */
#define REDISMODULE_OPEN_KEY_NOEFFECTS (1<<20) #define REDISMODULE_OPEN_KEY_NOEFFECTS (1<<20)
/* Allow access expired key that haven't deleted yet */
#define REDISMODULE_OPEN_KEY_ACCESS_EXPIRED (1<<21)
/* Mask of all REDISMODULE_OPEN_KEY_* values. Any new mode should be added to this list. /* Mask of all REDISMODULE_OPEN_KEY_* values. Any new mode should be added to this list.
* Should not be used directly by the module, use RM_GetOpenKeyModesAll instead. * Should not be used directly by the module, use RM_GetOpenKeyModesAll instead.
* Located here so when we will add new modes we will not forget to update it. */ * Located here so when we will add new modes we will not forget to update it. */
#define _REDISMODULE_OPEN_KEY_ALL REDISMODULE_READ | REDISMODULE_WRITE | REDISMODULE_OPEN_KEY_NOTOUCH | REDISMODULE_OPEN_KEY_NONOTIFY | REDISMODULE_OPEN_KEY_NOSTATS | REDISMODULE_OPEN_KEY_NOEXPIRE | REDISMODULE_OPEN_KEY_NOEFFECTS #define _REDISMODULE_OPEN_KEY_ALL REDISMODULE_READ | REDISMODULE_WRITE | REDISMODULE_OPEN_KEY_NOTOUCH | REDISMODULE_OPEN_KEY_NONOTIFY | REDISMODULE_OPEN_KEY_NOSTATS | REDISMODULE_OPEN_KEY_NOEXPIRE | REDISMODULE_OPEN_KEY_NOEFFECTS | REDISMODULE_OPEN_KEY_ACCESS_EXPIRED
/* List push and pop */ /* List push and pop */
#define REDISMODULE_LIST_HEAD 0 #define REDISMODULE_LIST_HEAD 0
......
...@@ -34,6 +34,7 @@ ...@@ -34,6 +34,7 @@
* ---------------------------------------------------------------------------------------- * ----------------------------------------------------------------------------------------
*/ */
#include "fast_float_strtod.h"
#include "resp_parser.h" #include "resp_parser.h"
#include "server.h" #include "server.h"
...@@ -132,7 +133,7 @@ static int parseDouble(ReplyParser *parser, void *p_ctx) { ...@@ -132,7 +133,7 @@ static int parseDouble(ReplyParser *parser, void *p_ctx) {
if (len <= MAX_LONG_DOUBLE_CHARS) { if (len <= MAX_LONG_DOUBLE_CHARS) {
memcpy(buf,proto+1,len); memcpy(buf,proto+1,len);
buf[len] = '\0'; buf[len] = '\0';
d = strtod(buf,NULL); /* We expect a valid representation. */ d = fast_float_strtod(buf,NULL); /* We expect a valid representation. */
} else { } else {
d = 0; d = 0;
} }
......
...@@ -3221,6 +3221,7 @@ typedef struct dictExpireMetadata { ...@@ -3221,6 +3221,7 @@ typedef struct dictExpireMetadata {
#define HFE_LAZY_AVOID_HASH_DEL (1<<1) /* Avoid deleting hash if the field is the last one */ #define HFE_LAZY_AVOID_HASH_DEL (1<<1) /* Avoid deleting hash if the field is the last one */
#define HFE_LAZY_NO_NOTIFICATION (1<<2) /* Do not send notification, used when multiple fields #define HFE_LAZY_NO_NOTIFICATION (1<<2) /* Do not send notification, used when multiple fields
* may expire and only one notification is desired. */ * may expire and only one notification is desired. */
#define HFE_LAZY_ACCESS_EXPIRED (1<<3) /* Avoid lazy expire and allow access to expired fields */
void hashTypeConvert(robj *o, int enc, ebuckets *hexpires); void hashTypeConvert(robj *o, int enc, ebuckets *hexpires);
void hashTypeTryConversion(redisDb *db, robj *subject, robj **argv, int start, int end); void hashTypeTryConversion(redisDb *db, robj *subject, robj **argv, int start, int end);
...@@ -3383,11 +3384,12 @@ robj *objectCommandLookupOrReply(client *c, robj *key, robj *reply); ...@@ -3383,11 +3384,12 @@ robj *objectCommandLookupOrReply(client *c, robj *key, robj *reply);
int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
long long lru_clock, int lru_multiplier); long long lru_clock, int lru_multiplier);
#define LOOKUP_NONE 0 #define LOOKUP_NONE 0
#define LOOKUP_NOTOUCH (1<<0) /* Don't update LRU. */ #define LOOKUP_NOTOUCH (1<<0) /* Don't update LRU. */
#define LOOKUP_NONOTIFY (1<<1) /* Don't trigger keyspace event on key misses. */ #define LOOKUP_NONOTIFY (1<<1) /* Don't trigger keyspace event on key misses. */
#define LOOKUP_NOSTATS (1<<2) /* Don't update keyspace hits/misses counters. */ #define LOOKUP_NOSTATS (1<<2) /* Don't update keyspace hits/misses counters. */
#define LOOKUP_WRITE (1<<3) /* Delete expired keys even in replicas. */ #define LOOKUP_WRITE (1<<3) /* Delete expired keys even in replicas. */
#define LOOKUP_NOEXPIRE (1<<4) /* Avoid deleting lazy expired keys. */ #define LOOKUP_NOEXPIRE (1<<4) /* Avoid deleting lazy expired keys. */
#define LOOKUP_ACCESS_EXPIRED (1<<5) /* Allow lookup to expired key. */
#define LOOKUP_NOEFFECTS (LOOKUP_NONOTIFY | LOOKUP_NOSTATS | LOOKUP_NOTOUCH | LOOKUP_NOEXPIRE) /* Avoid any effects from fetching the key */ #define LOOKUP_NOEFFECTS (LOOKUP_NONOTIFY | LOOKUP_NOSTATS | LOOKUP_NOTOUCH | LOOKUP_NOEXPIRE) /* Avoid any effects from fetching the key */
dictEntry *dbAdd(redisDb *db, robj *key, robj *val); dictEntry *dbAdd(redisDb *db, robj *key, robj *val);
......
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
* (RSALv2) or the Server Side Public License v1 (SSPLv1). * (RSALv2) or the Server Side Public License v1 (SSPLv1).
*/ */
#include "fast_float_strtod.h"
#include "server.h" #include "server.h"
#include "pqsort.h" /* Partial qsort for SORT+LIMIT */ #include "pqsort.h" /* Partial qsort for SORT+LIMIT */
#include <math.h> /* isnan() */ #include <math.h> /* isnan() */
...@@ -472,7 +472,7 @@ void sortCommandGeneric(client *c, int readonly) { ...@@ -472,7 +472,7 @@ void sortCommandGeneric(client *c, int readonly) {
if (sdsEncodedObject(byval)) { if (sdsEncodedObject(byval)) {
char *eptr; char *eptr;
vector[j].u.score = strtod(byval->ptr,&eptr); vector[j].u.score = fast_float_strtod(byval->ptr,&eptr);
if (eptr[0] != '\0' || errno == ERANGE || if (eptr[0] != '\0' || errno == ERANGE ||
isnan(vector[j].u.score)) isnan(vector[j].u.score))
{ {
......
...@@ -734,7 +734,7 @@ GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vs ...@@ -734,7 +734,7 @@ GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vs
serverPanic("Unknown hash encoding"); serverPanic("Unknown hash encoding");
} }
if (expiredAt >= (uint64_t) commandTimeSnapshot()) if ((expiredAt >= (uint64_t) commandTimeSnapshot()) || (hfeFlags & HFE_LAZY_ACCESS_EXPIRED))
return GETF_OK; return GETF_OK;
if (server.masterhost) { if (server.masterhost) {
...@@ -1134,6 +1134,20 @@ int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cm ...@@ -1134,6 +1134,20 @@ int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cm
dictEntry *de = dbFind(c->db, key->ptr); dictEntry *de = dbFind(c->db, key->ptr);
serverAssert(de != NULL); serverAssert(de != NULL);
lpt->key = dictGetKey(de); lpt->key = dictGetKey(de);
} else if (o->encoding == OBJ_ENCODING_LISTPACK_EX) {
listpackEx *lpt = o->ptr;
/* If the hash previously had HFEs but later no longer does, the key ref
* (lpt->key) in the hash might become outdated after a MOVE/COPY/RENAME/RESTORE
* operation. These commands maintain the key ref only if HFEs are present.
* That is, we can only be sure that key ref is valid as long as it is not
* "trash". (TODO: dbFind() can be avoided. Instead need to extend the
* lookupKey*() to return dictEntry). */
if (lpt->meta.trash) {
dictEntry *de = dbFind(c->db, key->ptr);
serverAssert(de != NULL);
lpt->key = dictGetKey(de);
}
} else if (o->encoding == OBJ_ENCODING_HT) { } else if (o->encoding == OBJ_ENCODING_HT) {
/* Take care dict has HFE metadata */ /* Take care dict has HFE metadata */
if (!isDictWithMetaHFE(ht)) { if (!isDictWithMetaHFE(ht)) {
...@@ -1151,6 +1165,18 @@ int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cm ...@@ -1151,6 +1165,18 @@ int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cm
m->key = dictGetKey(de); /* reference key in keyspace */ m->key = dictGetKey(de); /* reference key in keyspace */
m->hfe = ebCreate(); /* Allocate HFE DS */ m->hfe = ebCreate(); /* Allocate HFE DS */
m->expireMeta.trash = 1; /* mark as trash (as long it wasn't ebAdd()) */ m->expireMeta.trash = 1; /* mark as trash (as long it wasn't ebAdd()) */
} else {
dictExpireMetadata *m = (dictExpireMetadata *) dictMetadata(ht);
/* If the hash previously had HFEs but later no longer does, the key ref
* (m->key) in the hash might become outdated after a MOVE/COPY/RENAME/RESTORE
* operation. These commands maintain the key ref only if HFEs are present.
* That is, we can only be sure that key ref is valid as long as it is not
* "trash". */
if (m->expireMeta.trash) {
dictEntry *de = dbFind(db, key->ptr);
serverAssert(de != NULL);
m->key = dictGetKey(de); /* reference key in keyspace */
}
} }
} }
......
...@@ -2,8 +2,13 @@ ...@@ -2,8 +2,13 @@
* Copyright (c) 2009-Present, Redis Ltd. * Copyright (c) 2009-Present, Redis Ltd.
* All rights reserved. * All rights reserved.
* *
* Copyright (c) 2024-present, Valkey contributors.
* All rights reserved.
*
* Licensed under your choice of the Redis Source Available License 2.0 * Licensed under your choice of the Redis Source Available License 2.0
* (RSALv2) or the Server Side Public License v1 (SSPLv1). * (RSALv2) or the Server Side Public License v1 (SSPLv1).
*
* Portions of this file are available under BSD3 terms; see REDISCONTRIBUTIONS for more information.
*/ */
#include "server.h" #include "server.h"
...@@ -1492,6 +1497,7 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum, ...@@ -1492,6 +1497,7 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum,
robj **sets = zmalloc(sizeof(robj*)*setnum); robj **sets = zmalloc(sizeof(robj*)*setnum);
setTypeIterator *si; setTypeIterator *si;
robj *dstset = NULL; robj *dstset = NULL;
int dstset_encoding = OBJ_ENCODING_INTSET;
char *str; char *str;
size_t len; size_t len;
int64_t llval; int64_t llval;
...@@ -1510,6 +1516,23 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum, ...@@ -1510,6 +1516,23 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum,
zfree(sets); zfree(sets);
return; return;
} }
/* For a SET's encoding, according to the factory method setTypeCreate(), currently have 3 types:
* 1. OBJ_ENCODING_INTSET
* 2. OBJ_ENCODING_LISTPACK
* 3. OBJ_ENCODING_HT
* 'dstset_encoding' is used to determine which kind of encoding to use when initialize 'dstset'.
*
* If all sets are all OBJ_ENCODING_INTSET encoding or 'dstkey' is not null, keep 'dstset'
* OBJ_ENCODING_INTSET encoding when initialize. Otherwise it is not efficient to create the 'dstset'
* from intset and then convert to listpack or hashtable.
*
* If one of the set is OBJ_ENCODING_LISTPACK, let's set 'dstset' to hashtable default encoding,
* the hashtable is more efficient when find and compare than the listpack. The corresponding
* time complexity are O(1) vs O(n). */
if (!dstkey && dstset_encoding == OBJ_ENCODING_INTSET &&
(setobj->encoding == OBJ_ENCODING_LISTPACK || setobj->encoding == OBJ_ENCODING_HT)) {
dstset_encoding = OBJ_ENCODING_HT;
}
sets[j] = setobj; sets[j] = setobj;
if (j > 0 && sets[0] == sets[j]) { if (j > 0 && sets[0] == sets[j]) {
sameset = 1; sameset = 1;
...@@ -1552,7 +1575,11 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum, ...@@ -1552,7 +1575,11 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum,
/* We need a temp set object to store our union/diff. If the dstkey /* We need a temp set object to store our union/diff. If the dstkey
* is not NULL (that is, we are inside an SUNIONSTORE/SDIFFSTORE operation) then * is not NULL (that is, we are inside an SUNIONSTORE/SDIFFSTORE operation) then
* this set object will be the resulting object to set into the target key*/ * this set object will be the resulting object to set into the target key*/
dstset = createIntsetObject(); if (dstset_encoding == OBJ_ENCODING_INTSET) {
dstset = createIntsetObject();
} else {
dstset = createSetObject();
}
if (op == SET_OP_UNION) { if (op == SET_OP_UNION) {
/* Union is trivial, just add every element of every set to the /* Union is trivial, just add every element of every set to the
......
This diff is collapsed.
...@@ -30,6 +30,7 @@ ...@@ -30,6 +30,7 @@
#include "fmacros.h" #include "fmacros.h"
#include "fpconv_dtoa.h" #include "fpconv_dtoa.h"
#include "fast_float_strtod.h"
#include <stdlib.h> #include <stdlib.h>
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
...@@ -612,7 +613,7 @@ int string2ld(const char *s, size_t slen, long double *dp) { ...@@ -612,7 +613,7 @@ int string2ld(const char *s, size_t slen, long double *dp) {
int string2d(const char *s, size_t slen, double *dp) { int string2d(const char *s, size_t slen, double *dp) {
errno = 0; errno = 0;
char *eptr; char *eptr;
*dp = strtod(s, &eptr); *dp = fast_float_strtod(s, &eptr);
if (slen == 0 || if (slen == 0 ||
isspace(((const char*)s)[0]) || isspace(((const char*)s)[0]) ||
(size_t)(eptr-(char*)s) != slen || (size_t)(eptr-(char*)s) != slen ||
......
...@@ -897,5 +897,39 @@ test {corrupt payload: fuzzer findings - set with invalid length causes smembers ...@@ -897,5 +897,39 @@ test {corrupt payload: fuzzer findings - set with invalid length causes smembers
} }
} }
test {corrupt payload: fuzzer findings - set with invalid length causes sscan to hang} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
# In the past, it generated a broken protocol and left the client hung in smembers
r config set sanitize-dump-payload no
assert_equal {OK} [r restore _set 0 "\x14\x16\x16\x00\x00\x00\x0c\x00\x81\x61\x02\x81\x62\x02\x81\x63\x02\x01\x01\x02\x01\x03\x01\xff\x0c\x00\x91\x00\x56\x73\xc1\x82\xd5\xbd" replace]
assert_encoding listpack _set
catch { r SSCAN _set 0 } err
assert_equal [count_log_message 0 "crashed by signal"] 0
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
}
}
test {corrupt payload: zset listpack encoded with invalid length causes zscan to hang} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no
assert_equal {OK} [r restore _zset 0 "\x11\x16\x16\x00\x00\x00\x1a\x00\x81\x61\x02\x01\x01\x81\x62\x02\x02\x01\x81\x63\x02\x03\x01\xff\x0c\x00\x81\xa7\xcd\x31\x22\x6c\xef\xf7" replace]
assert_encoding listpack _zset
catch { r ZSCAN _zset 0 } err
assert_equal [count_log_message 0 "crashed by signal"] 0
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
}
}
test {corrupt payload: hash listpack encoded with invalid length causes hscan to hang} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no
assert_equal {OK} [r restore _hash 0 "\x10\x17\x17\x00\x00\x00\x0e\x00\x82\x66\x31\x03\x82\x76\x31\x03\x82\x66\x32\x03\x82\x76\x32\x03\xff\x0c\x00\xf1\xc5\x36\x92\x29\x6a\x8c\xc5" replace]
assert_encoding listpack _hash
catch { r HSCAN _hash 0 } err
assert_equal [count_log_message 0 "crashed by signal"] 0
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
}
}
} ;# tags } ;# tags
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment