Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
80458d04
Commit
80458d04
authored
Sep 26, 2024
by
YaacovHazan
Browse files
Merge remote-tracking branch 'upstream/unstable' into HEAD
parents
5fe3e74a
99c40ab5
Changes
34
Expand all
Show whitespace changes
Inline
Side-by-side
src/redismodule.h
View file @
80458d04
...
@@ -60,10 +60,13 @@ typedef long long ustime_t;
...
@@ -60,10 +60,13 @@ typedef long long ustime_t;
#define REDISMODULE_OPEN_KEY_NOEXPIRE (1<<19)
#define REDISMODULE_OPEN_KEY_NOEXPIRE (1<<19)
/* Avoid any effects from fetching the key */
/* Avoid any effects from fetching the key */
#define REDISMODULE_OPEN_KEY_NOEFFECTS (1<<20)
#define REDISMODULE_OPEN_KEY_NOEFFECTS (1<<20)
/* Allow access expired key that haven't deleted yet */
#define REDISMODULE_OPEN_KEY_ACCESS_EXPIRED (1<<21)
/* Mask of all REDISMODULE_OPEN_KEY_* values. Any new mode should be added to this list.
/* Mask of all REDISMODULE_OPEN_KEY_* values. Any new mode should be added to this list.
* Should not be used directly by the module, use RM_GetOpenKeyModesAll instead.
* Should not be used directly by the module, use RM_GetOpenKeyModesAll instead.
* Located here so when we will add new modes we will not forget to update it. */
* Located here so when we will add new modes we will not forget to update it. */
#define _REDISMODULE_OPEN_KEY_ALL REDISMODULE_READ | REDISMODULE_WRITE | REDISMODULE_OPEN_KEY_NOTOUCH | REDISMODULE_OPEN_KEY_NONOTIFY | REDISMODULE_OPEN_KEY_NOSTATS | REDISMODULE_OPEN_KEY_NOEXPIRE | REDISMODULE_OPEN_KEY_NOEFFECTS
#define _REDISMODULE_OPEN_KEY_ALL REDISMODULE_READ | REDISMODULE_WRITE | REDISMODULE_OPEN_KEY_NOTOUCH | REDISMODULE_OPEN_KEY_NONOTIFY | REDISMODULE_OPEN_KEY_NOSTATS | REDISMODULE_OPEN_KEY_NOEXPIRE | REDISMODULE_OPEN_KEY_NOEFFECTS
| REDISMODULE_OPEN_KEY_ACCESS_EXPIRED
/* List push and pop */
/* List push and pop */
#define REDISMODULE_LIST_HEAD 0
#define REDISMODULE_LIST_HEAD 0
...
...
src/resp_parser.c
View file @
80458d04
...
@@ -34,6 +34,7 @@
...
@@ -34,6 +34,7 @@
* ----------------------------------------------------------------------------------------
* ----------------------------------------------------------------------------------------
*/
*/
#include "fast_float_strtod.h"
#include "resp_parser.h"
#include "resp_parser.h"
#include "server.h"
#include "server.h"
...
@@ -132,7 +133,7 @@ static int parseDouble(ReplyParser *parser, void *p_ctx) {
...
@@ -132,7 +133,7 @@ static int parseDouble(ReplyParser *parser, void *p_ctx) {
if
(
len
<=
MAX_LONG_DOUBLE_CHARS
)
{
if
(
len
<=
MAX_LONG_DOUBLE_CHARS
)
{
memcpy
(
buf
,
proto
+
1
,
len
);
memcpy
(
buf
,
proto
+
1
,
len
);
buf
[
len
]
=
'\0'
;
buf
[
len
]
=
'\0'
;
d
=
strtod
(
buf
,
NULL
);
/* We expect a valid representation. */
d
=
fast_float_
strtod
(
buf
,
NULL
);
/* We expect a valid representation. */
}
else
{
}
else
{
d
=
0
;
d
=
0
;
}
}
...
...
src/server.h
View file @
80458d04
...
@@ -3221,6 +3221,7 @@ typedef struct dictExpireMetadata {
...
@@ -3221,6 +3221,7 @@ typedef struct dictExpireMetadata {
#define HFE_LAZY_AVOID_HASH_DEL (1<<1)
/* Avoid deleting hash if the field is the last one */
#define HFE_LAZY_AVOID_HASH_DEL (1<<1)
/* Avoid deleting hash if the field is the last one */
#define HFE_LAZY_NO_NOTIFICATION (1<<2)
/* Do not send notification, used when multiple fields
#define HFE_LAZY_NO_NOTIFICATION (1<<2)
/* Do not send notification, used when multiple fields
* may expire and only one notification is desired. */
* may expire and only one notification is desired. */
#define HFE_LAZY_ACCESS_EXPIRED (1<<3)
/* Avoid lazy expire and allow access to expired fields */
void
hashTypeConvert
(
robj
*
o
,
int
enc
,
ebuckets
*
hexpires
);
void
hashTypeConvert
(
robj
*
o
,
int
enc
,
ebuckets
*
hexpires
);
void
hashTypeTryConversion
(
redisDb
*
db
,
robj
*
subject
,
robj
**
argv
,
int
start
,
int
end
);
void
hashTypeTryConversion
(
redisDb
*
db
,
robj
*
subject
,
robj
**
argv
,
int
start
,
int
end
);
...
@@ -3388,6 +3389,7 @@ int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
...
@@ -3388,6 +3389,7 @@ int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
#define LOOKUP_NOSTATS (1<<2)
/* Don't update keyspace hits/misses counters. */
#define LOOKUP_NOSTATS (1<<2)
/* Don't update keyspace hits/misses counters. */
#define LOOKUP_WRITE (1<<3)
/* Delete expired keys even in replicas. */
#define LOOKUP_WRITE (1<<3)
/* Delete expired keys even in replicas. */
#define LOOKUP_NOEXPIRE (1<<4)
/* Avoid deleting lazy expired keys. */
#define LOOKUP_NOEXPIRE (1<<4)
/* Avoid deleting lazy expired keys. */
#define LOOKUP_ACCESS_EXPIRED (1<<5)
/* Allow lookup to expired key. */
#define LOOKUP_NOEFFECTS (LOOKUP_NONOTIFY | LOOKUP_NOSTATS | LOOKUP_NOTOUCH | LOOKUP_NOEXPIRE)
/* Avoid any effects from fetching the key */
#define LOOKUP_NOEFFECTS (LOOKUP_NONOTIFY | LOOKUP_NOSTATS | LOOKUP_NOTOUCH | LOOKUP_NOEXPIRE)
/* Avoid any effects from fetching the key */
dictEntry
*
dbAdd
(
redisDb
*
db
,
robj
*
key
,
robj
*
val
);
dictEntry
*
dbAdd
(
redisDb
*
db
,
robj
*
key
,
robj
*
val
);
...
...
src/sort.c
View file @
80458d04
...
@@ -7,7 +7,7 @@
...
@@ -7,7 +7,7 @@
* (RSALv2) or the Server Side Public License v1 (SSPLv1).
* (RSALv2) or the Server Side Public License v1 (SSPLv1).
*/
*/
#include "fast_float_strtod.h"
#include "server.h"
#include "server.h"
#include "pqsort.h"
/* Partial qsort for SORT+LIMIT */
#include "pqsort.h"
/* Partial qsort for SORT+LIMIT */
#include <math.h>
/* isnan() */
#include <math.h>
/* isnan() */
...
@@ -472,7 +472,7 @@ void sortCommandGeneric(client *c, int readonly) {
...
@@ -472,7 +472,7 @@ void sortCommandGeneric(client *c, int readonly) {
if
(
sdsEncodedObject
(
byval
))
{
if
(
sdsEncodedObject
(
byval
))
{
char
*
eptr
;
char
*
eptr
;
vector
[
j
].
u
.
score
=
strtod
(
byval
->
ptr
,
&
eptr
);
vector
[
j
].
u
.
score
=
fast_float_
strtod
(
byval
->
ptr
,
&
eptr
);
if
(
eptr
[
0
]
!=
'\0'
||
errno
==
ERANGE
||
if
(
eptr
[
0
]
!=
'\0'
||
errno
==
ERANGE
||
isnan
(
vector
[
j
].
u
.
score
))
isnan
(
vector
[
j
].
u
.
score
))
{
{
...
...
src/t_hash.c
View file @
80458d04
...
@@ -734,7 +734,7 @@ GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vs
...
@@ -734,7 +734,7 @@ GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vs
serverPanic
(
"Unknown hash encoding"
);
serverPanic
(
"Unknown hash encoding"
);
}
}
if
(
expiredAt
>=
(
uint64_t
)
commandTimeSnapshot
())
if
(
(
expiredAt
>=
(
uint64_t
)
commandTimeSnapshot
())
||
(
hfeFlags
&
HFE_LAZY_ACCESS_EXPIRED
))
return
GETF_OK
;
return
GETF_OK
;
if
(
server
.
masterhost
)
{
if
(
server
.
masterhost
)
{
...
@@ -1134,6 +1134,20 @@ int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cm
...
@@ -1134,6 +1134,20 @@ int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cm
dictEntry
*
de
=
dbFind
(
c
->
db
,
key
->
ptr
);
dictEntry
*
de
=
dbFind
(
c
->
db
,
key
->
ptr
);
serverAssert
(
de
!=
NULL
);
serverAssert
(
de
!=
NULL
);
lpt
->
key
=
dictGetKey
(
de
);
lpt
->
key
=
dictGetKey
(
de
);
}
else
if
(
o
->
encoding
==
OBJ_ENCODING_LISTPACK_EX
)
{
listpackEx
*
lpt
=
o
->
ptr
;
/* If the hash previously had HFEs but later no longer does, the key ref
* (lpt->key) in the hash might become outdated after a MOVE/COPY/RENAME/RESTORE
* operation. These commands maintain the key ref only if HFEs are present.
* That is, we can only be sure that key ref is valid as long as it is not
* "trash". (TODO: dbFind() can be avoided. Instead need to extend the
* lookupKey*() to return dictEntry). */
if
(
lpt
->
meta
.
trash
)
{
dictEntry
*
de
=
dbFind
(
c
->
db
,
key
->
ptr
);
serverAssert
(
de
!=
NULL
);
lpt
->
key
=
dictGetKey
(
de
);
}
}
else
if
(
o
->
encoding
==
OBJ_ENCODING_HT
)
{
}
else
if
(
o
->
encoding
==
OBJ_ENCODING_HT
)
{
/* Take care dict has HFE metadata */
/* Take care dict has HFE metadata */
if
(
!
isDictWithMetaHFE
(
ht
))
{
if
(
!
isDictWithMetaHFE
(
ht
))
{
...
@@ -1151,6 +1165,18 @@ int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cm
...
@@ -1151,6 +1165,18 @@ int hashTypeSetExInit(robj *key, robj *o, client *c, redisDb *db, const char *cm
m
->
key
=
dictGetKey
(
de
);
/* reference key in keyspace */
m
->
key
=
dictGetKey
(
de
);
/* reference key in keyspace */
m
->
hfe
=
ebCreate
();
/* Allocate HFE DS */
m
->
hfe
=
ebCreate
();
/* Allocate HFE DS */
m
->
expireMeta
.
trash
=
1
;
/* mark as trash (as long it wasn't ebAdd()) */
m
->
expireMeta
.
trash
=
1
;
/* mark as trash (as long it wasn't ebAdd()) */
}
else
{
dictExpireMetadata
*
m
=
(
dictExpireMetadata
*
)
dictMetadata
(
ht
);
/* If the hash previously had HFEs but later no longer does, the key ref
* (m->key) in the hash might become outdated after a MOVE/COPY/RENAME/RESTORE
* operation. These commands maintain the key ref only if HFEs are present.
* That is, we can only be sure that key ref is valid as long as it is not
* "trash". */
if
(
m
->
expireMeta
.
trash
)
{
dictEntry
*
de
=
dbFind
(
db
,
key
->
ptr
);
serverAssert
(
de
!=
NULL
);
m
->
key
=
dictGetKey
(
de
);
/* reference key in keyspace */
}
}
}
}
}
...
...
src/t_set.c
View file @
80458d04
...
@@ -2,8 +2,13 @@
...
@@ -2,8 +2,13 @@
* Copyright (c) 2009-Present, Redis Ltd.
* Copyright (c) 2009-Present, Redis Ltd.
* All rights reserved.
* All rights reserved.
*
*
* Copyright (c) 2024-present, Valkey contributors.
* All rights reserved.
*
* Licensed under your choice of the Redis Source Available License 2.0
* Licensed under your choice of the Redis Source Available License 2.0
* (RSALv2) or the Server Side Public License v1 (SSPLv1).
* (RSALv2) or the Server Side Public License v1 (SSPLv1).
*
* Portions of this file are available under BSD3 terms; see REDISCONTRIBUTIONS for more information.
*/
*/
#include "server.h"
#include "server.h"
...
@@ -1492,6 +1497,7 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum,
...
@@ -1492,6 +1497,7 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum,
robj
**
sets
=
zmalloc
(
sizeof
(
robj
*
)
*
setnum
);
robj
**
sets
=
zmalloc
(
sizeof
(
robj
*
)
*
setnum
);
setTypeIterator
*
si
;
setTypeIterator
*
si
;
robj
*
dstset
=
NULL
;
robj
*
dstset
=
NULL
;
int
dstset_encoding
=
OBJ_ENCODING_INTSET
;
char
*
str
;
char
*
str
;
size_t
len
;
size_t
len
;
int64_t
llval
;
int64_t
llval
;
...
@@ -1510,6 +1516,23 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum,
...
@@ -1510,6 +1516,23 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum,
zfree
(
sets
);
zfree
(
sets
);
return
;
return
;
}
}
/* For a SET's encoding, according to the factory method setTypeCreate(), currently have 3 types:
* 1. OBJ_ENCODING_INTSET
* 2. OBJ_ENCODING_LISTPACK
* 3. OBJ_ENCODING_HT
* 'dstset_encoding' is used to determine which kind of encoding to use when initialize 'dstset'.
*
* If all sets are all OBJ_ENCODING_INTSET encoding or 'dstkey' is not null, keep 'dstset'
* OBJ_ENCODING_INTSET encoding when initialize. Otherwise it is not efficient to create the 'dstset'
* from intset and then convert to listpack or hashtable.
*
* If one of the set is OBJ_ENCODING_LISTPACK, let's set 'dstset' to hashtable default encoding,
* the hashtable is more efficient when find and compare than the listpack. The corresponding
* time complexity are O(1) vs O(n). */
if
(
!
dstkey
&&
dstset_encoding
==
OBJ_ENCODING_INTSET
&&
(
setobj
->
encoding
==
OBJ_ENCODING_LISTPACK
||
setobj
->
encoding
==
OBJ_ENCODING_HT
))
{
dstset_encoding
=
OBJ_ENCODING_HT
;
}
sets
[
j
]
=
setobj
;
sets
[
j
]
=
setobj
;
if
(
j
>
0
&&
sets
[
0
]
==
sets
[
j
])
{
if
(
j
>
0
&&
sets
[
0
]
==
sets
[
j
])
{
sameset
=
1
;
sameset
=
1
;
...
@@ -1552,7 +1575,11 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum,
...
@@ -1552,7 +1575,11 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum,
/* We need a temp set object to store our union/diff. If the dstkey
/* We need a temp set object to store our union/diff. If the dstkey
* is not NULL (that is, we are inside an SUNIONSTORE/SDIFFSTORE operation) then
* is not NULL (that is, we are inside an SUNIONSTORE/SDIFFSTORE operation) then
* this set object will be the resulting object to set into the target key*/
* this set object will be the resulting object to set into the target key*/
if
(
dstset_encoding
==
OBJ_ENCODING_INTSET
)
{
dstset
=
createIntsetObject
();
dstset
=
createIntsetObject
();
}
else
{
dstset
=
createSetObject
();
}
if
(
op
==
SET_OP_UNION
)
{
if
(
op
==
SET_OP_UNION
)
{
/* Union is trivial, just add every element of every set to the
/* Union is trivial, just add every element of every set to the
...
...
src/t_zset.c
View file @
80458d04
This diff is collapsed.
Click to expand it.
src/util.c
View file @
80458d04
...
@@ -30,6 +30,7 @@
...
@@ -30,6 +30,7 @@
#include "fmacros.h"
#include "fmacros.h"
#include "fpconv_dtoa.h"
#include "fpconv_dtoa.h"
#include "fast_float_strtod.h"
#include <stdlib.h>
#include <stdlib.h>
#include <stdio.h>
#include <stdio.h>
#include <string.h>
#include <string.h>
...
@@ -612,7 +613,7 @@ int string2ld(const char *s, size_t slen, long double *dp) {
...
@@ -612,7 +613,7 @@ int string2ld(const char *s, size_t slen, long double *dp) {
int
string2d
(
const
char
*
s
,
size_t
slen
,
double
*
dp
)
{
int
string2d
(
const
char
*
s
,
size_t
slen
,
double
*
dp
)
{
errno
=
0
;
errno
=
0
;
char
*
eptr
;
char
*
eptr
;
*
dp
=
strtod
(
s
,
&
eptr
);
*
dp
=
fast_float_
strtod
(
s
,
&
eptr
);
if
(
slen
==
0
||
if
(
slen
==
0
||
isspace
(((
const
char
*
)
s
)[
0
])
||
isspace
(((
const
char
*
)
s
)[
0
])
||
(
size_t
)(
eptr
-
(
char
*
)
s
)
!=
slen
||
(
size_t
)(
eptr
-
(
char
*
)
s
)
!=
slen
||
...
...
tests/integration/corrupt-dump.tcl
View file @
80458d04
...
@@ -897,5 +897,39 @@ test {corrupt payload: fuzzer findings - set with invalid length causes smembers
...
@@ -897,5 +897,39 @@ test {corrupt payload: fuzzer findings - set with invalid length causes smembers
}
}
}
}
test
{
corrupt payload: fuzzer findings - set with invalid length causes sscan to hang
}
{
start_server
[
list overrides
[
list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no
]
]
{
# In the past, it generated a broken protocol and left the client hung in smembers
r config set sanitize-dump-payload no
assert_equal
{
OK
}
[
r restore _set 0
"
\x14\x16\x16\x00\x00\x00\x0c\x00\x81\x61\x02\x81\x62\x02\x81\x63\x02\x01\x01\x02\x01\x03\x01\xff\x0c\x00\x91\x00\x56\x73\xc1\x82\xd5\xbd
"
replace
]
assert_encoding listpack _set
catch
{
r SSCAN _set 0
}
err
assert_equal
[
count_log_message 0
"crashed by signal"
]
0
assert_equal
[
count_log_message 0
"ASSERTION FAILED"
]
1
}
}
test
{
corrupt payload: zset listpack encoded with invalid length causes zscan to hang
}
{
start_server
[
list overrides
[
list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no
]
]
{
r config set sanitize-dump-payload no
assert_equal
{
OK
}
[
r restore _zset 0
"
\x11\x16\x16\x00\x00\x00\x1a\x00\x81\x61\x02\x01\x01\x81\x62\x02\x02\x01\x81\x63\x02\x03\x01\xff\x0c\x00\x81\xa7\xcd\x31\x22\x6c\xef\xf7
"
replace
]
assert_encoding listpack _zset
catch
{
r ZSCAN _zset 0
}
err
assert_equal
[
count_log_message 0
"crashed by signal"
]
0
assert_equal
[
count_log_message 0
"ASSERTION FAILED"
]
1
}
}
test
{
corrupt payload: hash listpack encoded with invalid length causes hscan to hang
}
{
start_server
[
list overrides
[
list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no
]
]
{
r config set sanitize-dump-payload no
assert_equal
{
OK
}
[
r restore _hash 0
"
\x10\x17\x17\x00\x00\x00\x0e\x00\x82\x66\x31\x03\x82\x76\x31\x03\x82\x66\x32\x03\x82\x76\x32\x03\xff\x0c\x00\xf1\xc5\x36\x92\x29\x6a\x8c\xc5
"
replace
]
assert_encoding listpack _hash
catch
{
r HSCAN _hash 0
}
err
assert_equal
[
count_log_message 0
"crashed by signal"
]
0
assert_equal
[
count_log_message 0
"ASSERTION FAILED"
]
1
}
}
}
;
# tags
}
;
# tags
tests/integration/redis-cli.tcl
View file @
80458d04
...
@@ -808,6 +808,12 @@ if {!$::tls} { ;# fake_redis_node doesn't support TLS
...
@@ -808,6 +808,12 @@ if {!$::tls} { ;# fake_redis_node doesn't support TLS
assert_equal 3
[
exec
{*}
$cmdline
ZCARD new_zset
]
assert_equal 3
[
exec
{*}
$cmdline
ZCARD new_zset
]
assert_equal
"a
\n
1
\n
b
\n
2
\n
c
\n
3"
[
exec
{*}
$cmdline
ZRANGE new_zset 0 -1 WITHSCORES
]
assert_equal
"a
\n
1
\n
b
\n
2
\n
c
\n
3"
[
exec
{*}
$cmdline
ZRANGE new_zset 0 -1 WITHSCORES
]
}
}
test
"Send eval command by using --eval option"
{
set tmpfile
[
write_tmpfile
{
return ARGV
[
1
]}]
set cmdline
[
rediscli
[
srv host
]
[
srv port
]]
assert_equal
"foo"
[
exec
{*}
$cmdline
--eval $tmpfile , foo
]
}
}
}
start_server
{
tags
{
"cli external:skip"
}}
{
start_server
{
tags
{
"cli external:skip"
}}
{
...
...
tests/modules/hash.c
View file @
80458d04
This diff is collapsed.
Click to expand it.
tests/unit/introspection-2.tcl
View file @
80458d04
This diff is collapsed.
Click to expand it.
tests/unit/moduleapi/hash.tcl
View file @
80458d04
This diff is collapsed.
Click to expand it.
tests/unit/type/hash-field-expire.tcl
View file @
80458d04
This diff is collapsed.
Click to expand it.
Prev
1
2
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment