Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
4d4a7c73
Commit
4d4a7c73
authored
Mar 22, 2023
by
Vitaly Arbuzov
Browse files
Use binary search on top of binary index tree for fair random
parent
9269c410
Changes
4
Hide whitespace changes
Inline
Side-by-side
src/db.c
View file @
4d4a7c73
...
@@ -47,6 +47,7 @@
...
@@ -47,6 +47,7 @@
int
expireIfNeeded
(
redisDb
*
db
,
robj
*
key
,
int
flags
);
int
expireIfNeeded
(
redisDb
*
db
,
robj
*
key
,
int
flags
);
int
keyIsExpired
(
redisDb
*
db
,
robj
*
key
);
int
keyIsExpired
(
redisDb
*
db
,
robj
*
key
);
void
cumulativeKeyCountAdd
(
redisDb
*
db
,
int
idx
,
long
delta
);
/* Returns next dictionary from the iterator, or NULL if iteration is complete. */
/* Returns next dictionary from the iterator, or NULL if iteration is complete. */
dict
*
dbIteratorNextDict
(
dbIterator
*
dbit
)
{
dict
*
dbIteratorNextDict
(
dbIterator
*
dbit
)
{
...
@@ -255,6 +256,9 @@ void dbAdd(redisDb *db, robj *key, robj *val) {
...
@@ -255,6 +256,9 @@ void dbAdd(redisDb *db, robj *key, robj *val) {
db
->
non_empty_dicts
=
intsetAdd
(
db
->
non_empty_dicts
,
slot
,
&
success
);
db
->
non_empty_dicts
=
intsetAdd
(
db
->
non_empty_dicts
,
slot
,
&
success
);
serverAssert
(
success
);
serverAssert
(
success
);
}
}
if
(
server
.
cluster_enabled
)
{
cumulativeKeyCountAdd
(
db
,
slot
,
1
);
}
signalKeyAsReady
(
db
,
key
,
val
->
type
);
signalKeyAsReady
(
db
,
key
,
val
->
type
);
notifyKeyspaceEvent
(
NOTIFY_NEW
,
"new"
,
key
,
db
->
id
);
notifyKeyspaceEvent
(
NOTIFY_NEW
,
"new"
,
key
,
db
->
id
);
}
}
...
@@ -295,6 +299,9 @@ int dbAddRDBLoad(redisDb *db, sds key, robj *val) {
...
@@ -295,6 +299,9 @@ int dbAddRDBLoad(redisDb *db, sds key, robj *val) {
db
->
non_empty_dicts
=
intsetAdd
(
db
->
non_empty_dicts
,
slot
,
&
success
);
db
->
non_empty_dicts
=
intsetAdd
(
db
->
non_empty_dicts
,
slot
,
&
success
);
serverAssert
(
success
);
serverAssert
(
success
);
}
}
if
(
server
.
cluster_enabled
)
{
cumulativeKeyCountAdd
(
db
,
slot
,
1
);
}
return
1
;
return
1
;
}
}
...
@@ -390,7 +397,7 @@ robj *dbRandomKey(redisDb *db) {
...
@@ -390,7 +397,7 @@ robj *dbRandomKey(redisDb *db) {
while
(
1
)
{
while
(
1
)
{
sds
key
;
sds
key
;
robj
*
keyobj
;
robj
*
keyobj
;
dict
*
randomDict
=
getRandomDict
(
db
);
dict
*
randomDict
=
get
Fair
RandomDict
(
db
);
de
=
dictGetFairRandomKey
(
randomDict
);
de
=
dictGetFairRandomKey
(
randomDict
);
if
(
de
==
NULL
)
return
NULL
;
if
(
de
==
NULL
)
return
NULL
;
...
@@ -417,11 +424,68 @@ robj *dbRandomKey(redisDb *db) {
...
@@ -417,11 +424,68 @@ robj *dbRandomKey(redisDb *db) {
}
}
}
}
/* Return random non-empty dictionary from this DB. */
/* Updates binary index tree (also known as Fenwick tree), increasing key count for a given slot.
dict
*
getRandomDict
(
redisDb
*
db
)
{
* You can read more about this data structure here https://en.wikipedia.org/wiki/Fenwick_tree
* Time complexity is O(log(CLUSTER_SLOTS)). */
void
cumulativeKeyCountAdd
(
redisDb
*
db
,
int
slot
,
long
delta
)
{
if
(
!
server
.
cluster_enabled
)
return
;
int
idx
=
slot
+
1
;
/* Unlike slots, BIT is 1-based, so we need to add 1. */
while
(
idx
<=
CLUSTER_SLOTS
)
{
db
->
binary_index
[
idx
]
+=
delta
;
idx
+=
(
idx
&
-
idx
);
}
}
/* Returns total (cumulative) number of keys up until given slot (inclusive).
* Time complexity is O(log(CLUSTER_SLOTS)). */
unsigned
long
long
cumulativeKeyCountRead
(
redisDb
*
db
,
int
slot
)
{
if
(
!
server
.
cluster_enabled
)
{
serverAssert
(
slot
==
0
);
return
dbSize
(
db
);
}
int
idx
=
slot
+
1
;
unsigned
long
long
sum
=
0
;
while
(
idx
>
0
)
{
sum
+=
db
->
binary_index
[
idx
];
idx
-=
(
idx
&
-
idx
);
}
return
sum
;
}
/* Returns fair random dictionary, probability of each dictionary being returned is proportional to the number of elements that it holds.
* Implementation uses binary search on top of binary index tree.
* Time complexity of this function is O(log^2(CLUSTER_SLOTS)). */
dict
*
getFairRandomDict
(
redisDb
*
db
)
{
if
(
!
server
.
cluster_enabled
||
dbSize
(
db
)
==
0
)
return
db
->
dict
[
0
];
if
(
!
server
.
cluster_enabled
||
dbSize
(
db
)
==
0
)
return
db
->
dict
[
0
];
int64_t
slot
=
intsetRandom
(
db
->
non_empty_dicts
);
/* We want to find a slot containing target-th element in a key space ordered by slot id.
return
db
->
dict
[
slot
];
* Consider this example. Slots are represented by brackets and keys by dots:
* #0 #1 #2 #3 #4
* [..][....][...][.......][.]
* ^
* target
*
* In this case slot #3 contains key that we are trying to find.
* */
unsigned
long
target
=
(
randomULong
()
%
dbSize
(
db
))
+
1
;
int
lo
=
0
,
hi
=
CLUSTER_SLOTS
-
1
;
/* We use binary search to find a slot, we are allowed to do this, because we have a quick way to find a total number of keys
* up until certain slot, using binary index tree. */
while
(
lo
<
hi
)
{
int
mid
=
lo
+
(
hi
-
lo
)
/
2
;
unsigned
long
keys_up_to_mid
=
cumulativeKeyCountRead
(
db
,
mid
);
/* Total number of keys up until a given slot (inclusive). */
unsigned
long
keys_in_mid
=
dictSize
(
db
->
dict
[
mid
]);
/* Number of keys in the slot. */
if
(
target
>
keys_up_to_mid
)
{
/* Target is to the right from mid. */
lo
=
mid
+
1
;
}
else
if
(
target
<=
keys_up_to_mid
-
keys_in_mid
)
{
/* Target is to the left from mid. */
hi
=
mid
-
1
;
}
else
{
/* Located target. */
break
;
}
}
int
slot
=
lo
+
(
hi
-
lo
)
/
2
;
dict
*
result
=
db
->
dict
[
slot
];
serverAssert
(
!
dictIsEmpty
(
result
));
return
result
;
}
}
/* Helper for sync and async delete. */
/* Helper for sync and async delete. */
...
@@ -458,6 +522,9 @@ int dbGenericDelete(redisDb *db, robj *key, int async, int flags) {
...
@@ -458,6 +522,9 @@ int dbGenericDelete(redisDb *db, robj *key, int async, int flags) {
db
->
non_empty_dicts
=
intsetRemove
(
db
->
non_empty_dicts
,
slot
,
&
success
);
db
->
non_empty_dicts
=
intsetRemove
(
db
->
non_empty_dicts
,
slot
,
&
success
);
serverAssert
(
success
);
serverAssert
(
success
);
}
}
if
(
server
.
cluster_enabled
)
{
cumulativeKeyCountAdd
(
db
,
slot
,
-
1
);
}
db
->
key_count
--
;
db
->
key_count
--
;
return
1
;
return
1
;
}
else
{
}
else
{
...
...
src/evict.c
View file @
4d4a7c73
...
@@ -600,7 +600,7 @@ int performEvictions(void) {
...
@@ -600,7 +600,7 @@ int performEvictions(void) {
db
=
server
.
db
+
i
;
db
=
server
.
db
+
i
;
do
{
do
{
dict
=
(
server
.
maxmemory_policy
&
MAXMEMORY_FLAG_ALLKEYS
)
?
dict
=
(
server
.
maxmemory_policy
&
MAXMEMORY_FLAG_ALLKEYS
)
?
getRandomDict
(
db
)
:
db
->
expires
;
get
Fair
RandomDict
(
db
)
:
db
->
expires
;
if
((
keys
=
dictSize
(
dict
))
!=
0
)
{
if
((
keys
=
dictSize
(
dict
))
!=
0
)
{
evictionPoolPopulate
(
i
,
dict
,
db
,
pool
);
evictionPoolPopulate
(
i
,
dict
,
db
,
pool
);
total_keys
+=
keys
;
total_keys
+=
keys
;
...
@@ -655,7 +655,7 @@ int performEvictions(void) {
...
@@ -655,7 +655,7 @@ int performEvictions(void) {
j
=
(
++
next_db
)
%
server
.
dbnum
;
j
=
(
++
next_db
)
%
server
.
dbnum
;
db
=
server
.
db
+
j
;
db
=
server
.
db
+
j
;
dict
=
(
server
.
maxmemory_policy
==
MAXMEMORY_ALLKEYS_RANDOM
)
?
dict
=
(
server
.
maxmemory_policy
==
MAXMEMORY_ALLKEYS_RANDOM
)
?
getRandomDict
(
db
)
:
db
->
expires
;
get
Fair
RandomDict
(
db
)
:
db
->
expires
;
if
(
dictSize
(
dict
)
!=
0
)
{
if
(
dictSize
(
dict
)
!=
0
)
{
de
=
dictGetRandomKey
(
dict
);
de
=
dictGetRandomKey
(
dict
);
bestkey
=
dictGetKey
(
de
);
bestkey
=
dictGetKey
(
de
);
...
...
src/server.c
View file @
4d4a7c73
...
@@ -2634,6 +2634,7 @@ void initServer(void) {
...
@@ -2634,6 +2634,7 @@ void initServer(void) {
server
.
db
[
j
].
dict_count
=
slotCount
;
server
.
db
[
j
].
dict_count
=
slotCount
;
server
.
db
[
j
].
key_count
=
0
;
server
.
db
[
j
].
key_count
=
0
;
server
.
db
[
j
].
non_empty_dicts
=
server
.
cluster_enabled
?
intsetNew
()
:
NULL
;
server
.
db
[
j
].
non_empty_dicts
=
server
.
cluster_enabled
?
intsetNew
()
:
NULL
;
server
.
db
[
j
].
binary_index
=
server
.
cluster_enabled
?
zmalloc
(
sizeof
(
unsigned
long
long
)
*
(
CLUSTER_SLOTS
+
1
))
:
NULL
;
listSetFreeMethod
(
server
.
db
[
j
].
defrag_later
,(
void
(
*
)(
void
*
))
sdsfree
);
listSetFreeMethod
(
server
.
db
[
j
].
defrag_later
,(
void
(
*
)(
void
*
))
sdsfree
);
}
}
evictionPoolAlloc
();
/* Initialize the LRU keys pool. */
evictionPoolAlloc
();
/* Initialize the LRU keys pool. */
...
...
src/server.h
View file @
4d4a7c73
...
@@ -976,6 +976,7 @@ typedef struct redisDb {
...
@@ -976,6 +976,7 @@ typedef struct redisDb {
int
dict_count
;
/* Indicates total number of dictionaires owned by this DB, 1 dict per slot in cluster mode. */
int
dict_count
;
/* Indicates total number of dictionaires owned by this DB, 1 dict per slot in cluster mode. */
unsigned
long
long
key_count
;
/* Total number of keys in this DB. */
unsigned
long
long
key_count
;
/* Total number of keys in this DB. */
intset
*
non_empty_dicts
;
/* Set of non-empty dictionaries. */
intset
*
non_empty_dicts
;
/* Set of non-empty dictionaries. */
unsigned
long
long
*
binary_index
;
/* Binary indexed tree (BIT) that describes cumulative key frequencies up until given slot. */
}
redisDb
;
}
redisDb
;
/* forward declaration for functions ctx */
/* forward declaration for functions ctx */
...
@@ -3126,9 +3127,10 @@ void dismissMemoryInChild(void);
...
@@ -3126,9 +3127,10 @@ void dismissMemoryInChild(void);
int
restartServer
(
int
flags
,
mstime_t
delay
);
int
restartServer
(
int
flags
,
mstime_t
delay
);
unsigned
long
long
int
dbSize
(
redisDb
*
db
);
unsigned
long
long
int
dbSize
(
redisDb
*
db
);
int
getKeySlot
(
sds
key
);
int
getKeySlot
(
sds
key
);
dict
*
getRandomDict
(
redisDb
*
db
);
unsigned
long
dbSlots
(
redisDb
*
db
);
unsigned
long
dbSlots
(
redisDb
*
db
);
void
expandDb
(
const
redisDb
*
db
,
uint64_t
db_size
);
void
expandDb
(
const
redisDb
*
db
,
uint64_t
db_size
);
unsigned
long
long
cumulativeKeyCountRead
(
redisDb
*
db
,
int
idx
);
dict
*
getFairRandomDict
(
redisDb
*
db
);
/* Set data type */
/* Set data type */
robj
*
setTypeCreate
(
sds
value
);
robj
*
setTypeCreate
(
sds
value
);
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment