Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
8504bf18
Commit
8504bf18
authored
Sep 03, 2016
by
antirez
Browse files
dict.c clustered buckets.
parent
560e6787
Changes
6
Show whitespace changes
Inline
Side-by-side
src/debug.c
View file @
8504bf18
...
@@ -434,7 +434,7 @@ void debugCommand(client *c) {
...
@@ -434,7 +434,7 @@ void debugCommand(client *c) {
if
(
getLongFromObjectOrReply
(
c
,
c
->
argv
[
2
],
&
keys
,
NULL
)
!=
C_OK
)
if
(
getLongFromObjectOrReply
(
c
,
c
->
argv
[
2
],
&
keys
,
NULL
)
!=
C_OK
)
return
;
return
;
dictExpand
(
c
->
db
->
dict
,
keys
);
dictExpand
ToOptimalSize
(
c
->
db
->
dict
,
keys
);
for
(
j
=
0
;
j
<
keys
;
j
++
)
{
for
(
j
=
0
;
j
<
keys
;
j
++
)
{
snprintf
(
buf
,
sizeof
(
buf
),
"%s:%lu"
,
snprintf
(
buf
,
sizeof
(
buf
),
"%s:%lu"
,
(
c
->
argc
==
3
)
?
"key"
:
(
char
*
)
c
->
argv
[
3
]
->
ptr
,
j
);
(
c
->
argc
==
3
)
?
"key"
:
(
char
*
)
c
->
argv
[
3
]
->
ptr
,
j
);
...
...
src/dict.c
View file @
8504bf18
...
@@ -60,7 +60,8 @@
...
@@ -60,7 +60,8 @@
* prevented: a hash table is still allowed to grow if the ratio between
* prevented: a hash table is still allowed to grow if the ratio between
* the number of elements and the buckets > dict_force_resize_ratio. */
* the number of elements and the buckets > dict_force_resize_ratio. */
static
int
dict_can_resize
=
1
;
static
int
dict_can_resize
=
1
;
static
unsigned
int
dict_force_resize_ratio
=
5
;
static
unsigned
int
dict_resize_ratio
=
20
;
static
unsigned
int
dict_force_resize_ratio
=
60
;
/* -------------------------- private prototypes ---------------------------- */
/* -------------------------- private prototypes ---------------------------- */
...
@@ -71,18 +72,6 @@ static int _dictInit(dict *ht, dictType *type, void *privDataPtr);
...
@@ -71,18 +72,6 @@ static int _dictInit(dict *ht, dictType *type, void *privDataPtr);
/* -------------------------- hash functions -------------------------------- */
/* -------------------------- hash functions -------------------------------- */
/* Thomas Wang's 32 bit Mix Function */
unsigned
int
dictIntHashFunction
(
unsigned
int
key
)
{
key
+=
~
(
key
<<
15
);
key
^=
(
key
>>
10
);
key
+=
(
key
<<
3
);
key
^=
(
key
>>
6
);
key
+=
~
(
key
<<
11
);
key
^=
(
key
>>
16
);
return
key
;
}
static
uint32_t
dict_hash_function_seed
=
5381
;
static
uint32_t
dict_hash_function_seed
=
5381
;
void
dictSetHashFunctionSeed
(
uint32_t
seed
)
{
void
dictSetHashFunctionSeed
(
uint32_t
seed
)
{
...
@@ -158,6 +147,35 @@ unsigned int dictGenCaseHashFunction(const unsigned char *buf, int len) {
...
@@ -158,6 +147,35 @@ unsigned int dictGenCaseHashFunction(const unsigned char *buf, int len) {
/* ----------------------------- API implementation ------------------------- */
/* ----------------------------- API implementation ------------------------- */
dictEntry
*
dictPushEntry
(
dictEntryVector
**
table
,
unsigned
int
h
,
const
void
*
key
,
const
void
*
val
)
{
dictEntryVector
*
dv
=
table
[
h
];
dictEntry
*
he
;
if
(
dv
==
NULL
)
{
dv
=
table
[
h
]
=
zmalloc
(
sizeof
(
dictEntryVector
)
+
sizeof
(
dictEntry
));
dv
->
used
=
1
;
dv
->
free
=
0
;
he
=
dv
->
entry
;
}
else
if
(
dv
->
free
==
0
)
{
dv
=
table
[
h
]
=
zrealloc
(
table
[
h
],
sizeof
(
dictEntryVector
)
+
sizeof
(
dictEntry
)
*
(
dv
->
used
+
1
));
he
=
dv
->
entry
+
dv
->
used
;
dv
->
used
++
;
}
else
{
uint32_t
entries
=
dv
->
used
+
dv
->
free
;
uint32_t
j
;
for
(
j
=
0
;
j
<
entries
;
j
++
)
{
if
(
dv
->
entry
[
j
].
key
==
NULL
)
break
;
}
he
=
dv
->
entry
+
j
;
dv
->
used
++
;
dv
->
free
--
;
}
he
->
key
=
(
void
*
)
key
;
he
->
v
.
val
=
(
void
*
)
val
;
return
he
;
}
/* Reset a hash table already initialized with ht_init().
/* Reset a hash table already initialized with ht_init().
* NOTE: This function should only be called by ht_destroy(). */
* NOTE: This function should only be called by ht_destroy(). */
static
void
_dictReset
(
dictht
*
ht
)
static
void
_dictReset
(
dictht
*
ht
)
...
@@ -212,8 +230,7 @@ int dictExpand(dict *d, unsigned long size)
...
@@ -212,8 +230,7 @@ int dictExpand(dict *d, unsigned long size)
/* the size is invalid if it is smaller than the number of
/* the size is invalid if it is smaller than the number of
* elements already inside the hash table */
* elements already inside the hash table */
if
(
dictIsRehashing
(
d
)
||
d
->
ht
[
0
].
used
>
size
)
if
(
dictIsRehashing
(
d
))
return
DICT_ERR
;
return
DICT_ERR
;
/* Rehashing to the same table size is not useful. */
/* Rehashing to the same table size is not useful. */
if
(
realsize
==
d
->
ht
[
0
].
size
)
return
DICT_ERR
;
if
(
realsize
==
d
->
ht
[
0
].
size
)
return
DICT_ERR
;
...
@@ -221,7 +238,7 @@ int dictExpand(dict *d, unsigned long size)
...
@@ -221,7 +238,7 @@ int dictExpand(dict *d, unsigned long size)
/* Allocate the new hash table and initialize all pointers to NULL */
/* Allocate the new hash table and initialize all pointers to NULL */
n
.
size
=
realsize
;
n
.
size
=
realsize
;
n
.
sizemask
=
realsize
-
1
;
n
.
sizemask
=
realsize
-
1
;
n
.
table
=
zcalloc
(
realsize
*
sizeof
(
dictEntry
*
));
n
.
table
=
zcalloc
(
realsize
*
sizeof
(
dictEntry
Vector
*
));
n
.
used
=
0
;
n
.
used
=
0
;
/* Is this the first initialization? If so it's not really a rehashing
/* Is this the first initialization? If so it's not really a rehashing
...
@@ -237,6 +254,12 @@ int dictExpand(dict *d, unsigned long size)
...
@@ -237,6 +254,12 @@ int dictExpand(dict *d, unsigned long size)
return
DICT_OK
;
return
DICT_OK
;
}
}
/* Expand the dictionary to the optimal number of elements needed to
* hold 'entries' keys. */
int
dictExpandToOptimalSize
(
dict
*
d
,
unsigned
long
entries
)
{
return
dictExpand
(
d
,
entries
/
(
dict_resize_ratio
/
2
));
}
/* Performs N steps of incremental rehashing. Returns 1 if there are still
/* Performs N steps of incremental rehashing. Returns 1 if there are still
* keys to move from the old to the new hash table, otherwise 0 is returned.
* keys to move from the old to the new hash table, otherwise 0 is returned.
*
*
...
@@ -251,7 +274,7 @@ int dictRehash(dict *d, int n) {
...
@@ -251,7 +274,7 @@ int dictRehash(dict *d, int n) {
if
(
!
dictIsRehashing
(
d
))
return
0
;
if
(
!
dictIsRehashing
(
d
))
return
0
;
while
(
n
--
&&
d
->
ht
[
0
].
used
!=
0
)
{
while
(
n
--
&&
d
->
ht
[
0
].
used
!=
0
)
{
dictEntry
*
de
,
*
nextde
;
dictEntry
Vector
*
dv
;
/* Note that rehashidx can't overflow as we are sure there are more
/* Note that rehashidx can't overflow as we are sure there are more
* elements because ht[0].used != 0 */
* elements because ht[0].used != 0 */
...
@@ -260,20 +283,26 @@ int dictRehash(dict *d, int n) {
...
@@ -260,20 +283,26 @@ int dictRehash(dict *d, int n) {
d
->
rehashidx
++
;
d
->
rehashidx
++
;
if
(
--
empty_visits
==
0
)
return
1
;
if
(
--
empty_visits
==
0
)
return
1
;
}
}
de
=
d
->
ht
[
0
].
table
[
d
->
rehashidx
];
dv
=
d
->
ht
[
0
].
table
[
d
->
rehashidx
];
/* Move all the keys in this bucket from the old to the new hash HT */
/* Move all the keys in this bucket from the old to the new hash HT.
while
(
de
)
{
* TODO: if the table we are rehasing to is smaller than the current
* table, we can just move the whole entries vector from one table
* to the next, since all the entries of this table will hash into
* the same slot of the target table. */
uint32_t
entries
=
dv
?
(
dv
->
used
+
dv
->
free
)
:
0
;
uint32_t
j
;
for
(
j
=
0
;
j
<
entries
;
j
++
)
{
unsigned
int
h
;
unsigned
int
h
;
nextde
=
de
->
next
;
/* Get the index in the new hash table */
/* Get the index in the new hash table */
h
=
dictHashKey
(
d
,
de
->
key
)
&
d
->
ht
[
1
].
sizemask
;
if
(
dv
->
entry
[
j
].
key
==
NULL
)
continue
;
de
->
next
=
d
->
ht
[
1
].
table
[
h
];
h
=
dictHashKey
(
d
,
dv
->
entry
[
j
].
key
)
&
d
->
ht
[
1
].
sizemask
;
d
->
ht
[
1
].
table
[
h
]
=
de
;
dictPushEntry
(
d
->
ht
[
1
].
table
,
h
,
dv
->
entry
[
j
].
key
,
dv
->
entry
[
j
].
v
.
val
);
d
->
ht
[
0
].
used
--
;
d
->
ht
[
1
].
used
++
;
d
->
ht
[
1
].
used
++
;
d
e
=
nextde
;
d
->
ht
[
0
].
used
--
;
}
}
zfree
(
dv
);
d
->
ht
[
0
].
table
[
d
->
rehashidx
]
=
NULL
;
d
->
ht
[
0
].
table
[
d
->
rehashidx
]
=
NULL
;
d
->
rehashidx
++
;
d
->
rehashidx
++
;
}
}
...
@@ -360,18 +389,11 @@ dictEntry *dictAddRaw(dict *d, void *key)
...
@@ -360,18 +389,11 @@ dictEntry *dictAddRaw(dict *d, void *key)
if
((
index
=
_dictKeyIndex
(
d
,
key
))
==
-
1
)
if
((
index
=
_dictKeyIndex
(
d
,
key
))
==
-
1
)
return
NULL
;
return
NULL
;
/* Allocate the memory and store the new entry.
/* Store the new entry: if we are rehashing all new entries go to
* Insert the element in top, with the assumption that in a database
* the new table. */
* system it is more likely that recently added entries are accessed
* more frequently. */
ht
=
dictIsRehashing
(
d
)
?
&
d
->
ht
[
1
]
:
&
d
->
ht
[
0
];
ht
=
dictIsRehashing
(
d
)
?
&
d
->
ht
[
1
]
:
&
d
->
ht
[
0
];
entry
=
zmalloc
(
sizeof
(
*
entry
));
entry
=
dictPushEntry
(
ht
->
table
,
index
,
key
,
NULL
);
entry
->
next
=
ht
->
table
[
index
];
ht
->
table
[
index
]
=
entry
;
ht
->
used
++
;
ht
->
used
++
;
/* Set the hash entry fields. */
dictSetKey
(
d
,
entry
,
key
);
return
entry
;
return
entry
;
}
}
...
@@ -413,10 +435,9 @@ dictEntry *dictReplaceRaw(dict *d, void *key) {
...
@@ -413,10 +435,9 @@ dictEntry *dictReplaceRaw(dict *d, void *key) {
}
}
/* Search and remove an element */
/* Search and remove an element */
static
int
dictGenericDelete
(
dict
*
d
,
const
void
*
key
,
int
nofree
)
static
int
dictGenericDelete
(
dict
*
d
,
const
void
*
key
,
int
nofree
)
{
{
unsigned
int
h
,
idx
;
unsigned
int
h
,
idx
;
dictEntry
*
he
,
*
prevHe
;
dictEntry
Vector
*
dv
;
int
table
;
int
table
;
if
(
d
->
ht
[
0
].
size
==
0
)
return
DICT_ERR
;
/* d->ht[0].table is NULL */
if
(
d
->
ht
[
0
].
size
==
0
)
return
DICT_ERR
;
/* d->ht[0].table is NULL */
...
@@ -425,25 +446,36 @@ static int dictGenericDelete(dict *d, const void *key, int nofree)
...
@@ -425,25 +446,36 @@ static int dictGenericDelete(dict *d, const void *key, int nofree)
for
(
table
=
0
;
table
<=
1
;
table
++
)
{
for
(
table
=
0
;
table
<=
1
;
table
++
)
{
idx
=
h
&
d
->
ht
[
table
].
sizemask
;
idx
=
h
&
d
->
ht
[
table
].
sizemask
;
he
=
d
->
ht
[
table
].
table
[
idx
];
dv
=
d
->
ht
[
table
].
table
[
idx
];
prevHe
=
NULL
;
if
(
dv
!=
NULL
)
{
while
(
he
)
{
uint32_t
entries
=
dv
->
used
+
dv
->
free
;
uint32_t
j
;
for
(
j
=
0
;
j
<
entries
;
j
++
)
{
dictEntry
*
he
=
dv
->
entry
+
j
;
if
(
he
->
key
==
NULL
)
continue
;
if
(
key
==
he
->
key
||
dictCompareKeys
(
d
,
key
,
he
->
key
))
{
if
(
key
==
he
->
key
||
dictCompareKeys
(
d
,
key
,
he
->
key
))
{
/* Unlink the element from the list */
if
(
prevHe
)
prevHe
->
next
=
he
->
next
;
else
d
->
ht
[
table
].
table
[
idx
]
=
he
->
next
;
if
(
!
nofree
)
{
if
(
!
nofree
)
{
dictFreeKey
(
d
,
he
);
dictFreeKey
(
d
,
he
);
dictFreeVal
(
d
,
he
);
dictFreeVal
(
d
,
he
);
}
}
zfree
(
he
);
he
->
key
=
NULL
;
he
->
v
.
val
=
NULL
;
d
->
ht
[
table
].
used
--
;
d
->
ht
[
table
].
used
--
;
dv
->
free
++
;
dv
->
used
--
;
if
(
dv
->
used
==
0
)
{
zfree
(
dv
);
d
->
ht
[
table
].
table
[
idx
]
=
NULL
;
}
/* TODO: Compact the entries vector if there are not
* safe iterators active? Alternatively we could do it
* in a single place when scanning entries for read
* accesses. */
return
DICT_OK
;
return
DICT_OK
;
}
}
prevHe
=
he
;
}
he
=
he
->
next
;
}
}
if
(
!
dictIsRehashing
(
d
))
break
;
if
(
!
dictIsRehashing
(
d
))
break
;
}
}
...
@@ -464,21 +496,24 @@ int _dictClear(dict *d, dictht *ht, void(callback)(void *)) {
...
@@ -464,21 +496,24 @@ int _dictClear(dict *d, dictht *ht, void(callback)(void *)) {
/* Free all the elements */
/* Free all the elements */
for
(
i
=
0
;
i
<
ht
->
size
&&
ht
->
used
>
0
;
i
++
)
{
for
(
i
=
0
;
i
<
ht
->
size
&&
ht
->
used
>
0
;
i
++
)
{
dictEntry
*
he
,
*
nextHe
;
dictEntry
Vector
*
dv
;
if
(
callback
&&
(
i
&
65535
)
==
0
)
callback
(
d
->
privdata
);
if
(
callback
&&
(
i
&
65535
)
==
0
)
callback
(
d
->
privdata
);
if
((
he
=
ht
->
table
[
i
])
==
NULL
)
continue
;
if
((
dv
=
ht
->
table
[
i
])
==
NULL
)
continue
;
while
(
he
)
{
uint32_t
entries
=
dv
->
used
+
dv
->
free
;
nextHe
=
he
->
next
;
uint32_t
j
;
for
(
j
=
0
;
j
<
entries
;
j
++
)
{
dictEntry
*
he
=
dv
->
entry
+
j
;
if
(
he
->
key
==
NULL
)
continue
;
dictFreeKey
(
d
,
he
);
dictFreeKey
(
d
,
he
);
dictFreeVal
(
d
,
he
);
dictFreeVal
(
d
,
he
);
zfree
(
he
);
ht
->
used
--
;
ht
->
used
--
;
he
=
nextHe
;
}
}
zfree
(
dv
);
ht
->
table
[
i
]
=
NULL
;
}
}
/* Free the table
and the allocated cache structure
*/
/* Free the table
itself.
*/
zfree
(
ht
->
table
);
zfree
(
ht
->
table
);
/* Re-initialize the table */
/* Re-initialize the table */
_dictReset
(
ht
);
_dictReset
(
ht
);
...
@@ -495,7 +530,7 @@ void dictRelease(dict *d)
...
@@ -495,7 +530,7 @@ void dictRelease(dict *d)
dictEntry
*
dictFind
(
dict
*
d
,
const
void
*
key
)
dictEntry
*
dictFind
(
dict
*
d
,
const
void
*
key
)
{
{
dictEntry
*
he
;
dictEntry
Vector
*
dv
;
unsigned
int
h
,
idx
,
table
;
unsigned
int
h
,
idx
,
table
;
if
(
d
->
ht
[
0
].
used
+
d
->
ht
[
1
].
used
==
0
)
return
NULL
;
/* dict is empty */
if
(
d
->
ht
[
0
].
used
+
d
->
ht
[
1
].
used
==
0
)
return
NULL
;
/* dict is empty */
...
@@ -503,11 +538,16 @@ dictEntry *dictFind(dict *d, const void *key)
...
@@ -503,11 +538,16 @@ dictEntry *dictFind(dict *d, const void *key)
h
=
dictHashKey
(
d
,
key
);
h
=
dictHashKey
(
d
,
key
);
for
(
table
=
0
;
table
<=
1
;
table
++
)
{
for
(
table
=
0
;
table
<=
1
;
table
++
)
{
idx
=
h
&
d
->
ht
[
table
].
sizemask
;
idx
=
h
&
d
->
ht
[
table
].
sizemask
;
he
=
d
->
ht
[
table
].
table
[
idx
];
dv
=
d
->
ht
[
table
].
table
[
idx
];
while
(
he
)
{
if
(
dv
!=
NULL
)
{
uint32_t
entries
=
dv
->
used
+
dv
->
free
;
uint32_t
j
;
for
(
j
=
0
;
j
<
entries
;
j
++
)
{
dictEntry
*
he
=
dv
->
entry
+
j
;
if
(
he
->
key
==
NULL
)
continue
;
/* Empty slot. */
if
(
key
==
he
->
key
||
dictCompareKeys
(
d
,
key
,
he
->
key
))
if
(
key
==
he
->
key
||
dictCompareKeys
(
d
,
key
,
he
->
key
))
return
he
;
return
he
;
he
=
he
->
next
;
}
}
}
if
(
!
dictIsRehashing
(
d
))
return
NULL
;
if
(
!
dictIsRehashing
(
d
))
return
NULL
;
}
}
...
@@ -523,8 +563,8 @@ void *dictFetchValue(dict *d, const void *key) {
...
@@ -523,8 +563,8 @@ void *dictFetchValue(dict *d, const void *key) {
/* A fingerprint is a 64 bit number that represents the state of the dictionary
/* A fingerprint is a 64 bit number that represents the state of the dictionary
* at a given time, it's just a few dict properties xored together.
* at a given time, it's just a few dict properties xored together.
* When an unsafe iterator is initialized, we get the dict fingerprint, and
check
* When an unsafe iterator is initialized, we get the dict fingerprint, and
* the fingerprint again when the iterator is released.
*
check
the fingerprint again when the iterator is released.
* If the two fingerprints are different it means that the user of the iterator
* If the two fingerprints are different it means that the user of the iterator
* performed forbidden operations against the dictionary while iterating. */
* performed forbidden operations against the dictionary while iterating. */
long
long
dictFingerprint
(
dict
*
d
)
{
long
long
dictFingerprint
(
dict
*
d
)
{
...
@@ -543,8 +583,8 @@ long long dictFingerprint(dict *d) {
...
@@ -543,8 +583,8 @@ long long dictFingerprint(dict *d) {
*
*
* Result = hash(hash(hash(int1)+int2)+int3) ...
* Result = hash(hash(hash(int1)+int2)+int3) ...
*
*
* This way the same set of integers in a different order will (likely)
hash
* This way the same set of integers in a different order will (likely)
* to a different number. */
*
hash
to a different number. */
for
(
j
=
0
;
j
<
6
;
j
++
)
{
for
(
j
=
0
;
j
<
6
;
j
++
)
{
hash
+=
integers
[
j
];
hash
+=
integers
[
j
];
/* For the hashing step we use Tomas Wang's 64 bit integer hash. */
/* For the hashing step we use Tomas Wang's 64 bit integer hash. */
...
@@ -567,8 +607,7 @@ dictIterator *dictGetIterator(dict *d)
...
@@ -567,8 +607,7 @@ dictIterator *dictGetIterator(dict *d)
iter
->
table
=
0
;
iter
->
table
=
0
;
iter
->
index
=
-
1
;
iter
->
index
=
-
1
;
iter
->
safe
=
0
;
iter
->
safe
=
0
;
iter
->
entry
=
NULL
;
iter
->
entry
=
-
1
;
iter
->
nextEntry
=
NULL
;
return
iter
;
return
iter
;
}
}
...
@@ -579,10 +618,9 @@ dictIterator *dictGetSafeIterator(dict *d) {
...
@@ -579,10 +618,9 @@ dictIterator *dictGetSafeIterator(dict *d) {
return
i
;
return
i
;
}
}
dictEntry
*
dictNext
(
dictIterator
*
iter
)
dictEntry
*
dictNext
(
dictIterator
*
iter
)
{
{
while
(
1
)
{
while
(
1
)
{
if
(
iter
->
entry
==
NULL
)
{
if
(
iter
->
entry
==
-
1
)
{
dictht
*
ht
=
&
iter
->
d
->
ht
[
iter
->
table
];
dictht
*
ht
=
&
iter
->
d
->
ht
[
iter
->
table
];
if
(
iter
->
index
==
-
1
&&
iter
->
table
==
0
)
{
if
(
iter
->
index
==
-
1
&&
iter
->
table
==
0
)
{
if
(
iter
->
safe
)
if
(
iter
->
safe
)
...
@@ -600,15 +638,17 @@ dictEntry *dictNext(dictIterator *iter)
...
@@ -600,15 +638,17 @@ dictEntry *dictNext(dictIterator *iter)
break
;
break
;
}
}
}
}
iter
->
entry
=
ht
->
table
[
iter
->
index
]
;
iter
->
entry
=
0
;
}
else
{
}
else
{
iter
->
entry
=
iter
->
nextEntry
;
iter
->
entry
++
;
}
}
if
(
iter
->
entry
)
{
/* We need to save the 'next' here, the iterator user
dictEntryVector
*
dv
=
iter
->
d
->
ht
[
iter
->
table
].
table
[
iter
->
index
];
* may delete the entry we are returning. */
if
(
dv
==
NULL
||
(
unsigned
)
iter
->
entry
>=
(
dv
->
used
+
dv
->
free
))
iter
->
nextEntry
=
iter
->
entry
->
next
;
iter
->
entry
=
-
1
;
return
iter
->
entry
;
if
(
iter
->
entry
>=
0
&&
dv
->
entry
[
iter
->
entry
].
key
!=
NULL
)
{
return
dv
->
entry
+
iter
->
entry
;
}
}
}
}
return
NULL
;
return
NULL
;
...
@@ -629,9 +669,8 @@ void dictReleaseIterator(dictIterator *iter)
...
@@ -629,9 +669,8 @@ void dictReleaseIterator(dictIterator *iter)
* implement randomized algorithms */
* implement randomized algorithms */
dictEntry
*
dictGetRandomKey
(
dict
*
d
)
dictEntry
*
dictGetRandomKey
(
dict
*
d
)
{
{
dictEntry
*
he
,
*
orighe
;
dictEntry
Vector
*
dv
;
unsigned
int
h
;
unsigned
int
h
;
int
listlen
,
listele
;
if
(
dictSize
(
d
)
==
0
)
return
NULL
;
if
(
dictSize
(
d
)
==
0
)
return
NULL
;
if
(
dictIsRehashing
(
d
))
_dictRehashStep
(
d
);
if
(
dictIsRehashing
(
d
))
_dictRehashStep
(
d
);
...
@@ -642,29 +681,26 @@ dictEntry *dictGetRandomKey(dict *d)
...
@@ -642,29 +681,26 @@ dictEntry *dictGetRandomKey(dict *d)
h
=
d
->
rehashidx
+
(
random
()
%
(
d
->
ht
[
0
].
size
+
h
=
d
->
rehashidx
+
(
random
()
%
(
d
->
ht
[
0
].
size
+
d
->
ht
[
1
].
size
-
d
->
ht
[
1
].
size
-
d
->
rehashidx
));
d
->
rehashidx
));
he
=
(
h
>=
d
->
ht
[
0
].
size
)
?
d
->
ht
[
1
].
table
[
h
-
d
->
ht
[
0
].
size
]
:
dv
=
(
h
>=
d
->
ht
[
0
].
size
)
?
d
->
ht
[
1
].
table
[
h
-
d
->
ht
[
0
].
size
]
:
d
->
ht
[
0
].
table
[
h
];
d
->
ht
[
0
].
table
[
h
];
}
while
(
he
==
NULL
);
}
while
(
dv
==
NULL
);
}
else
{
}
else
{
do
{
do
{
h
=
random
()
&
d
->
ht
[
0
].
sizemask
;
h
=
random
()
&
d
->
ht
[
0
].
sizemask
;
he
=
d
->
ht
[
0
].
table
[
h
];
dv
=
d
->
ht
[
0
].
table
[
h
];
}
while
(
he
==
NULL
);
}
while
(
dv
==
NULL
);
}
}
/* Now we found a non empty
bucket, but it is a linked
/* Now we found a non empty
vector. We need to get a random element from
*
list and we need to get a random element from the list.
*
the vector now. */
* The only sane way to do so is counting the elements and
dictEntry
*
he
=
NULL
;
* select a random index. */
while
(
he
==
NULL
)
{
listlen
=
0
;
uint32_t
r
=
random
()
%
(
dv
->
used
+
dv
->
free
)
;
orighe
=
he
;
if
(
dv
->
entry
[
r
].
key
!=
NULL
)
{
while
(
he
)
{
he
=
dv
->
entry
+
r
;
he
=
he
->
next
;
break
;
listlen
++
;
}
}
}
listele
=
random
()
%
listlen
;
he
=
orighe
;
while
(
listele
--
)
he
=
he
->
next
;
return
he
;
return
he
;
}
}
...
@@ -729,11 +765,11 @@ unsigned int dictGetSomeKeys(dict *d, dictEntry **des, unsigned int count) {
...
@@ -729,11 +765,11 @@ unsigned int dictGetSomeKeys(dict *d, dictEntry **des, unsigned int count) {
continue
;
continue
;
}
}
if
(
i
>=
d
->
ht
[
j
].
size
)
continue
;
/* Out of range for this table. */
if
(
i
>=
d
->
ht
[
j
].
size
)
continue
;
/* Out of range for this table. */
dictEntry
*
he
=
d
->
ht
[
j
].
table
[
i
];
dictEntry
Vector
*
dv
=
d
->
ht
[
j
].
table
[
i
];
/* Count contiguous empty buckets, and jump to other
/* Count contiguous empty buckets, and jump to other
* locations if they reach 'count' (with a minimum of 5). */
* locations if they reach 'count' (with a minimum of 5). */
if
(
he
==
NULL
)
{
if
(
dv
==
NULL
)
{
emptylen
++
;
emptylen
++
;
if
(
emptylen
>=
5
&&
emptylen
>
count
)
{
if
(
emptylen
>=
5
&&
emptylen
>
count
)
{
i
=
random
()
&
maxsizemask
;
i
=
random
()
&
maxsizemask
;
...
@@ -741,17 +777,20 @@ unsigned int dictGetSomeKeys(dict *d, dictEntry **des, unsigned int count) {
...
@@ -741,17 +777,20 @@ unsigned int dictGetSomeKeys(dict *d, dictEntry **des, unsigned int count) {
}
}
}
else
{
}
else
{
emptylen
=
0
;
emptylen
=
0
;
while
(
he
)
{
uint32_t
entries
=
dv
->
used
+
dv
->
free
;
uint32_t
j
;
for
(
j
=
0
;
j
<
entries
;
j
++
)
{
if
(
dv
->
entry
[
j
].
key
!=
NULL
)
{
/* Collect all the elements of the buckets found non
/* Collect all the elements of the buckets found non
* empty while iterating. */
* empty while iterating. */
*
des
=
he
;
*
des
=
dv
->
entry
+
j
;
des
++
;
des
++
;
he
=
he
->
next
;
stored
++
;
stored
++
;
if
(
stored
==
count
)
return
stored
;
if
(
stored
==
count
)
return
stored
;
}
}
}
}
}
}
}
i
=
(
i
+
1
)
&
maxsizemask
;
i
=
(
i
+
1
)
&
maxsizemask
;
}
}
return
stored
;
return
stored
;
...
@@ -859,7 +898,8 @@ unsigned long dictScan(dict *d,
...
@@ -859,7 +898,8 @@ unsigned long dictScan(dict *d,
void
*
privdata
)
void
*
privdata
)
{
{
dictht
*
t0
,
*
t1
;
dictht
*
t0
,
*
t1
;
const
dictEntry
*
de
,
*
next
;
const
dictEntryVector
*
dv
;
uint32_t
j
;
unsigned
long
m0
,
m1
;
unsigned
long
m0
,
m1
;
if
(
dictSize
(
d
)
==
0
)
return
0
;
if
(
dictSize
(
d
)
==
0
)
return
0
;
...
@@ -869,13 +909,11 @@ unsigned long dictScan(dict *d,
...
@@ -869,13 +909,11 @@ unsigned long dictScan(dict *d,
m0
=
t0
->
sizemask
;
m0
=
t0
->
sizemask
;
/* Emit entries at cursor */
/* Emit entries at cursor */
de
=
t0
->
table
[
v
&
m0
];
dv
=
t0
->
table
[
v
&
m0
];
while
(
de
)
{
for
(
j
=
0
;
dv
&&
j
<
(
dv
->
used
+
dv
->
free
);
j
++
)
{
next
=
de
->
next
;
if
(
dv
->
entry
[
j
].
key
==
NULL
)
continue
;
fn
(
privdata
,
de
);
fn
(
privdata
,
dv
->
entry
+
j
);
de
=
next
;
}
}
}
else
{
}
else
{
t0
=
&
d
->
ht
[
0
];
t0
=
&
d
->
ht
[
0
];
t1
=
&
d
->
ht
[
1
];
t1
=
&
d
->
ht
[
1
];
...
@@ -890,22 +928,20 @@ unsigned long dictScan(dict *d,
...
@@ -890,22 +928,20 @@ unsigned long dictScan(dict *d,
m1
=
t1
->
sizemask
;
m1
=
t1
->
sizemask
;
/* Emit entries at cursor */
/* Emit entries at cursor */
de
=
t0
->
table
[
v
&
m0
];
dv
=
t0
->
table
[
v
&
m0
];
while
(
de
)
{
for
(
j
=
0
;
dv
&&
j
<
(
dv
->
used
+
dv
->
free
);
j
++
)
{
next
=
de
->
next
;
if
(
dv
->
entry
[
j
].
key
==
NULL
)
continue
;
fn
(
privdata
,
de
);
fn
(
privdata
,
dv
->
entry
+
j
);
de
=
next
;
}
}
/* Iterate over indices in larger table that are the expansion
/* Iterate over indices in larger table that are the expansion
* of the index pointed to by the cursor in the smaller table */
* of the index pointed to by the cursor in the smaller table */
do
{
do
{
/* Emit entries at cursor */
/* Emit entries at cursor */
de
=
t1
->
table
[
v
&
m1
];
dv
=
t1
->
table
[
v
&
m1
];
while
(
de
)
{
for
(
j
=
0
;
dv
&&
j
<
(
dv
->
used
+
dv
->
free
);
j
++
)
{
next
=
de
->
next
;
if
(
dv
->
entry
[
j
].
key
==
NULL
)
continue
;
fn
(
privdata
,
de
);
fn
(
privdata
,
dv
->
entry
+
j
);
de
=
next
;
}
}
/* Increment bits not covered by the smaller mask */
/* Increment bits not covered by the smaller mask */
...
@@ -942,11 +978,11 @@ static int _dictExpandIfNeeded(dict *d)
...
@@ -942,11 +978,11 @@ static int _dictExpandIfNeeded(dict *d)
* table (global setting) or we should avoid it but the ratio between
* table (global setting) or we should avoid it but the ratio between
* elements/buckets is over the "safe" threshold, we resize doubling
* elements/buckets is over the "safe" threshold, we resize doubling
* the number of buckets. */
* the number of buckets. */
if
(
d
->
ht
[
0
].
used
>=
d
->
ht
[
0
].
size
&&
if
(
d
->
ht
[
0
].
used
>=
(
d
->
ht
[
0
].
size
*
dict_resize_ratio
)
&&
(
dict_can_resize
||
(
dict_can_resize
||
d
->
ht
[
0
].
used
/
d
->
ht
[
0
].
size
>
dict_force_resize_ratio
))
d
->
ht
[
0
].
used
/
d
->
ht
[
0
].
size
>
dict_force_resize_ratio
))
{
{
return
dictExpand
(
d
,
d
->
ht
[
0
].
used
*
2
);
return
dictExpand
(
d
,
d
->
ht
[
0
].
used
/
(
dict_resize_ratio
/
2
)
);
}
}
return
DICT_OK
;
return
DICT_OK
;
}
}
...
@@ -973,7 +1009,7 @@ static unsigned long _dictNextPower(unsigned long size)
...
@@ -973,7 +1009,7 @@ static unsigned long _dictNextPower(unsigned long size)
static
int
_dictKeyIndex
(
dict
*
d
,
const
void
*
key
)
static
int
_dictKeyIndex
(
dict
*
d
,
const
void
*
key
)
{
{
unsigned
int
h
,
idx
,
table
;
unsigned
int
h
,
idx
,
table
;
dictEntry
*
he
;
dictEntry
Vector
*
dv
;
/* Expand the hash table if needed */
/* Expand the hash table if needed */
if
(
_dictExpandIfNeeded
(
d
)
==
DICT_ERR
)
if
(
_dictExpandIfNeeded
(
d
)
==
DICT_ERR
)
...
@@ -983,11 +1019,16 @@ static int _dictKeyIndex(dict *d, const void *key)
...
@@ -983,11 +1019,16 @@ static int _dictKeyIndex(dict *d, const void *key)
for
(
table
=
0
;
table
<=
1
;
table
++
)
{
for
(
table
=
0
;
table
<=
1
;
table
++
)
{
idx
=
h
&
d
->
ht
[
table
].
sizemask
;
idx
=
h
&
d
->
ht
[
table
].
sizemask
;
/* Search if this slot does not already contain the given key */
/* Search if this slot does not already contain the given key */
he
=
d
->
ht
[
table
].
table
[
idx
];
dv
=
d
->
ht
[
table
].
table
[
idx
];
while
(
he
)
{
if
(
dv
!=
NULL
)
{
uint32_t
entries
=
dv
->
used
+
dv
->
free
;
uint32_t
j
;
for
(
j
=
0
;
j
<
entries
;
j
++
)
{
dictEntry
*
he
=
dv
->
entry
+
j
;
if
(
he
->
key
==
NULL
)
continue
;
if
(
key
==
he
->
key
||
dictCompareKeys
(
d
,
key
,
he
->
key
))
if
(
key
==
he
->
key
||
dictCompareKeys
(
d
,
key
,
he
->
key
))
return
-
1
;
return
-
1
;
he
=
he
->
next
;
}
}
}
if
(
!
dictIsRehashing
(
d
))
break
;
if
(
!
dictIsRehashing
(
d
))
break
;
}
}
...
@@ -1026,20 +1067,14 @@ size_t _dictGetStatsHt(char *buf, size_t bufsize, dictht *ht, int tableid) {
...
@@ -1026,20 +1067,14 @@ size_t _dictGetStatsHt(char *buf, size_t bufsize, dictht *ht, int tableid) {
/* Compute stats. */
/* Compute stats. */
for
(
i
=
0
;
i
<
DICT_STATS_VECTLEN
;
i
++
)
clvector
[
i
]
=
0
;
for
(
i
=
0
;
i
<
DICT_STATS_VECTLEN
;
i
++
)
clvector
[
i
]
=
0
;
for
(
i
=
0
;
i
<
ht
->
size
;
i
++
)
{
for
(
i
=
0
;
i
<
ht
->
size
;
i
++
)
{
dictEntry
*
he
;
if
(
ht
->
table
[
i
]
==
NULL
)
{
if
(
ht
->
table
[
i
]
==
NULL
)
{
clvector
[
0
]
++
;
clvector
[
0
]
++
;
continue
;
continue
;
}
}
slots
++
;
slots
++
;
/* For each hash entry on this slot... */
/* For each hash entry on this slot... */
chainlen
=
0
;
dictEntryVector
*
dv
=
ht
->
table
[
i
];
he
=
ht
->
table
[
i
];
chainlen
=
dv
->
used
;
while
(
he
)
{
chainlen
++
;
he
=
he
->
next
;
}
clvector
[(
chainlen
<
DICT_STATS_VECTLEN
)
?
chainlen
:
(
DICT_STATS_VECTLEN
-
1
)]
++
;
clvector
[(
chainlen
<
DICT_STATS_VECTLEN
)
?
chainlen
:
(
DICT_STATS_VECTLEN
-
1
)]
++
;
if
(
chainlen
>
maxchainlen
)
maxchainlen
=
chainlen
;
if
(
chainlen
>
maxchainlen
)
maxchainlen
=
chainlen
;
totchainlen
+=
chainlen
;
totchainlen
+=
chainlen
;
...
...
src/dict.h
View file @
8504bf18
...
@@ -54,10 +54,11 @@ typedef struct dictEntry {
...
@@ -54,10 +54,11 @@ typedef struct dictEntry {
}
v
;
}
v
;
}
dictEntry
;
}
dictEntry
;
typedef
struct
dictEntrySlot
{
typedef
struct
dictEntryVector
{
unsigned
long
numentries
;
uint32_t
used
;
/* Number of used entries. */
dictEntry
*
entries
;
uint32_t
free
;
/* Number of free entries (with key field = NULL). */
}
dictEntrySlot
;
dictEntry
entry
[];
}
dictEntryVector
;
typedef
struct
dictType
{
typedef
struct
dictType
{
unsigned
int
(
*
hashFunction
)(
const
void
*
key
);
unsigned
int
(
*
hashFunction
)(
const
void
*
key
);
...
@@ -71,7 +72,7 @@ typedef struct dictType {
...
@@ -71,7 +72,7 @@ typedef struct dictType {
/* This is our hash table structure. Every dictionary has two of this as we
/* This is our hash table structure. Every dictionary has two of this as we
* implement incremental rehashing, for the old to the new table. */
* implement incremental rehashing, for the old to the new table. */
typedef
struct
dictht
{
typedef
struct
dictht
{
dictEntry
Slot
**
table
;
dictEntry
Vector
**
table
;
unsigned
long
size
;
unsigned
long
size
;
unsigned
long
sizemask
;
unsigned
long
sizemask
;
unsigned
long
used
;
unsigned
long
used
;
...
@@ -93,7 +94,7 @@ typedef struct dictIterator {
...
@@ -93,7 +94,7 @@ typedef struct dictIterator {
dict
*
d
;
dict
*
d
;
long
index
;
long
index
;
int
table
,
safe
;
int
table
,
safe
;
dictEntry
*
entry
,
*
nextEntry
;
long
entry
;
/* Current entry position in the cluster. */
/* unsafe iterator fingerprint for misuse detection. */
/* unsafe iterator fingerprint for misuse detection. */
long
long
fingerprint
;
long
long
fingerprint
;
}
dictIterator
;
}
dictIterator
;
...
@@ -153,6 +154,7 @@ typedef void (dictScanFunction)(void *privdata, const dictEntry *de);
...
@@ -153,6 +154,7 @@ typedef void (dictScanFunction)(void *privdata, const dictEntry *de);
/* API */
/* API */
dict
*
dictCreate
(
dictType
*
type
,
void
*
privDataPtr
);
dict
*
dictCreate
(
dictType
*
type
,
void
*
privDataPtr
);
int
dictExpand
(
dict
*
d
,
unsigned
long
size
);
int
dictExpand
(
dict
*
d
,
unsigned
long
size
);
int
dictExpandToOptimalSize
(
dict
*
d
,
unsigned
long
entries
);
int
dictAdd
(
dict
*
d
,
void
*
key
,
void
*
val
);
int
dictAdd
(
dict
*
d
,
void
*
key
,
void
*
val
);
dictEntry
*
dictAddRaw
(
dict
*
d
,
void
*
key
);
dictEntry
*
dictAddRaw
(
dict
*
d
,
void
*
key
);
int
dictReplace
(
dict
*
d
,
void
*
key
,
void
*
val
);
int
dictReplace
(
dict
*
d
,
void
*
key
,
void
*
val
);
...
...
src/rdb.c
View file @
8504bf18
...
@@ -1086,7 +1086,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
...
@@ -1086,7 +1086,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
/* It's faster to expand the dict to the right size asap in order
/* It's faster to expand the dict to the right size asap in order
* to avoid rehashing */
* to avoid rehashing */
if
(
len
>
DICT_HT_INITIAL_SIZE
)
if
(
len
>
DICT_HT_INITIAL_SIZE
)
dictExpand
(
o
->
ptr
,
len
);
dictExpand
ToOptimalSize
(
o
->
ptr
,
len
);
}
else
{
}
else
{
o
=
createIntsetObject
();
o
=
createIntsetObject
();
}
}
...
@@ -1105,7 +1105,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
...
@@ -1105,7 +1105,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
o
->
ptr
=
intsetAdd
(
o
->
ptr
,
llval
,
NULL
);
o
->
ptr
=
intsetAdd
(
o
->
ptr
,
llval
,
NULL
);
}
else
{
}
else
{
setTypeConvert
(
o
,
OBJ_ENCODING_HT
);
setTypeConvert
(
o
,
OBJ_ENCODING_HT
);
dictExpand
(
o
->
ptr
,
len
);
dictExpand
ToOptimalSize
(
o
->
ptr
,
len
);
}
}
}
}
...
@@ -1452,8 +1452,8 @@ int rdbLoad(char *filename) {
...
@@ -1452,8 +1452,8 @@ int rdbLoad(char *filename) {
goto
eoferr
;
goto
eoferr
;
if
((
expires_size
=
rdbLoadLen
(
&
rdb
,
NULL
))
==
RDB_LENERR
)
if
((
expires_size
=
rdbLoadLen
(
&
rdb
,
NULL
))
==
RDB_LENERR
)
goto
eoferr
;
goto
eoferr
;
dictExpand
(
db
->
dict
,
db_size
);
dictExpand
ToOptimalSize
(
db
->
dict
,
db_size
);
dictExpand
(
db
->
expires
,
expires_size
);
dictExpand
ToOptimalSize
(
db
->
expires
,
expires_size
);
continue
;
/* Read type again. */
continue
;
/* Read type again. */
}
else
if
(
type
==
RDB_OPCODE_AUX
)
{
}
else
if
(
type
==
RDB_OPCODE_AUX
)
{
/* AUX: generic string-string fields. Use to add state to RDB
/* AUX: generic string-string fields. Use to add state to RDB
...
...
src/t_set.c
View file @
8504bf18
...
@@ -243,7 +243,7 @@ void setTypeConvert(robj *setobj, int enc) {
...
@@ -243,7 +243,7 @@ void setTypeConvert(robj *setobj, int enc) {
sds
element
;
sds
element
;
/* Presize the dict to avoid rehashing */
/* Presize the dict to avoid rehashing */
dictExpand
(
d
,
intsetLen
(
setobj
->
ptr
));
dictExpand
ToOptimalSize
(
d
,
intsetLen
(
setobj
->
ptr
));
/* To add the elements we extract integers and create redis objects */
/* To add the elements we extract integers and create redis objects */
si
=
setTypeInitIterator
(
setobj
);
si
=
setTypeInitIterator
(
setobj
);
...
...
src/t_zset.c
View file @
8504bf18
...
@@ -2268,7 +2268,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
...
@@ -2268,7 +2268,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
if
(
setnum
)
{
if
(
setnum
)
{
/* Our union is at least as large as the largest set.
/* Our union is at least as large as the largest set.
* Resize the dictionary ASAP to avoid useless rehashing. */
* Resize the dictionary ASAP to avoid useless rehashing. */
dictExpand
(
accumulator
,
zuiLength
(
&
src
[
setnum
-
1
]));
dictExpand
ToOptimalSize
(
accumulator
,
zuiLength
(
&
src
[
setnum
-
1
]));
}
}
/* Step 1: Create a dictionary of elements -> aggregated-scores
/* Step 1: Create a dictionary of elements -> aggregated-scores
...
@@ -2313,7 +2313,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
...
@@ -2313,7 +2313,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
/* We now are aware of the final size of the resulting sorted set,
/* We now are aware of the final size of the resulting sorted set,
* let's resize the dictionary embedded inside the sorted set to the
* let's resize the dictionary embedded inside the sorted set to the
* right size, in order to save rehashing time. */
* right size, in order to save rehashing time. */
dictExpand
(
dstzset
->
dict
,
dictSize
(
accumulator
));
dictExpand
ToOptimalSize
(
dstzset
->
dict
,
dictSize
(
accumulator
));
while
((
de
=
dictNext
(
di
))
!=
NULL
)
{
while
((
de
=
dictNext
(
di
))
!=
NULL
)
{
sds
ele
=
dictGetKey
(
de
);
sds
ele
=
dictGetKey
(
de
);
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment