Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
402110f9
Commit
402110f9
authored
Apr 15, 2014
by
antirez
Browse files
User-defined switch point between sparse-dense HLL encodings.
parent
d541f65d
Changes
5
Show whitespace changes
Inline
Side-by-side
redis.conf
View file @
402110f9
...
...
@@ -682,6 +682,20 @@ set-max-intset-entries 512
zset
-
max
-
ziplist
-
entries
128
zset
-
max
-
ziplist
-
value
64
# HyperLogLog sparse representation bytes limit. The limit includes the
# 16 bytes header. When an HyperLogLog using the sparse representation crosses
# this limit, it is convereted into the dense representation.
#
# A value greater than 16000 is totally useless, since at that point the
# dense representation is more memory efficient.
#
# The suggested value is ~ 3000 in order to have the benefits of
# the space efficient encoding without slowing down too much PFADD,
# which is O(N) with the sparse encoding. Thev value can be raised to
# ~ 10000 when CPU is not a concern, but space is, and the data set is
# composed of many HyperLogLogs with cardinality in the 0 - 15000 range.
hll
-
sparse
-
max
-
bytes
3000
# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in
# order to help rehashing the main Redis hash table (the one mapping top-level
# keys to values). The hash table implementation Redis uses (see dict.c)
...
...
src/config.c
View file @
402110f9
...
...
@@ -391,6 +391,8 @@ void loadServerConfigFromString(char *config) {
server.zset_max_ziplist_entries = memtoll(argv[1], NULL);
} else if (!strcasecmp(argv[0],"zset-max-ziplist-value") && argc == 2) {
server.zset_max_ziplist_value = memtoll(argv[1], NULL);
} else if (!strcasecmp(argv[0],"hll-sparse-max-bytes") && argc == 2) {
server.hll_sparse_max_bytes = memtoll(argv[1], NULL);
} else if (!strcasecmp(argv[0],"rename-command") && argc == 3) {
struct redisCommand *cmd = lookupCommand(argv[1]);
int retval;
...
...
@@ -765,6 +767,9 @@ void configSetCommand(redisClient *c) {
} else if (!strcasecmp(c->argv[2]->ptr,"zset-max-ziplist-value")) {
if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt;
server.zset_max_ziplist_value = ll;
} else if (!strcasecmp(c->argv[2]->ptr,"hll-sparse-max-bytes")) {
if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt;
server.hll_sparse_max_bytes = ll;
} else if (!strcasecmp(c->argv[2]->ptr,"lua-time-limit")) {
if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt;
server.lua_time_limit = ll;
...
...
@@ -974,6 +979,8 @@ void configGetCommand(redisClient *c) {
server.zset_max_ziplist_entries);
config_get_numerical_field("zset-max-ziplist-value",
server.zset_max_ziplist_value);
config_get_numerical_field("hll-sparse-max-bytes",
server.hll_sparse_max_bytes);
config_get_numerical_field("lua-time-limit",server.lua_time_limit);
config_get_numerical_field("slowlog-log-slower-than",
server.slowlog_log_slower_than);
...
...
@@ -1773,6 +1780,7 @@ int rewriteConfig(char *path) {
rewriteConfigNumericalOption(state,"set-max-intset-entries",server.set_max_intset_entries,REDIS_SET_MAX_INTSET_ENTRIES);
rewriteConfigNumericalOption(state,"zset-max-ziplist-entries",server.zset_max_ziplist_entries,REDIS_ZSET_MAX_ZIPLIST_ENTRIES);
rewriteConfigNumericalOption(state,"zset-max-ziplist-value",server.zset_max_ziplist_value,REDIS_ZSET_MAX_ZIPLIST_VALUE);
rewriteConfigNumericalOption(state,"hll-sparse-max-bytes",server.hll_sparse_max_bytes,REDIS_DEFAULT_HLL_SPARSE_MAX_BYTES);
rewriteConfigYesNoOption(state,"activerehashing",server.activerehashing,REDIS_DEFAULT_ACTIVE_REHASHING);
rewriteConfigClientoutputbufferlimitOption(state);
rewriteConfigNumericalOption(state,"hz",server.hz,REDIS_DEFAULT_HZ);
...
...
src/hyperloglog.c
View file @
402110f9
...
...
@@ -176,7 +176,7 @@
* involved in updating the sparse representation is not justified by the
* memory savings. The exact maximum length of the sparse representation
* when this implementation switches to the dense representation is
* configured via the define
HLL_SPARSE_MAX
.
* configured via the define
server.hll_sparse_max_bytes
.
*/
struct
hllhdr
{
...
...
@@ -202,8 +202,6 @@ struct hllhdr {
#define HLL_SPARSE 1
/* Sparse encoding */
#define HLL_MAX_ENCODING 1
#define HLL_SPARSE_MAX 3000
static
char
*
invalid_hll_err
=
"Corrupted HLL object detected"
;
/* =========================== Low level bit macros ========================= */
...
...
@@ -634,7 +632,7 @@ int hllSparseToDense(robj *o) {
* As a side effect the function may promote the HLL representation from
* sparse to dense: this happens when a register requires to be set to a value
* not representable with the sparse representation, or when the resulting
* size would be greater than
HLL_SPARSE_MAX
. */
* size would be greater than
server.hll_sparse_max_bytes
. */
int
hllSparseAdd
(
robj
*
o
,
unsigned
char
*
ele
,
size_t
elesize
)
{
struct
hllhdr
*
hdr
;
uint8_t
oldcount
,
count
,
*
sparse
,
*
end
,
*
p
,
*
prev
,
*
next
;
...
...
@@ -815,7 +813,8 @@ int hllSparseAdd(robj *o, unsigned char *ele, size_t elesize) {
int
oldlen
=
is_xzero
?
2
:
1
;
int
deltalen
=
seqlen
-
oldlen
;
if
(
deltalen
>
0
&&
sdslen
(
o
->
ptr
)
+
deltalen
>
HLL_SPARSE_MAX
)
goto
promote
;
if
(
deltalen
>
0
&&
sdslen
(
o
->
ptr
)
+
deltalen
>
server
.
hll_sparse_max_bytes
)
goto
promote
;
if
(
deltalen
&&
next
)
memmove
(
next
+
deltalen
,
next
,
end
-
next
);
sdsIncrLen
(
o
->
ptr
,
deltalen
);
memcpy
(
p
,
seq
,
seqlen
);
...
...
@@ -1312,7 +1311,7 @@ void pfselftestCommand(redisClient *c) {
/* Make sure that for small cardinalities we use sparse
* encoding. */
if
(
j
==
checkpoint
&&
j
<
HLL_SPARSE_MAX
/
2
)
{
if
(
j
==
checkpoint
&&
j
<
server
.
hll_sparse_max_bytes
/
2
)
{
hdr2
=
o
->
ptr
;
if
(
hdr2
->
encoding
!=
HLL_SPARSE
)
{
addReplyError
(
c
,
"TESTFAILED sparse encoding not used"
);
...
...
src/redis.c
View file @
402110f9
...
...
@@ -1421,6 +1421,7 @@ void initServerConfig() {
server
.
set_max_intset_entries
=
REDIS_SET_MAX_INTSET_ENTRIES
;
server
.
zset_max_ziplist_entries
=
REDIS_ZSET_MAX_ZIPLIST_ENTRIES
;
server
.
zset_max_ziplist_value
=
REDIS_ZSET_MAX_ZIPLIST_VALUE
;
server
.
hll_sparse_max_bytes
=
REDIS_DEFAULT_HLL_SPARSE_MAX_BYTES
;
server
.
shutdown_asap
=
0
;
server
.
repl_ping_slave_period
=
REDIS_REPL_PING_SLAVE_PERIOD
;
server
.
repl_timeout
=
REDIS_REPL_TIMEOUT
;
...
...
src/redis.h
View file @
402110f9
...
...
@@ -312,6 +312,9 @@
#define REDIS_ZSET_MAX_ZIPLIST_ENTRIES 128
#define REDIS_ZSET_MAX_ZIPLIST_VALUE 64
/* HyperLogLog defines */
#define REDIS_DEFAULT_HLL_SPARSE_MAX_BYTES 3000
/* Sets operations codes */
#define REDIS_OP_UNION 0
#define REDIS_OP_DIFF 1
...
...
@@ -809,6 +812,7 @@ struct redisServer {
size_t
set_max_intset_entries
;
size_t
zset_max_ziplist_entries
;
size_t
zset_max_ziplist_value
;
size_t
hll_sparse_max_bytes
;
time_t
unixtime
;
/* Unix time sampled every cron cycle. */
long
long
mstime
;
/* Like 'unixtime' but with milliseconds resolution. */
/* Pubsub */
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment