Unverified Commit 3ca6972e authored by sundb's avatar sundb Committed by GitHub
Browse files

Replace all usage of ziplist with listpack for t_zset (#9366)

Part two of implementing #8702 (zset), after #8887.

## Description of the feature
Replaced all uses of ziplist with listpack in t_zset, and optimized some of the code to optimize performance.

## Rdb format changes
New `RDB_TYPE_ZSET_LISTPACK` rdb type.

## Rdb loading improvements:
1) Pre-expansion of dict for validation of duplicate data for listpack and ziplist.
2) Simplifying the release of empty key objects when RDB loading.
3) Unify ziplist and listpack data verify methods for zset and hash, and move code to rdb.c.

## Interface changes
1) New `zset-max-listpack-entries` config is an alias for `zset-max-ziplist-entries` (same with `zset-max-listpack-value`).
2) OBJECT ENCODING will return listpack instead of ziplist.

## Listpack improvements:
1) Add `lpDeleteRange` and `lpDeleteRangeWithEntry` functions to delete a range of entries from listpack.
2) Improve the performance of `lpCompare`, converting from string to integer is faster than converting from integer to string.
3) Replace `snprintf` with `ll2string` to improve performance in converting numbers to strings in `lpGet()`.

## Zset improvements:
1) Improve the performance of `zzlFind` method, use `lpFind` instead of `lpCompare` in a loop.
2) Use `lpDeleteRangeWithEntry` instead of `lpDelete` twice to delete a element of zset.

## Tests
1) Add some unittests for `lpDeleteRange` and `lpDeleteRangeWithEntry` function.
2) Add zset RDB loading test.
3) Add benchmark test for `lpCompare` and `ziplsitCompare`.
4) Add empty listpack zset corrupt dump test.
parent 86b0de5c
tags {"external:skip"} {
# Copy RDB with ziplist encoded hash to server path
set server_path [tmpdir "server.convert-ziplist-hash-on-load"]
exec cp -f tests/assets/zset-ziplist.rdb $server_path
start_server [list overrides [list "dir" $server_path "dbfilename" "zset-ziplist.rdb"]] {
test "RDB load ziplist zset: converts to listpack when RDB loading" {
r select 0
assert_encoding listpack zset
assert_equal 2 [r zcard zset]
assert_match {one 1 two 2} [r zrange zset 0 -1 withscores]
}
}
exec cp -f tests/assets/zset-ziplist.rdb $server_path
start_server [list overrides [list "dir" $server_path "dbfilename" "zset-ziplist.rdb" "zset-max-ziplist-entries" 1]] {
test "RDB load ziplist zset: converts to skiplist when zset-max-ziplist-entries is exceeded" {
r select 0
assert_encoding skiplist zset
assert_equal 2 [r zcard zset]
assert_match {one 1 two 2} [r zrange zset 0 -1 withscores]
}
}
}
......@@ -181,7 +181,8 @@ foreach sanitize_dump {no yes} {
verify_log_message 0 "*skipping empty key: hash_ziplist*" 0
verify_log_message 0 "*skipping empty key: zset*" 0
verify_log_message 0 "*skipping empty key: zset_ziplist*" 0
verify_log_message 0 "*empty keys skipped: 8*" 0
verify_log_message 0 "*skipping empty key: zset_listpack*" 0
verify_log_message 0 "*empty keys skipped: 9*" 0
}
}
}
......@@ -394,14 +395,13 @@ test {corrupt payload: fuzzer findings - empty intset} {
}
}
test {corrupt payload: fuzzer findings - valgrind ziplist - crash report prints freed memory} {
test {corrupt payload: fuzzer findings - zset ziplist entry lensize is 0} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no
r debug set-skip-checksum-validation 1
r RESTORE _zsetbig 0 "\x0C\x3D\x3D\x00\x00\x00\x3A\x00\x00\x00\x14\x00\x00\xF1\x02\xF1\x02\x02\x5F\x31\x04\xF2\x02\xF3\x02\xF3\x02\x02\x5F\x33\x04\xF4\x02\xEE\x02\xF5\x02\x02\x5F\x35\x04\xF6\x02\xF7\x02\xF7\x02\x02\x5F\x37\x04\xF8\x02\xF9\x02\xF9\x02\x02\x5F\x39\x04\xFA\xFF\x09\x00\xAE\xF9\x77\x2A\x47\x24\x33\xF6"
catch { r ZREMRANGEBYSCORE _zsetbig -1050966020 724 }
assert_equal [count_log_message 0 "crashed by signal"] 0
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
catch {r RESTORE _zsetbig 0 "\x0C\x3D\x3D\x00\x00\x00\x3A\x00\x00\x00\x14\x00\x00\xF1\x02\xF1\x02\x02\x5F\x31\x04\xF2\x02\xF3\x02\xF3\x02\x02\x5F\x33\x04\xF4\x02\xEE\x02\xF5\x02\x02\x5F\x35\x04\xF6\x02\xF7\x02\xF7\x02\x02\x5F\x37\x04\xF8\x02\xF9\x02\xF9\x02\x02\x5F\x39\x04\xFA\xFF\x09\x00\xAE\xF9\x77\x2A\x47\x24\x33\xF6"} err
assert_match "*Bad data format*" $err
verify_log_message 0 "*Zset ziplist integrity check failed*" 0
}
}
......@@ -523,15 +523,13 @@ test {corrupt payload: fuzzer findings - OOM in dictExpand} {
}
test {corrupt payload: fuzzer findings - invalid tail offset after removal} {
test {corrupt payload: fuzzer findings - zset ziplist invalid tail offset} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no
r debug set-skip-checksum-validation 1
r RESTORE _zset 0 "\x0C\x19\x19\x00\x00\x00\x02\x00\x00\x00\x06\x00\x00\xF1\x02\xF1\x02\x02\x5F\x31\x04\xF2\x02\xF3\x02\xF3\xFF\x09\x00\x4D\x72\x7B\x97\xCD\x9A\x70\xC1"
catch {r ZPOPMIN _zset}
catch {r ZPOPMAX _zset}
assert_equal [count_log_message 0 "crashed by signal"] 0
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
catch {r RESTORE _zset 0 "\x0C\x19\x19\x00\x00\x00\x02\x00\x00\x00\x06\x00\x00\xF1\x02\xF1\x02\x02\x5F\x31\x04\xF2\x02\xF3\x02\xF3\xFF\x09\x00\x4D\x72\x7B\x97\xCD\x9A\x70\xC1"} err
assert_match "*Bad data format*" $err
verify_log_message 0 "*Zset ziplist integrity check failed*" 0
}
}
......
......@@ -49,6 +49,7 @@ set ::all_tests {
integration/corrupt-dump-fuzzer
integration/convert-zipmap-hash-on-load
integration/convert-ziplist-hash-on-load
integration/convert-ziplist-zset-on-load
integration/logging
integration/psync2
integration/psync2-reg
......
......@@ -153,10 +153,10 @@ start_server {tags {"aofrw external:skip"} overrides {aof-use-rdb-preamble no}}
}
foreach d {string int} {
foreach e {ziplist skiplist} {
foreach e {listpack skiplist} {
test "AOF rewrite of zset with $e encoding, $d data" {
r flushall
if {$e eq {ziplist}} {set len 10} else {set len 1000}
if {$e eq {listpack}} {set len 10} else {set len 1000}
for {set j 0} {$j < $len} {incr j} {
if {$d eq {string}} {
set data [randstring 0 16 alpha]
......
......@@ -282,10 +282,10 @@ start_server {tags {"keyspace"}} {
assert_equal $digest [debug_digest_value newset2{t}]
}
test {COPY basic usage for ziplist sorted set} {
test {COPY basic usage for listpack sorted set} {
r del zset1{t} newzset1{t}
r zadd zset1{t} 123 foobar
assert_encoding ziplist zset1{t}
assert_encoding listpack zset1{t}
r copy zset1{t} newzset1{t}
set digest [debug_digest_value zset1{t}]
assert_equal $digest [debug_digest_value newzset1{t}]
......
......@@ -172,11 +172,11 @@ start_server {tags {"scan network"}} {
}
}
foreach enc {ziplist skiplist} {
foreach enc {listpack skiplist} {
test "ZSCAN with encoding $enc" {
# Create the Sorted Set
r del zset
if {$enc eq {ziplist}} {
if {$enc eq {listpack}} {
set count 30
} else {
set count 1000
......
......@@ -9,7 +9,7 @@ start_server {tags {"zset"}} {
proc basics {encoding} {
set original_max_entries [lindex [r config get zset-max-ziplist-entries] 1]
set original_max_value [lindex [r config get zset-max-ziplist-value] 1]
if {$encoding == "ziplist"} {
if {$encoding == "listpack"} {
r config set zset-max-ziplist-entries 128
r config set zset-max-ziplist-value 64
} elseif {$encoding == "skiplist"} {
......@@ -1007,7 +1007,7 @@ start_server {tags {"zset"}} {
r config set zset-max-ziplist-value $original_max_value
}
basics ziplist
basics listpack
basics skiplist
test {ZINTERSTORE regression with two sets, intset+hashtable} {
......@@ -1104,7 +1104,7 @@ start_server {tags {"zset"}} {
proc stressers {encoding} {
set original_max_entries [lindex [r config get zset-max-ziplist-entries] 1]
set original_max_value [lindex [r config get zset-max-ziplist-value] 1]
if {$encoding == "ziplist"} {
if {$encoding == "listpack"} {
# Little extra to allow proper fuzzing in the sorting stresser
r config set zset-max-ziplist-entries 256
r config set zset-max-ziplist-value 64
......@@ -1533,7 +1533,7 @@ start_server {tags {"zset"}} {
}
tags {"slow"} {
stressers ziplist
stressers listpack
stressers skiplist
}
......@@ -1686,7 +1686,7 @@ start_server {tags {"zset"}} {
}
}
foreach {type contents} "ziplist {1 a 2 b 3 c} skiplist {1 a 2 b 3 [randstring 70 90 alpha]}" {
foreach {type contents} "listpack {1 a 2 b 3 c} skiplist {1 a 2 b 3 [randstring 70 90 alpha]}" {
set original_max_value [lindex [r config get zset-max-ziplist-value] 1]
r config set zset-max-ziplist-value 10
create_zset myzset $contents
......@@ -1739,7 +1739,7 @@ start_server {tags {"zset"}} {
foreach {type contents} "
skiplist {1 a 2 b 3 c 4 d 5 e 6 f 7 g 7 h 9 i 10 [randstring 70 90 alpha]}
ziplist {1 a 2 b 3 c 4 d 5 e 6 f 7 g 7 h 9 i 10 j} " {
listpack {1 a 2 b 3 c 4 d 5 e 6 f 7 g 7 h 9 i 10 j} " {
test "ZRANDMEMBER with <count> - $type" {
set original_max_value [lindex [r config get zset-max-ziplist-value] 1]
r config set zset-max-ziplist-value 10
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment