Unverified Commit 02fd76b9 authored by sundb's avatar sundb Committed by GitHub
Browse files

Replace all usage of ziplist with listpack for t_hash (#8887)



Part one of implementing #8702 (taking hashes first before other types)

## Description of the feature
1. Change ziplist encoded hash objects to listpack encoding.
2. Convert existing ziplists on RDB loading time. an O(n) operation.

## Rdb format changes
1. Add RDB_TYPE_HASH_LISTPACK rdb type.
2. Bump RDB_VERSION to 10

## Interface changes
1. New `hash-max-listpack-entries` config is an alias for `hash-max-ziplist-entries` (same with `hash-max-listpack-value`)
2. OBJECT ENCODING will return `listpack` instead of `ziplist`

## Listpack improvements:
1. Support direct insert, replace integer element (rather than convert back and forth from string)
3. Add more listpack capabilities to match the ziplist ones (like `lpFind`, `lpRandomPairs` and such)
4. Optimize element length fetching, avoid multiple calculations
5. Use inline to avoid function call overhead.

## Tests
1. Add a new test to the RDB load time conversion
2. Adding the listpack unit tests. (based on the one in ziplist.c)
3. Add a few "corrupt payload: fuzzer findings" tests, and slightly modify existing ones.
Co-authored-by: default avatarOran Agra <oran@redislabs.com>
parent cbda4929
......@@ -48,6 +48,7 @@ set ::all_tests {
integration/corrupt-dump
integration/corrupt-dump-fuzzer
integration/convert-zipmap-hash-on-load
integration/convert-ziplist-hash-on-load
integration/logging
integration/psync2
integration/psync2-reg
......
......@@ -127,10 +127,10 @@ start_server {tags {"aofrw external:skip"} overrides {aof-use-rdb-preamble no}}
}
foreach d {string int} {
foreach e {ziplist hashtable} {
foreach e {listpack hashtable} {
test "AOF rewrite of hash with $e encoding, $d data" {
r flushall
if {$e eq {ziplist}} {set len 10} else {set len 1000}
if {$e eq {listpack}} {set len 10} else {set len 1000}
for {set j 0} {$j < $len} {incr j} {
if {$d eq {string}} {
set data [randstring 0 16 alpha]
......
......@@ -313,10 +313,10 @@ start_server {tags {"keyspace"}} {
r config set zset-max-ziplist-entries $original_max
}
test {COPY basic usage for ziplist hash} {
test {COPY basic usage for listpack hash} {
r del hash1{t} newhash1{t}
r hset hash1{t} tmp 17179869184
assert_encoding ziplist hash1{t}
assert_encoding listpack hash1{t}
r copy hash1{t} newhash1{t}
set digest [debug_digest_value hash1{t}]
assert_equal $digest [debug_digest_value newhash1{t}]
......
......@@ -132,11 +132,11 @@ start_server {tags {"scan network"}} {
}
}
foreach enc {ziplist hashtable} {
foreach enc {listpack hashtable} {
test "HSCAN with encoding $enc" {
# Create the Hash
r del hash
if {$enc eq {ziplist}} {
if {$enc eq {listpack}} {
set count 30
} else {
set count 1000
......
......@@ -14,8 +14,8 @@ start_server {tags {"hash"}} {
list [r hlen smallhash]
} {8}
test {Is the small hash encoded with a ziplist?} {
assert_encoding ziplist smallhash
test {Is the small hash encoded with a listpack?} {
assert_encoding listpack smallhash
}
proc create_hash {key entries} {
......@@ -34,12 +34,15 @@ start_server {tags {"hash"}} {
return $res
}
foreach {type contents} "ziplist {{a 1} {b 2} {c 3}} hashtable {{a 1} {b 2} {[randstring 70 90 alpha] 3}}" {
foreach {type contents} "listpack {{a 1} {b 2} {c 3}} hashtable {{a 1} {b 2} {[randstring 70 90 alpha] 3}}" {
set original_max_value [lindex [r config get hash-max-ziplist-value] 1]
r config set hash-max-ziplist-value 10
create_hash myhash $contents
assert_encoding $type myhash
# coverage for objectComputeSize
assert_morethan [r memory usage myhash] 0
test "HRANDFIELD - $type" {
unset -nocomplain myhash
array set myhash {}
......@@ -87,7 +90,7 @@ start_server {tags {"hash"}} {
foreach {type contents} "
hashtable {{a 1} {b 2} {c 3} {d 4} {e 5} {6 f} {7 g} {8 h} {9 i} {[randstring 70 90 alpha] 10}}
ziplist {{a 1} {b 2} {c 3} {d 4} {e 5} {6 f} {7 g} {8 h} {9 i} {10 j}} " {
listpack {{a 1} {b 2} {c 3} {d 4} {e 5} {6 f} {7 g} {8 h} {9 i} {10 j}} " {
test "HRANDFIELD with <count> - $type" {
set original_max_value [lindex [r config get hash-max-ziplist-value] 1]
r config set hash-max-ziplist-value 10
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment