Unverified Commit 02fd76b9 authored by sundb's avatar sundb Committed by GitHub
Browse files

Replace all usage of ziplist with listpack for t_hash (#8887)



Part one of implementing #8702 (taking hashes first before other types)

## Description of the feature
1. Change ziplist encoded hash objects to listpack encoding.
2. Convert existing ziplists on RDB loading time. an O(n) operation.

## Rdb format changes
1. Add RDB_TYPE_HASH_LISTPACK rdb type.
2. Bump RDB_VERSION to 10

## Interface changes
1. New `hash-max-listpack-entries` config is an alias for `hash-max-ziplist-entries` (same with `hash-max-listpack-value`)
2. OBJECT ENCODING will return `listpack` instead of `ziplist`

## Listpack improvements:
1. Support direct insert, replace integer element (rather than convert back and forth from string)
3. Add more listpack capabilities to match the ziplist ones (like `lpFind`, `lpRandomPairs` and such)
4. Optimize element length fetching, avoid multiple calculations
5. Use inline to avoid function call overhead.

## Tests
1. Add a new test to the RDB load time conversion
2. Adding the listpack unit tests. (based on the one in ziplist.c)
3. Add a few "corrupt payload: fuzzer findings" tests, and slightly modify existing ones.
Co-authored-by: default avatarOran Agra <oran@redislabs.com>
parent cbda4929
...@@ -48,6 +48,7 @@ set ::all_tests { ...@@ -48,6 +48,7 @@ set ::all_tests {
integration/corrupt-dump integration/corrupt-dump
integration/corrupt-dump-fuzzer integration/corrupt-dump-fuzzer
integration/convert-zipmap-hash-on-load integration/convert-zipmap-hash-on-load
integration/convert-ziplist-hash-on-load
integration/logging integration/logging
integration/psync2 integration/psync2
integration/psync2-reg integration/psync2-reg
......
...@@ -127,10 +127,10 @@ start_server {tags {"aofrw external:skip"} overrides {aof-use-rdb-preamble no}} ...@@ -127,10 +127,10 @@ start_server {tags {"aofrw external:skip"} overrides {aof-use-rdb-preamble no}}
} }
foreach d {string int} { foreach d {string int} {
foreach e {ziplist hashtable} { foreach e {listpack hashtable} {
test "AOF rewrite of hash with $e encoding, $d data" { test "AOF rewrite of hash with $e encoding, $d data" {
r flushall r flushall
if {$e eq {ziplist}} {set len 10} else {set len 1000} if {$e eq {listpack}} {set len 10} else {set len 1000}
for {set j 0} {$j < $len} {incr j} { for {set j 0} {$j < $len} {incr j} {
if {$d eq {string}} { if {$d eq {string}} {
set data [randstring 0 16 alpha] set data [randstring 0 16 alpha]
......
...@@ -313,10 +313,10 @@ start_server {tags {"keyspace"}} { ...@@ -313,10 +313,10 @@ start_server {tags {"keyspace"}} {
r config set zset-max-ziplist-entries $original_max r config set zset-max-ziplist-entries $original_max
} }
test {COPY basic usage for ziplist hash} { test {COPY basic usage for listpack hash} {
r del hash1{t} newhash1{t} r del hash1{t} newhash1{t}
r hset hash1{t} tmp 17179869184 r hset hash1{t} tmp 17179869184
assert_encoding ziplist hash1{t} assert_encoding listpack hash1{t}
r copy hash1{t} newhash1{t} r copy hash1{t} newhash1{t}
set digest [debug_digest_value hash1{t}] set digest [debug_digest_value hash1{t}]
assert_equal $digest [debug_digest_value newhash1{t}] assert_equal $digest [debug_digest_value newhash1{t}]
......
...@@ -132,11 +132,11 @@ start_server {tags {"scan network"}} { ...@@ -132,11 +132,11 @@ start_server {tags {"scan network"}} {
} }
} }
foreach enc {ziplist hashtable} { foreach enc {listpack hashtable} {
test "HSCAN with encoding $enc" { test "HSCAN with encoding $enc" {
# Create the Hash # Create the Hash
r del hash r del hash
if {$enc eq {ziplist}} { if {$enc eq {listpack}} {
set count 30 set count 30
} else { } else {
set count 1000 set count 1000
......
...@@ -14,8 +14,8 @@ start_server {tags {"hash"}} { ...@@ -14,8 +14,8 @@ start_server {tags {"hash"}} {
list [r hlen smallhash] list [r hlen smallhash]
} {8} } {8}
test {Is the small hash encoded with a ziplist?} { test {Is the small hash encoded with a listpack?} {
assert_encoding ziplist smallhash assert_encoding listpack smallhash
} }
proc create_hash {key entries} { proc create_hash {key entries} {
...@@ -34,12 +34,15 @@ start_server {tags {"hash"}} { ...@@ -34,12 +34,15 @@ start_server {tags {"hash"}} {
return $res return $res
} }
foreach {type contents} "ziplist {{a 1} {b 2} {c 3}} hashtable {{a 1} {b 2} {[randstring 70 90 alpha] 3}}" { foreach {type contents} "listpack {{a 1} {b 2} {c 3}} hashtable {{a 1} {b 2} {[randstring 70 90 alpha] 3}}" {
set original_max_value [lindex [r config get hash-max-ziplist-value] 1] set original_max_value [lindex [r config get hash-max-ziplist-value] 1]
r config set hash-max-ziplist-value 10 r config set hash-max-ziplist-value 10
create_hash myhash $contents create_hash myhash $contents
assert_encoding $type myhash assert_encoding $type myhash
# coverage for objectComputeSize
assert_morethan [r memory usage myhash] 0
test "HRANDFIELD - $type" { test "HRANDFIELD - $type" {
unset -nocomplain myhash unset -nocomplain myhash
array set myhash {} array set myhash {}
...@@ -87,7 +90,7 @@ start_server {tags {"hash"}} { ...@@ -87,7 +90,7 @@ start_server {tags {"hash"}} {
foreach {type contents} " foreach {type contents} "
hashtable {{a 1} {b 2} {c 3} {d 4} {e 5} {6 f} {7 g} {8 h} {9 i} {[randstring 70 90 alpha] 10}} hashtable {{a 1} {b 2} {c 3} {d 4} {e 5} {6 f} {7 g} {8 h} {9 i} {[randstring 70 90 alpha] 10}}
ziplist {{a 1} {b 2} {c 3} {d 4} {e 5} {6 f} {7 g} {8 h} {9 i} {10 j}} " { listpack {{a 1} {b 2} {c 3} {d 4} {e 5} {6 f} {7 g} {8 h} {9 i} {10 j}} " {
test "HRANDFIELD with <count> - $type" { test "HRANDFIELD with <count> - $type" {
set original_max_value [lindex [r config get hash-max-ziplist-value] 1] set original_max_value [lindex [r config get hash-max-ziplist-value] 1]
r config set hash-max-ziplist-value 10 r config set hash-max-ziplist-value 10
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment