Unverified Commit 02fd76b9 authored by sundb's avatar sundb Committed by GitHub
Browse files

Replace all usage of ziplist with listpack for t_hash (#8887)



Part one of implementing #8702 (taking hashes first before other types)

## Description of the feature
1. Change ziplist encoded hash objects to listpack encoding.
2. Convert existing ziplists on RDB loading time. an O(n) operation.

## Rdb format changes
1. Add RDB_TYPE_HASH_LISTPACK rdb type.
2. Bump RDB_VERSION to 10

## Interface changes
1. New `hash-max-listpack-entries` config is an alias for `hash-max-ziplist-entries` (same with `hash-max-listpack-value`)
2. OBJECT ENCODING will return `listpack` instead of `ziplist`

## Listpack improvements:
1. Support direct insert, replace integer element (rather than convert back and forth from string)
3. Add more listpack capabilities to match the ziplist ones (like `lpFind`, `lpRandomPairs` and such)
4. Optimize element length fetching, avoid multiple calculations
5. Use inline to avoid function call overhead.

## Tests
1. Add a new test to the RDB load time conversion
2. Adding the listpack unit tests. (based on the one in ziplist.c)
3. Add a few "corrupt payload: fuzzer findings" tests, and slightly modify existing ones.
Co-authored-by: default avatarOran Agra <oran@redislabs.com>
parent cbda4929
...@@ -1704,8 +1704,8 @@ notify-keyspace-events "" ...@@ -1704,8 +1704,8 @@ notify-keyspace-events ""
# Hashes are encoded using a memory efficient data structure when they have a # Hashes are encoded using a memory efficient data structure when they have a
# small number of entries, and the biggest entry does not exceed a given # small number of entries, and the biggest entry does not exceed a given
# threshold. These thresholds can be configured using the following directives. # threshold. These thresholds can be configured using the following directives.
hash-max-ziplist-entries 512 hash-max-listpack-entries 512
hash-max-ziplist-value 64 hash-max-listpack-value 64
# Lists are also encoded in a special way to save a lot of space. # Lists are also encoded in a special way to save a lot of space.
# The number of entries allowed per internal list node can be specified # The number of entries allowed per internal list node can be specified
......
...@@ -1105,12 +1105,12 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) { ...@@ -1105,12 +1105,12 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) {
* *
* The function returns 0 on error, non-zero on success. */ * The function returns 0 on error, non-zero on success. */
static int rioWriteHashIteratorCursor(rio *r, hashTypeIterator *hi, int what) { static int rioWriteHashIteratorCursor(rio *r, hashTypeIterator *hi, int what) {
if (hi->encoding == OBJ_ENCODING_ZIPLIST) { if (hi->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *vstr = NULL; unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX; unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX; long long vll = LLONG_MAX;
hashTypeCurrentFromZiplist(hi, what, &vstr, &vlen, &vll); hashTypeCurrentFromListpack(hi, what, &vstr, &vlen, &vll);
if (vstr) if (vstr)
return rioWriteBulkString(r, (char*)vstr, vlen); return rioWriteBulkString(r, (char*)vstr, vlen);
else else
......
...@@ -2636,11 +2636,11 @@ standardConfig configs[] = { ...@@ -2636,11 +2636,11 @@ standardConfig configs[] = {
createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.maxmemory, 0, MEMORY_CONFIG, NULL, updateMaxmemory), createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.maxmemory, 0, MEMORY_CONFIG, NULL, updateMaxmemory),
/* Size_t configs */ /* Size_t configs */
createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_entries, 512, INTEGER_CONFIG, NULL, NULL), createSizeTConfig("hash-max-listpack-entries", "hash-max-ziplist-entries", MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_listpack_entries, 512, INTEGER_CONFIG, NULL, NULL),
createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, 512, INTEGER_CONFIG, NULL, NULL), createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, 512, INTEGER_CONFIG, NULL, NULL),
createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_entries, 128, INTEGER_CONFIG, NULL, NULL), createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_entries, 128, INTEGER_CONFIG, NULL, NULL),
createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.active_defrag_ignore_bytes, 100<<20, MEMORY_CONFIG, NULL, NULL), /* Default: don't defrag if frag overhead is below 100mb */ createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.active_defrag_ignore_bytes, 100<<20, MEMORY_CONFIG, NULL, NULL), /* Default: don't defrag if frag overhead is below 100mb */
createSizeTConfig("hash-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("hash-max-listpack-value", "hash-max-ziplist-value", MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_listpack_value, 64, MEMORY_CONFIG, NULL, NULL),
createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, 4096, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, 4096, MEMORY_CONFIG, NULL, NULL),
createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL),
createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, 3000, MEMORY_CONFIG, NULL, NULL), createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, 3000, MEMORY_CONFIG, NULL, NULL),
......
...@@ -935,7 +935,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { ...@@ -935,7 +935,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) {
while(intsetGet(o->ptr,pos++,&ll)) while(intsetGet(o->ptr,pos++,&ll))
listAddNodeTail(keys,createStringObjectFromLongLong(ll)); listAddNodeTail(keys,createStringObjectFromLongLong(ll));
cursor = 0; cursor = 0;
} else if (o->type == OBJ_HASH || o->type == OBJ_ZSET) { } else if (o->type == OBJ_ZSET) {
unsigned char *p = ziplistIndex(o->ptr,0); unsigned char *p = ziplistIndex(o->ptr,0);
unsigned char *vstr; unsigned char *vstr;
unsigned int vlen; unsigned int vlen;
...@@ -949,6 +949,18 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { ...@@ -949,6 +949,18 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) {
p = ziplistNext(o->ptr,p); p = ziplistNext(o->ptr,p);
} }
cursor = 0; cursor = 0;
} else if (o->type == OBJ_HASH) {
unsigned char *p = lpFirst(o->ptr);
unsigned char *vstr;
int64_t vlen;
unsigned char intbuf[LP_INTBUF_SIZE];
while(p) {
vstr = lpGet(p,&vlen,intbuf);
listAddNodeTail(keys, createStringObject((char*)vstr,vlen));
p = lpNext(o->ptr,p);
}
cursor = 0;
} else { } else {
serverPanic("Not handled encoding in SCAN."); serverPanic("Not handled encoding in SCAN.");
} }
......
...@@ -864,7 +864,7 @@ long defragKey(redisDb *db, dictEntry *de) { ...@@ -864,7 +864,7 @@ long defragKey(redisDb *db, dictEntry *de) {
serverPanic("Unknown sorted set encoding"); serverPanic("Unknown sorted set encoding");
} }
} else if (ob->type == OBJ_HASH) { } else if (ob->type == OBJ_HASH) {
if (ob->encoding == OBJ_ENCODING_ZIPLIST) { if (ob->encoding == OBJ_ENCODING_LISTPACK) {
if ((newzl = activeDefragAlloc(ob->ptr))) if ((newzl = activeDefragAlloc(ob->ptr)))
defragged++, ob->ptr = newzl; defragged++, ob->ptr = newzl;
} else if (ob->encoding == OBJ_ENCODING_HT) { } else if (ob->encoding == OBJ_ENCODING_HT) {
......
...@@ -55,6 +55,7 @@ ...@@ -55,6 +55,7 @@
#define LP_ENCODING_7BIT_UINT 0 #define LP_ENCODING_7BIT_UINT 0
#define LP_ENCODING_7BIT_UINT_MASK 0x80 #define LP_ENCODING_7BIT_UINT_MASK 0x80
#define LP_ENCODING_IS_7BIT_UINT(byte) (((byte)&LP_ENCODING_7BIT_UINT_MASK)==LP_ENCODING_7BIT_UINT) #define LP_ENCODING_IS_7BIT_UINT(byte) (((byte)&LP_ENCODING_7BIT_UINT_MASK)==LP_ENCODING_7BIT_UINT)
#define LP_ENCODING_7BIT_UINT_ENTRY_SIZE 2
#define LP_ENCODING_6BIT_STR 0x80 #define LP_ENCODING_6BIT_STR 0x80
#define LP_ENCODING_6BIT_STR_MASK 0xC0 #define LP_ENCODING_6BIT_STR_MASK 0xC0
...@@ -63,6 +64,7 @@ ...@@ -63,6 +64,7 @@
#define LP_ENCODING_13BIT_INT 0xC0 #define LP_ENCODING_13BIT_INT 0xC0
#define LP_ENCODING_13BIT_INT_MASK 0xE0 #define LP_ENCODING_13BIT_INT_MASK 0xE0
#define LP_ENCODING_IS_13BIT_INT(byte) (((byte)&LP_ENCODING_13BIT_INT_MASK)==LP_ENCODING_13BIT_INT) #define LP_ENCODING_IS_13BIT_INT(byte) (((byte)&LP_ENCODING_13BIT_INT_MASK)==LP_ENCODING_13BIT_INT)
#define LP_ENCODING_13BIT_INT_ENTRY_SIZE 3
#define LP_ENCODING_12BIT_STR 0xE0 #define LP_ENCODING_12BIT_STR 0xE0
#define LP_ENCODING_12BIT_STR_MASK 0xF0 #define LP_ENCODING_12BIT_STR_MASK 0xF0
...@@ -71,18 +73,22 @@ ...@@ -71,18 +73,22 @@
#define LP_ENCODING_16BIT_INT 0xF1 #define LP_ENCODING_16BIT_INT 0xF1
#define LP_ENCODING_16BIT_INT_MASK 0xFF #define LP_ENCODING_16BIT_INT_MASK 0xFF
#define LP_ENCODING_IS_16BIT_INT(byte) (((byte)&LP_ENCODING_16BIT_INT_MASK)==LP_ENCODING_16BIT_INT) #define LP_ENCODING_IS_16BIT_INT(byte) (((byte)&LP_ENCODING_16BIT_INT_MASK)==LP_ENCODING_16BIT_INT)
#define LP_ENCODING_16BIT_INT_ENTRY_SIZE 4
#define LP_ENCODING_24BIT_INT 0xF2 #define LP_ENCODING_24BIT_INT 0xF2
#define LP_ENCODING_24BIT_INT_MASK 0xFF #define LP_ENCODING_24BIT_INT_MASK 0xFF
#define LP_ENCODING_IS_24BIT_INT(byte) (((byte)&LP_ENCODING_24BIT_INT_MASK)==LP_ENCODING_24BIT_INT) #define LP_ENCODING_IS_24BIT_INT(byte) (((byte)&LP_ENCODING_24BIT_INT_MASK)==LP_ENCODING_24BIT_INT)
#define LP_ENCODING_24BIT_INT_ENTRY_SIZE 5
#define LP_ENCODING_32BIT_INT 0xF3 #define LP_ENCODING_32BIT_INT 0xF3
#define LP_ENCODING_32BIT_INT_MASK 0xFF #define LP_ENCODING_32BIT_INT_MASK 0xFF
#define LP_ENCODING_IS_32BIT_INT(byte) (((byte)&LP_ENCODING_32BIT_INT_MASK)==LP_ENCODING_32BIT_INT) #define LP_ENCODING_IS_32BIT_INT(byte) (((byte)&LP_ENCODING_32BIT_INT_MASK)==LP_ENCODING_32BIT_INT)
#define LP_ENCODING_32BIT_INT_ENTRY_SIZE 6
#define LP_ENCODING_64BIT_INT 0xF4 #define LP_ENCODING_64BIT_INT 0xF4
#define LP_ENCODING_64BIT_INT_MASK 0xFF #define LP_ENCODING_64BIT_INT_MASK 0xFF
#define LP_ENCODING_IS_64BIT_INT(byte) (((byte)&LP_ENCODING_64BIT_INT_MASK)==LP_ENCODING_64BIT_INT) #define LP_ENCODING_IS_64BIT_INT(byte) (((byte)&LP_ENCODING_64BIT_INT_MASK)==LP_ENCODING_64BIT_INT)
#define LP_ENCODING_64BIT_INT_ENTRY_SIZE 10
#define LP_ENCODING_32BIT_STR 0xF0 #define LP_ENCODING_32BIT_STR 0xF0
#define LP_ENCODING_32BIT_STR_MASK 0xFF #define LP_ENCODING_32BIT_STR_MASK 0xFF
...@@ -246,6 +252,58 @@ unsigned char* lpShrinkToFit(unsigned char *lp) { ...@@ -246,6 +252,58 @@ unsigned char* lpShrinkToFit(unsigned char *lp) {
} }
} }
/* Stores the integer encoded representation of 'v' in the 'intenc' buffer. */
static inline void lpEncodeIntegerGetType(int64_t v, unsigned char *intenc, uint64_t *enclen) {
if (v >= 0 && v <= 127) {
/* Single byte 0-127 integer. */
intenc[0] = v;
*enclen = 1;
} else if (v >= -4096 && v <= 4095) {
/* 13 bit integer. */
if (v < 0) v = ((int64_t)1<<13)+v;
intenc[0] = (v>>8)|LP_ENCODING_13BIT_INT;
intenc[1] = v&0xff;
*enclen = 2;
} else if (v >= -32768 && v <= 32767) {
/* 16 bit integer. */
if (v < 0) v = ((int64_t)1<<16)+v;
intenc[0] = LP_ENCODING_16BIT_INT;
intenc[1] = v&0xff;
intenc[2] = v>>8;
*enclen = 3;
} else if (v >= -8388608 && v <= 8388607) {
/* 24 bit integer. */
if (v < 0) v = ((int64_t)1<<24)+v;
intenc[0] = LP_ENCODING_24BIT_INT;
intenc[1] = v&0xff;
intenc[2] = (v>>8)&0xff;
intenc[3] = v>>16;
*enclen = 4;
} else if (v >= -2147483648 && v <= 2147483647) {
/* 32 bit integer. */
if (v < 0) v = ((int64_t)1<<32)+v;
intenc[0] = LP_ENCODING_32BIT_INT;
intenc[1] = v&0xff;
intenc[2] = (v>>8)&0xff;
intenc[3] = (v>>16)&0xff;
intenc[4] = v>>24;
*enclen = 5;
} else {
/* 64 bit integer. */
uint64_t uv = v;
intenc[0] = LP_ENCODING_64BIT_INT;
intenc[1] = uv&0xff;
intenc[2] = (uv>>8)&0xff;
intenc[3] = (uv>>16)&0xff;
intenc[4] = (uv>>24)&0xff;
intenc[5] = (uv>>32)&0xff;
intenc[6] = (uv>>40)&0xff;
intenc[7] = (uv>>48)&0xff;
intenc[8] = uv>>56;
*enclen = 9;
}
}
/* Given an element 'ele' of size 'size', determine if the element can be /* Given an element 'ele' of size 'size', determine if the element can be
* represented inside the listpack encoded as integer, and returns * represented inside the listpack encoded as integer, and returns
* LP_ENCODING_INT if so. Otherwise returns LP_ENCODING_STR if no integer * LP_ENCODING_INT if so. Otherwise returns LP_ENCODING_STR if no integer
...@@ -257,57 +315,10 @@ unsigned char* lpShrinkToFit(unsigned char *lp) { ...@@ -257,57 +315,10 @@ unsigned char* lpShrinkToFit(unsigned char *lp) {
* Regardless of the returned encoding, 'enclen' is populated by reference to * Regardless of the returned encoding, 'enclen' is populated by reference to
* the number of bytes that the string or integer encoded element will require * the number of bytes that the string or integer encoded element will require
* in order to be represented. */ * in order to be represented. */
int lpEncodeGetType(unsigned char *ele, uint32_t size, unsigned char *intenc, uint64_t *enclen) { static inline int lpEncodeGetType(unsigned char *ele, uint32_t size, unsigned char *intenc, uint64_t *enclen) {
int64_t v; int64_t v;
if (lpStringToInt64((const char*)ele, size, &v)) { if (lpStringToInt64((const char*)ele, size, &v)) {
if (v >= 0 && v <= 127) { lpEncodeIntegerGetType(v, intenc, enclen);
/* Single byte 0-127 integer. */
intenc[0] = v;
*enclen = 1;
} else if (v >= -4096 && v <= 4095) {
/* 13 bit integer. */
if (v < 0) v = ((int64_t)1<<13)+v;
intenc[0] = (v>>8)|LP_ENCODING_13BIT_INT;
intenc[1] = v&0xff;
*enclen = 2;
} else if (v >= -32768 && v <= 32767) {
/* 16 bit integer. */
if (v < 0) v = ((int64_t)1<<16)+v;
intenc[0] = LP_ENCODING_16BIT_INT;
intenc[1] = v&0xff;
intenc[2] = v>>8;
*enclen = 3;
} else if (v >= -8388608 && v <= 8388607) {
/* 24 bit integer. */
if (v < 0) v = ((int64_t)1<<24)+v;
intenc[0] = LP_ENCODING_24BIT_INT;
intenc[1] = v&0xff;
intenc[2] = (v>>8)&0xff;
intenc[3] = v>>16;
*enclen = 4;
} else if (v >= -2147483648 && v <= 2147483647) {
/* 32 bit integer. */
if (v < 0) v = ((int64_t)1<<32)+v;
intenc[0] = LP_ENCODING_32BIT_INT;
intenc[1] = v&0xff;
intenc[2] = (v>>8)&0xff;
intenc[3] = (v>>16)&0xff;
intenc[4] = v>>24;
*enclen = 5;
} else {
/* 64 bit integer. */
uint64_t uv = v;
intenc[0] = LP_ENCODING_64BIT_INT;
intenc[1] = uv&0xff;
intenc[2] = (uv>>8)&0xff;
intenc[3] = (uv>>16)&0xff;
intenc[4] = (uv>>24)&0xff;
intenc[5] = (uv>>32)&0xff;
intenc[6] = (uv>>40)&0xff;
intenc[7] = (uv>>48)&0xff;
intenc[8] = uv>>56;
*enclen = 9;
}
return LP_ENCODING_INT; return LP_ENCODING_INT;
} else { } else {
if (size < 64) *enclen = 1+size; if (size < 64) *enclen = 1+size;
...@@ -322,7 +333,7 @@ int lpEncodeGetType(unsigned char *ele, uint32_t size, unsigned char *intenc, ui ...@@ -322,7 +333,7 @@ int lpEncodeGetType(unsigned char *ele, uint32_t size, unsigned char *intenc, ui
* The function returns the number of bytes used to encode it, from * The function returns the number of bytes used to encode it, from
* 1 to 5. If 'buf' is NULL the function just returns the number of bytes * 1 to 5. If 'buf' is NULL the function just returns the number of bytes
* needed in order to encode the backlen. */ * needed in order to encode the backlen. */
unsigned long lpEncodeBacklen(unsigned char *buf, uint64_t l) { static inline unsigned long lpEncodeBacklen(unsigned char *buf, uint64_t l) {
if (l <= 127) { if (l <= 127) {
if (buf) buf[0] = l; if (buf) buf[0] = l;
return 1; return 1;
...@@ -361,7 +372,7 @@ unsigned long lpEncodeBacklen(unsigned char *buf, uint64_t l) { ...@@ -361,7 +372,7 @@ unsigned long lpEncodeBacklen(unsigned char *buf, uint64_t l) {
/* Decode the backlen and returns it. If the encoding looks invalid (more than /* Decode the backlen and returns it. If the encoding looks invalid (more than
* 5 bytes are used), UINT64_MAX is returned to report the problem. */ * 5 bytes are used), UINT64_MAX is returned to report the problem. */
uint64_t lpDecodeBacklen(unsigned char *p) { static inline uint64_t lpDecodeBacklen(unsigned char *p) {
uint64_t val = 0; uint64_t val = 0;
uint64_t shift = 0; uint64_t shift = 0;
do { do {
...@@ -378,7 +389,7 @@ uint64_t lpDecodeBacklen(unsigned char *p) { ...@@ -378,7 +389,7 @@ uint64_t lpDecodeBacklen(unsigned char *p) {
* buffer 's'. The function should be called with 'buf' having always enough * buffer 's'. The function should be called with 'buf' having always enough
* space for encoding the string. This is done by calling lpEncodeGetType() * space for encoding the string. This is done by calling lpEncodeGetType()
* before calling this function. */ * before calling this function. */
void lpEncodeString(unsigned char *buf, unsigned char *s, uint32_t len) { static inline void lpEncodeString(unsigned char *buf, unsigned char *s, uint32_t len) {
if (len < 64) { if (len < 64) {
buf[0] = len | LP_ENCODING_6BIT_STR; buf[0] = len | LP_ENCODING_6BIT_STR;
memcpy(buf+1,s,len); memcpy(buf+1,s,len);
...@@ -403,7 +414,7 @@ void lpEncodeString(unsigned char *buf, unsigned char *s, uint32_t len) { ...@@ -403,7 +414,7 @@ void lpEncodeString(unsigned char *buf, unsigned char *s, uint32_t len) {
* str), so should only be called when we know 'p' was already validated by * str), so should only be called when we know 'p' was already validated by
* lpCurrentEncodedSizeBytes or ASSERT_INTEGRITY_LEN (possibly since 'p' is * lpCurrentEncodedSizeBytes or ASSERT_INTEGRITY_LEN (possibly since 'p' is
* a return value of another function that validated its return. */ * a return value of another function that validated its return. */
uint32_t lpCurrentEncodedSizeUnsafe(unsigned char *p) { static inline uint32_t lpCurrentEncodedSizeUnsafe(unsigned char *p) {
if (LP_ENCODING_IS_7BIT_UINT(p[0])) return 1; if (LP_ENCODING_IS_7BIT_UINT(p[0])) return 1;
if (LP_ENCODING_IS_6BIT_STR(p[0])) return 1+LP_ENCODING_6BIT_STR_LEN(p); if (LP_ENCODING_IS_6BIT_STR(p[0])) return 1+LP_ENCODING_6BIT_STR_LEN(p);
if (LP_ENCODING_IS_13BIT_INT(p[0])) return 2; if (LP_ENCODING_IS_13BIT_INT(p[0])) return 2;
...@@ -421,7 +432,7 @@ uint32_t lpCurrentEncodedSizeUnsafe(unsigned char *p) { ...@@ -421,7 +432,7 @@ uint32_t lpCurrentEncodedSizeUnsafe(unsigned char *p) {
* This includes just the encoding byte, and the bytes needed to encode the length * This includes just the encoding byte, and the bytes needed to encode the length
* of the element (excluding the element data itself) * of the element (excluding the element data itself)
* If the element encoding is wrong then 0 is returned. */ * If the element encoding is wrong then 0 is returned. */
uint32_t lpCurrentEncodedSizeBytes(unsigned char *p) { static inline uint32_t lpCurrentEncodedSizeBytes(unsigned char *p) {
if (LP_ENCODING_IS_7BIT_UINT(p[0])) return 1; if (LP_ENCODING_IS_7BIT_UINT(p[0])) return 1;
if (LP_ENCODING_IS_6BIT_STR(p[0])) return 1; if (LP_ENCODING_IS_6BIT_STR(p[0])) return 1;
if (LP_ENCODING_IS_13BIT_INT(p[0])) return 1; if (LP_ENCODING_IS_13BIT_INT(p[0])) return 1;
...@@ -492,7 +503,7 @@ unsigned char *lpLast(unsigned char *lp) { ...@@ -492,7 +503,7 @@ unsigned char *lpLast(unsigned char *lp) {
* needed. As a side effect of calling this function, the listpack header * needed. As a side effect of calling this function, the listpack header
* could be modified, because if the count is found to be already within * could be modified, because if the count is found to be already within
* the 'numele' header field range, the new value is set. */ * the 'numele' header field range, the new value is set. */
uint32_t lpLength(unsigned char *lp) { unsigned long lpLength(unsigned char *lp) {
uint32_t numele = lpGetNumElements(lp); uint32_t numele = lpGetNumElements(lp);
if (numele != LP_HDR_NUMELE_UNKNOWN) return numele; if (numele != LP_HDR_NUMELE_UNKNOWN) return numele;
...@@ -534,6 +545,10 @@ uint32_t lpLength(unsigned char *lp) { ...@@ -534,6 +545,10 @@ uint32_t lpLength(unsigned char *lp) {
* by the function, than to pass a buffer (and convert it back to a number) * by the function, than to pass a buffer (and convert it back to a number)
* is of course useless. * is of course useless.
* *
* If 'entry_size' is not NULL, *entry_size is set to the entry length of the
* listpack element pointed by 'p'. This includes the encoding bytes, length
* bytes, the element data itself, and the backlen bytes.
*
* If the function is called against a badly encoded ziplist, so that there * If the function is called against a badly encoded ziplist, so that there
* is no valid way to parse it, the function returns like if there was an * is no valid way to parse it, the function returns like if there was an
* integer encoded with value 12345678900000000 + <unrecognized byte>, this may * integer encoded with value 12345678900000000 + <unrecognized byte>, this may
...@@ -545,7 +560,7 @@ uint32_t lpLength(unsigned char *lp) { ...@@ -545,7 +560,7 @@ uint32_t lpLength(unsigned char *lp) {
* assumed to be valid, so that would be a very high API cost. However a function * assumed to be valid, so that would be a very high API cost. However a function
* in order to check the integrity of the listpack at load time is provided, * in order to check the integrity of the listpack at load time is provided,
* check lpIsValid(). */ * check lpIsValid(). */
unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) { static inline unsigned char *lpGetWithSize(unsigned char *p, int64_t *count, unsigned char *intbuf, uint64_t *entry_size) {
int64_t val; int64_t val;
uint64_t uval, negstart, negmax; uint64_t uval, negstart, negmax;
...@@ -554,24 +569,29 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) { ...@@ -554,24 +569,29 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) {
negstart = UINT64_MAX; /* 7 bit ints are always positive. */ negstart = UINT64_MAX; /* 7 bit ints are always positive. */
negmax = 0; negmax = 0;
uval = p[0] & 0x7f; uval = p[0] & 0x7f;
if (entry_size) *entry_size = LP_ENCODING_7BIT_UINT_ENTRY_SIZE;
} else if (LP_ENCODING_IS_6BIT_STR(p[0])) { } else if (LP_ENCODING_IS_6BIT_STR(p[0])) {
*count = LP_ENCODING_6BIT_STR_LEN(p); *count = LP_ENCODING_6BIT_STR_LEN(p);
if (entry_size) *entry_size = 1 + *count + lpEncodeBacklen(NULL, *count + 1);
return p+1; return p+1;
} else if (LP_ENCODING_IS_13BIT_INT(p[0])) { } else if (LP_ENCODING_IS_13BIT_INT(p[0])) {
uval = ((p[0]&0x1f)<<8) | p[1]; uval = ((p[0]&0x1f)<<8) | p[1];
negstart = (uint64_t)1<<12; negstart = (uint64_t)1<<12;
negmax = 8191; negmax = 8191;
if (entry_size) *entry_size = LP_ENCODING_13BIT_INT_ENTRY_SIZE;
} else if (LP_ENCODING_IS_16BIT_INT(p[0])) { } else if (LP_ENCODING_IS_16BIT_INT(p[0])) {
uval = (uint64_t)p[1] | uval = (uint64_t)p[1] |
(uint64_t)p[2]<<8; (uint64_t)p[2]<<8;
negstart = (uint64_t)1<<15; negstart = (uint64_t)1<<15;
negmax = UINT16_MAX; negmax = UINT16_MAX;
if (entry_size) *entry_size = LP_ENCODING_16BIT_INT_ENTRY_SIZE;
} else if (LP_ENCODING_IS_24BIT_INT(p[0])) { } else if (LP_ENCODING_IS_24BIT_INT(p[0])) {
uval = (uint64_t)p[1] | uval = (uint64_t)p[1] |
(uint64_t)p[2]<<8 | (uint64_t)p[2]<<8 |
(uint64_t)p[3]<<16; (uint64_t)p[3]<<16;
negstart = (uint64_t)1<<23; negstart = (uint64_t)1<<23;
negmax = UINT32_MAX>>8; negmax = UINT32_MAX>>8;
if (entry_size) *entry_size = LP_ENCODING_24BIT_INT_ENTRY_SIZE;
} else if (LP_ENCODING_IS_32BIT_INT(p[0])) { } else if (LP_ENCODING_IS_32BIT_INT(p[0])) {
uval = (uint64_t)p[1] | uval = (uint64_t)p[1] |
(uint64_t)p[2]<<8 | (uint64_t)p[2]<<8 |
...@@ -579,6 +599,7 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) { ...@@ -579,6 +599,7 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) {
(uint64_t)p[4]<<24; (uint64_t)p[4]<<24;
negstart = (uint64_t)1<<31; negstart = (uint64_t)1<<31;
negmax = UINT32_MAX; negmax = UINT32_MAX;
if (entry_size) *entry_size = LP_ENCODING_32BIT_INT_ENTRY_SIZE;
} else if (LP_ENCODING_IS_64BIT_INT(p[0])) { } else if (LP_ENCODING_IS_64BIT_INT(p[0])) {
uval = (uint64_t)p[1] | uval = (uint64_t)p[1] |
(uint64_t)p[2]<<8 | (uint64_t)p[2]<<8 |
...@@ -590,11 +611,14 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) { ...@@ -590,11 +611,14 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) {
(uint64_t)p[8]<<56; (uint64_t)p[8]<<56;
negstart = (uint64_t)1<<63; negstart = (uint64_t)1<<63;
negmax = UINT64_MAX; negmax = UINT64_MAX;
if (entry_size) *entry_size = LP_ENCODING_64BIT_INT_ENTRY_SIZE;
} else if (LP_ENCODING_IS_12BIT_STR(p[0])) { } else if (LP_ENCODING_IS_12BIT_STR(p[0])) {
*count = LP_ENCODING_12BIT_STR_LEN(p); *count = LP_ENCODING_12BIT_STR_LEN(p);
if (entry_size) *entry_size = 2 + *count + lpEncodeBacklen(NULL, *count + 2);
return p+2; return p+2;
} else if (LP_ENCODING_IS_32BIT_STR(p[0])) { } else if (LP_ENCODING_IS_32BIT_STR(p[0])) {
*count = LP_ENCODING_32BIT_STR_LEN(p); *count = LP_ENCODING_32BIT_STR_LEN(p);
if (entry_size) *entry_size = 5 + *count + lpEncodeBacklen(NULL, *count + 5);
return p+5; return p+5;
} else { } else {
uval = 12345678900000000ULL + p[0]; uval = 12345678900000000ULL + p[0];
...@@ -626,17 +650,105 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) { ...@@ -626,17 +650,105 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) {
} }
} }
/* Insert, delete or replace the specified element 'ele' of length 'len' at unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) {
* the specified position 'p', with 'p' being a listpack element pointer return lpGetWithSize(p, count, intbuf, NULL);
* obtained with lpFirst(), lpLast(), lpNext(), lpPrev() or lpSeek(). }
/* This is just a wrapper to lpGet() that is able to get entry value directly.
* When the function returns NULL, it populates the integer value by reference in 'lval'.
* Otherwise if the element is encoded as a string a pointer to the string (pointing
* inside the listpack itself) is returned, and 'slen' is set to the length of the
* string. */
unsigned char *lpGetValue(unsigned char *p, unsigned int *slen, long long *lval) {
unsigned char *vstr;
int64_t ele_len;
vstr = lpGet(p, &ele_len, NULL);
if (vstr) {
*slen = ele_len;
} else {
*lval = ele_len;
}
return vstr;
}
/* Find pointer to the entry equal to the specified entry. Skip 'skip' entries
* between every comparison. Returns NULL when the field could not be found. */
unsigned char *lpFind(unsigned char *lp, unsigned char *p, unsigned char *s,
uint32_t slen, unsigned int skip) {
int skipcnt = 0;
unsigned char vencoding = 0;
unsigned char *value;
int64_t ll, vll;
uint64_t entry_size = 123456789; /* initialized to avoid warning. */
uint32_t lp_bytes = lpBytes(lp);
assert(p);
while (p) {
if (skipcnt == 0) {
value = lpGetWithSize(p, &ll, NULL, &entry_size);
if (value) {
if (slen == ll && memcmp(value, s, slen) == 0) {
return p;
}
} else {
/* Find out if the searched field can be encoded. Note that
* we do it only the first time, once done vencoding is set
* to non-zero and vll is set to the integer value. */
if (vencoding == 0) {
/* If the entry can be encoded as integer we set it to
* 1, else set it to UCHAR_MAX, so that we don't retry
* again the next time. */
if (slen >= 32 || slen == 0 || !lpStringToInt64((const char*)s, slen, &vll)) {
vencoding = UCHAR_MAX;
} else {
vencoding = 1;
}
}
/* Compare current entry with specified entry, do it only
* if vencoding != UCHAR_MAX because if there is no encoding
* possible for the field it can't be a valid integer. */
if (vencoding != UCHAR_MAX && ll == vll) {
return p;
}
}
/* Reset skip count */
skipcnt = skip;
p += entry_size;
} else {
/* Skip entry */
skipcnt--;
/* Move to next entry, avoid use `lpNext` due to `ASSERT_INTEGRITY` in
* `lpNext` will call `lpBytes`, will cause performance degradation */
p = lpSkip(p);
}
assert(p >= lp + LP_HDR_SIZE && p < lp + lp_bytes);
if (p[0] == LP_EOF) break;
}
return NULL;
}
/* Insert, delete or replace the specified string element 'elestr' of length
* 'size' or integer element 'eleint' at the specified position 'p', with 'p'
* being a listpack element pointer obtained with lpFirst(), lpLast(), lpNext(),
* lpPrev() or lpSeek().
* *
* The element is inserted before, after, or replaces the element pointed * The element is inserted before, after, or replaces the element pointed
* by 'p' depending on the 'where' argument, that can be LP_BEFORE, LP_AFTER * by 'p' depending on the 'where' argument, that can be LP_BEFORE, LP_AFTER
* or LP_REPLACE. * or LP_REPLACE.
* *
* If 'ele' is set to NULL, the function removes the element pointed by 'p' * If both 'elestr' and `eleint` are NULL, the function removes the element
* instead of inserting one. * pointed by 'p' instead of inserting one.
* * If `eleint` is non-NULL, 'size' is the length of 'eleint', the function insert
* or replace with a 64 bit integer, which is stored in the 'eleint' buffer.
* If 'elestr` is non-NULL, 'size' is the length of 'elestr', the function insert
* or replace with a string, which is stored in the 'elestr' buffer.
*
* Returns NULL on out of memory or when the listpack total length would exceed * Returns NULL on out of memory or when the listpack total length would exceed
* the max allowed size of 2^32-1, otherwise the new pointer to the listpack * the max allowed size of 2^32-1, otherwise the new pointer to the listpack
* holding the new element is returned (and the old pointer passed is no longer * holding the new element is returned (and the old pointer passed is no longer
...@@ -646,19 +758,22 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) { ...@@ -646,19 +758,22 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) {
* to the address of the element just added, so that it will be possible to * to the address of the element just added, so that it will be possible to
* continue an interaction with lpNext() and lpPrev(). * continue an interaction with lpNext() and lpPrev().
* *
* For deletion operations ('ele' set to NULL) 'newp' is set to the next * For deletion operations (both 'elestr' and 'eleint' set to NULL) 'newp' is
* element, on the right of the deleted one, or to NULL if the deleted element * set to the next element, on the right of the deleted one, or to NULL if the
* was the last one. */ * deleted element was the last one. */
unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, unsigned char *p, int where, unsigned char **newp) { unsigned char *lpInsert(unsigned char *lp, unsigned char *elestr, unsigned char *eleint,
uint32_t size, unsigned char *p, int where, unsigned char **newp)
{
unsigned char intenc[LP_MAX_INT_ENCODING_LEN]; unsigned char intenc[LP_MAX_INT_ENCODING_LEN];
unsigned char backlen[LP_MAX_BACKLEN_SIZE]; unsigned char backlen[LP_MAX_BACKLEN_SIZE];
uint64_t enclen; /* The length of the encoded element. */ uint64_t enclen; /* The length of the encoded element. */
int delete = (elestr == NULL && eleint == NULL);
/* An element pointer set to NULL means deletion, which is conceptually /* when deletion, it is conceptually replacing the element with a
* replacing the element with a zero-length element. So whatever we * zero-length element. So whatever we get passed as 'where', set
* get passed as 'where', set it to LP_REPLACE. */ * it to LP_REPLACE. */
if (ele == NULL) where = LP_REPLACE; if (delete) where = LP_REPLACE;
/* If we need to insert after the current element, we just jump to the /* If we need to insert after the current element, we just jump to the
* next element (that could be the EOF one) and handle the case of * next element (that could be the EOF one) and handle the case of
...@@ -674,17 +789,21 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un ...@@ -674,17 +789,21 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un
* address again after a reallocation. */ * address again after a reallocation. */
unsigned long poff = p-lp; unsigned long poff = p-lp;
/* Calling lpEncodeGetType() results into the encoded version of the
* element to be stored into 'intenc' in case it is representable as
* an integer: in that case, the function returns LP_ENCODING_INT.
* Otherwise if LP_ENCODING_STR is returned, we'll have to call
* lpEncodeString() to actually write the encoded string on place later.
*
* Whatever the returned encoding is, 'enclen' is populated with the
* length of the encoded element. */
int enctype; int enctype;
if (ele) { if (elestr) {
enctype = lpEncodeGetType(ele,size,intenc,&enclen); /* Calling lpEncodeGetType() results into the encoded version of the
* element to be stored into 'intenc' in case it is representable as
* an integer: in that case, the function returns LP_ENCODING_INT.
* Otherwise if LP_ENCODING_STR is returned, we'll have to call
* lpEncodeString() to actually write the encoded string on place later.
*
* Whatever the returned encoding is, 'enclen' is populated with the
* length of the encoded element. */
enctype = lpEncodeGetType(elestr,size,intenc,&enclen);
if (enctype == LP_ENCODING_INT) eleint = intenc;
} else if (eleint) {
enctype = LP_ENCODING_INT;
enclen = size; /* 'size' is the length of the encoded integer element. */
} else { } else {
enctype = -1; enctype = -1;
enclen = 0; enclen = 0;
...@@ -693,7 +812,7 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un ...@@ -693,7 +812,7 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un
/* We need to also encode the backward-parsable length of the element /* We need to also encode the backward-parsable length of the element
* and append it to the end: this allows to traverse the listpack from * and append it to the end: this allows to traverse the listpack from
* the end to the start. */ * the end to the start. */
unsigned long backlen_size = ele ? lpEncodeBacklen(backlen,enclen) : 0; unsigned long backlen_size = (!delete) ? lpEncodeBacklen(backlen,enclen) : 0;
uint64_t old_listpack_bytes = lpGetTotalBytes(lp); uint64_t old_listpack_bytes = lpGetTotalBytes(lp);
uint32_t replaced_len = 0; uint32_t replaced_len = 0;
if (where == LP_REPLACE) { if (where == LP_REPLACE) {
...@@ -743,13 +862,13 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un ...@@ -743,13 +862,13 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un
*newp = dst; *newp = dst;
/* In case of deletion, set 'newp' to NULL if the next element is /* In case of deletion, set 'newp' to NULL if the next element is
* the EOF element. */ * the EOF element. */
if (!ele && dst[0] == LP_EOF) *newp = NULL; if (delete && dst[0] == LP_EOF) *newp = NULL;
} }
if (ele) { if (!delete) {
if (enctype == LP_ENCODING_INT) { if (enctype == LP_ENCODING_INT) {
memcpy(dst,intenc,enclen); memcpy(dst,eleint,enclen);
} else { } else {
lpEncodeString(dst,ele,size); lpEncodeString(dst,elestr,size);
} }
dst += enclen; dst += enclen;
memcpy(dst,backlen,backlen_size); memcpy(dst,backlen,backlen_size);
...@@ -757,10 +876,10 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un ...@@ -757,10 +876,10 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un
} }
/* Update header. */ /* Update header. */
if (where != LP_REPLACE || ele == NULL) { if (where != LP_REPLACE || delete) {
uint32_t num_elements = lpGetNumElements(lp); uint32_t num_elements = lpGetNumElements(lp);
if (num_elements != LP_HDR_NUMELE_UNKNOWN) { if (num_elements != LP_HDR_NUMELE_UNKNOWN) {
if (ele) if (!delete)
lpSetNumElements(lp,num_elements+1); lpSetNumElements(lp,num_elements+1);
else else
lpSetNumElements(lp,num_elements-1); lpSetNumElements(lp,num_elements-1);
...@@ -790,13 +909,59 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un ...@@ -790,13 +909,59 @@ unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, un
return lp; return lp;
} }
/* This is just a wrapper for lpInsert() to directly use a 64 bit integer
* instead of a string. */
unsigned char *lpInsertInteger(unsigned char *lp, long long lval, unsigned char *p, int where, unsigned char **newp) {
uint64_t enclen; /* The length of the encoded element. */
unsigned char intenc[LP_MAX_INT_ENCODING_LEN];
lpEncodeIntegerGetType(lval, intenc, &enclen);
return lpInsert(lp, NULL, intenc, enclen, p, where, newp);
}
/* Append the specified element 's' of length 'slen' at the head of the listpack. */
unsigned char *lpPrepend(unsigned char *lp, unsigned char *s, uint32_t slen) {
unsigned char *p = lpFirst(lp);
if (!p) return lpAppend(lp, s, slen);
return lpInsert(lp, s, NULL, slen, p, LP_BEFORE, NULL);
}
/* Append the specified integer element 'lval' at the head of the listpack. */
unsigned char *lpPrependInteger(unsigned char *lp, long long lval) {
unsigned char *p = lpFirst(lp);
if (!p) return lpAppendInteger(lp, lval);
return lpInsertInteger(lp, lval, p, LP_BEFORE, NULL);
}
/* Append the specified element 'ele' of length 'len' at the end of the /* Append the specified element 'ele' of length 'len' at the end of the
* listpack. It is implemented in terms of lpInsert(), so the return value is * listpack. It is implemented in terms of lpInsert(), so the return value is
* the same as lpInsert(). */ * the same as lpInsert(). */
unsigned char *lpAppend(unsigned char *lp, unsigned char *ele, uint32_t size) { unsigned char *lpAppend(unsigned char *lp, unsigned char *ele, uint32_t size) {
uint64_t listpack_bytes = lpGetTotalBytes(lp); uint64_t listpack_bytes = lpGetTotalBytes(lp);
unsigned char *eofptr = lp + listpack_bytes - 1; unsigned char *eofptr = lp + listpack_bytes - 1;
return lpInsert(lp,ele,size,eofptr,LP_BEFORE,NULL); return lpInsert(lp,ele,NULL,size,eofptr,LP_BEFORE,NULL);
}
/* Append the specified integer element 'lval' at the end of the listpack. */
unsigned char *lpAppendInteger(unsigned char *lp, long long lval) {
uint64_t listpack_bytes = lpGetTotalBytes(lp);
unsigned char *eofptr = lp + listpack_bytes - 1;
return lpInsertInteger(lp, lval, eofptr, LP_BEFORE, NULL);
}
/* This is just a wrapper for lpInsert() to directly use a string to replace
* the current element. The function returns the new listpack as return
* value, and also updates the current cursor by updating '*p'. */
unsigned char *lpReplace(unsigned char *lp, unsigned char **p, unsigned char *s, uint32_t slen) {
return lpInsert(lp, s, NULL, slen, *p, LP_REPLACE, p);
}
/* This is just a wrapper for lpInsertInteger() to directly use a 64 bit integer
* instead of a string to replace the current element. The function returns
* the new listpack as return value, and also updates the current cursor
* by updating '*p'. */
unsigned char *lpReplaceInteger(unsigned char *lp, unsigned char **p, long long lval) {
return lpInsertInteger(lp, lval, *p, LP_REPLACE, p);
} }
/* Remove the element pointed by 'p', and return the resulting listpack. /* Remove the element pointed by 'p', and return the resulting listpack.
...@@ -804,11 +969,11 @@ unsigned char *lpAppend(unsigned char *lp, unsigned char *ele, uint32_t size) { ...@@ -804,11 +969,11 @@ unsigned char *lpAppend(unsigned char *lp, unsigned char *ele, uint32_t size) {
* deleted one) is returned by reference. If the deleted element was the * deleted one) is returned by reference. If the deleted element was the
* last one, '*newp' is set to NULL. */ * last one, '*newp' is set to NULL. */
unsigned char *lpDelete(unsigned char *lp, unsigned char *p, unsigned char **newp) { unsigned char *lpDelete(unsigned char *lp, unsigned char *p, unsigned char **newp) {
return lpInsert(lp,NULL,0,p,LP_REPLACE,newp); return lpInsert(lp,NULL,NULL,0,p,LP_REPLACE,newp);
} }
/* Return the total number of bytes the listpack is composed of. */ /* Return the total number of bytes the listpack is composed of. */
uint32_t lpBytes(unsigned char *lp) { size_t lpBytes(unsigned char *lp) {
return lpGetTotalBytes(lp); return lpGetTotalBytes(lp);
} }
...@@ -927,7 +1092,8 @@ static inline void lpAssertValidEntry(unsigned char* lp, size_t lpbytes, unsigne ...@@ -927,7 +1092,8 @@ static inline void lpAssertValidEntry(unsigned char* lp, size_t lpbytes, unsigne
/* Validate the integrity of the data structure. /* Validate the integrity of the data structure.
* when `deep` is 0, only the integrity of the header is validated. * when `deep` is 0, only the integrity of the header is validated.
* when `deep` is 1, we scan all the entries one by one. */ * when `deep` is 1, we scan all the entries one by one. */
int lpValidateIntegrity(unsigned char *lp, size_t size, int deep){ int lpValidateIntegrity(unsigned char *lp, size_t size, int deep,
listpackValidateEntryCB entry_cb, void *cb_userdata) {
/* Check that we can actually read the header. (and EOF) */ /* Check that we can actually read the header. (and EOF) */
if (size < LP_HDR_SIZE + 1) if (size < LP_HDR_SIZE + 1)
return 0; return 0;
...@@ -948,8 +1114,17 @@ int lpValidateIntegrity(unsigned char *lp, size_t size, int deep){ ...@@ -948,8 +1114,17 @@ int lpValidateIntegrity(unsigned char *lp, size_t size, int deep){
uint32_t count = 0; uint32_t count = 0;
unsigned char *p = lp + LP_HDR_SIZE; unsigned char *p = lp + LP_HDR_SIZE;
while(p && p[0] != LP_EOF) { while(p && p[0] != LP_EOF) {
unsigned char *prev = p;
/* Validate this entry and move to the next entry in advance
* to avoid callback crash due to corrupt listpack. */
if (!lpValidateNext(lp, &p, bytes)) if (!lpValidateNext(lp, &p, bytes))
return 0; return 0;
/* Optionally let the caller validate the entry too. */
if (entry_cb && !entry_cb(prev, cb_userdata))
return 0;
count++; count++;
} }
...@@ -960,3 +1135,826 @@ int lpValidateIntegrity(unsigned char *lp, size_t size, int deep){ ...@@ -960,3 +1135,826 @@ int lpValidateIntegrity(unsigned char *lp, size_t size, int deep){
return 1; return 1;
} }
unsigned int lpCompare(unsigned char *p, unsigned char *s, uint32_t slen) {
unsigned char buf[LP_INTBUF_SIZE];
unsigned char *value;
int64_t sz;
if (p[0] == LP_EOF) return 0;
value = lpGet(p, &sz, buf);
return (slen == sz) && memcmp(value,s,slen) == 0;
}
/* uint compare for qsort */
static int uintCompare(const void *a, const void *b) {
return (*(unsigned int *) a - *(unsigned int *) b);
}
/* Helper method to store a string into from val or lval into dest */
static inline void lpSaveValue(unsigned char *val, unsigned int len, int64_t lval, listpackEntry *dest) {
dest->sval = val;
dest->slen = len;
dest->lval = lval;
}
/* Randomly select a pair of key and value.
* total_count is a pre-computed length/2 of the listpack (to avoid calls to lpLength)
* 'key' and 'val' are used to store the result key value pair.
* 'val' can be NULL if the value is not needed. */
void lpRandomPair(unsigned char *lp, unsigned long total_count, listpackEntry *key, listpackEntry *val) {
unsigned char *p;
/* Avoid div by zero on corrupt listpack */
assert(total_count);
/* Generate even numbers, because listpack saved K-V pair */
int r = (rand() % total_count) * 2;
assert((p = lpSeek(lp, r)));
key->sval = lpGetValue(p, &(key->slen), &(key->lval));
if (!val)
return;
assert((p = lpNext(lp, p)));
val->sval = lpGetValue(p, &(val->slen), &(val->lval));
}
/* Randomly select count of key value pairs and store into 'keys' and
* 'vals' args. The order of the picked entries is random, and the selections
* are non-unique (repetitions are possible).
* The 'vals' arg can be NULL in which case we skip these. */
void lpRandomPairs(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals) {
unsigned char *p, *key, *value;
unsigned int klen = 0, vlen = 0;
long long klval = 0, vlval = 0;
/* Notice: the index member must be first due to the use in uintCompare */
typedef struct {
unsigned int index;
unsigned int order;
} rand_pick;
rand_pick *picks = zmalloc(sizeof(rand_pick)*count);
unsigned int total_size = lpLength(lp)/2;
/* Avoid div by zero on corrupt listpack */
assert(total_size);
/* create a pool of random indexes (some may be duplicate). */
for (unsigned int i = 0; i < count; i++) {
picks[i].index = (rand() % total_size) * 2; /* Generate even indexes */
/* keep track of the order we picked them */
picks[i].order = i;
}
/* sort by indexes. */
qsort(picks, count, sizeof(rand_pick), uintCompare);
/* fetch the elements form the listpack into a output array respecting the original order. */
unsigned int lpindex = picks[0].index, pickindex = 0;
p = lpSeek(lp, lpindex);
while (p && pickindex < count) {
key = lpGetValue(p, &klen, &klval);
assert((p = lpNext(lp, p)));
value = lpGetValue(p, &vlen, &vlval);
while (pickindex < count && lpindex == picks[pickindex].index) {
int storeorder = picks[pickindex].order;
lpSaveValue(key, klen, klval, &keys[storeorder]);
if (vals)
lpSaveValue(value, vlen, vlval, &vals[storeorder]);
pickindex++;
}
lpindex += 2;
p = lpNext(lp, p);
}
zfree(picks);
}
/* Randomly select count of key value pairs and store into 'keys' and
* 'vals' args. The selections are unique (no repetitions), and the order of
* the picked entries is NOT-random.
* The 'vals' arg can be NULL in which case we skip these.
* The return value is the number of items picked which can be lower than the
* requested count if the listpack doesn't hold enough pairs. */
unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals) {
unsigned char *p, *key;
unsigned int klen = 0;
long long klval = 0;
unsigned int total_size = lpLength(lp)/2;
unsigned int index = 0;
if (count > total_size)
count = total_size;
/* To only iterate once, every time we try to pick a member, the probability
* we pick it is the quotient of the count left we want to pick and the
* count still we haven't visited in the dict, this way, we could make every
* member be equally picked.*/
p = lpFirst(lp);
unsigned int picked = 0, remaining = count;
while (picked < count && p) {
double randomDouble = ((double)rand()) / RAND_MAX;
double threshold = ((double)remaining) / (total_size - index);
if (randomDouble <= threshold) {
key = lpGetValue(p, &klen, &klval);
lpSaveValue(key, klen, klval, &keys[picked]);
assert((p = lpNext(lp, p)));
if (vals) {
key = lpGetValue(p, &klen, &klval);
lpSaveValue(key, klen, klval, &vals[picked]);
}
remaining--;
picked++;
} else {
assert((p = lpNext(lp, p)));
}
p = lpNext(lp, p);
index++;
}
return picked;
}
#ifdef REDIS_TEST
#include <sys/time.h>
#include "adlist.h"
#include "sds.h"
#define UNUSED(x) (void)(x)
#define TEST(name) printf("test — %s\n", name);
char *mixlist[] = {"hello", "foo", "quux", "1024"};
char *intlist[] = {"4294967296", "-100", "100", "128000",
"non integer", "much much longer non integer"};
static unsigned char *createList() {
unsigned char *lp = lpNew(0);
lp = lpAppend(lp, (unsigned char*)mixlist[1], strlen(mixlist[1]));
lp = lpAppend(lp, (unsigned char*)mixlist[2], strlen(mixlist[2]));
lp = lpPrepend(lp, (unsigned char*)mixlist[0], strlen(mixlist[0]));
lp = lpAppend(lp, (unsigned char*)mixlist[3], strlen(mixlist[3]));
return lp;
}
static unsigned char *createIntList() {
unsigned char *lp = lpNew(0);
lp = lpAppend(lp, (unsigned char*)intlist[2], strlen(intlist[2]));
lp = lpAppend(lp, (unsigned char*)intlist[3], strlen(intlist[3]));
lp = lpPrepend(lp, (unsigned char*)intlist[1], strlen(intlist[1]));
lp = lpPrepend(lp, (unsigned char*)intlist[0], strlen(intlist[0]));
lp = lpAppend(lp, (unsigned char*)intlist[4], strlen(intlist[4]));
lp = lpAppend(lp, (unsigned char*)intlist[5], strlen(intlist[5]));
return lp;
}
static long long usec(void) {
struct timeval tv;
gettimeofday(&tv, NULL);
return (((long long)tv.tv_sec)*1000000)+tv.tv_usec;
}
static void stress(int pos, int num, int maxsize, int dnum) {
int i, j, k;
unsigned char *lp;
char posstr[2][5] = { "HEAD", "TAIL" };
long long start;
for (i = 0; i < maxsize; i+=dnum) {
lp = lpNew(0);
for (j = 0; j < i; j++) {
lp = lpAppend(lp, (unsigned char*)"quux", 4);
}
/* Do num times a push+pop from pos */
start = usec();
for (k = 0; k < num; k++) {
if (pos == 0) {
lp = lpPrepend(lp, (unsigned char*)"quux", 4);
} else {
lp = lpAppend(lp, (unsigned char*)"quux", 4);
}
lp = lpDelete(lp, lpFirst(lp), NULL);
}
printf("List size: %8d, bytes: %8zu, %dx push+pop (%s): %6lld usec\n",
i, lpBytes(lp), num, posstr[pos], usec()-start);
lpFree(lp);
}
}
static unsigned char *pop(unsigned char *lp, int where) {
unsigned char *p, *vstr;
int64_t vlen;
p = lpSeek(lp, where == 0 ? 0 : -1);
vstr = lpGet(p, &vlen, NULL);
if (where == 0)
printf("Pop head: ");
else
printf("Pop tail: ");
if (vstr) {
if (vlen && fwrite(vstr, vlen, 1, stdout) == 0) perror("fwrite");
} else {
printf("%lld", (long long)vlen);
}
printf("\n");
return lpDelete(lp, p, &p);
}
static int randstring(char *target, unsigned int min, unsigned int max) {
int p = 0;
int len = min+rand()%(max-min+1);
int minval, maxval;
switch(rand() % 3) {
case 0:
minval = 0;
maxval = 255;
break;
case 1:
minval = 48;
maxval = 122;
break;
case 2:
minval = 48;
maxval = 52;
break;
default:
assert(NULL);
}
while(p < len)
target[p++] = minval+rand()%(maxval-minval+1);
return len;
}
static void verifyEntry(unsigned char *p, unsigned char *s, size_t slen) {
assert(lpCompare(p, s, slen));
}
static int lpValidation(unsigned char *p, void *userdata) {
UNUSED(p);
int ret;
long *count = userdata;
ret = lpCompare(p, (unsigned char *)mixlist[*count], strlen(mixlist[*count]));
(*count)++;
return ret;
}
int listpackTest(int argc, char *argv[], int accurate) {
UNUSED(argc);
UNUSED(argv);
UNUSED(accurate);
int i;
unsigned char *lp, *p, *vstr;
int64_t vlen;
unsigned char intbuf[LP_INTBUF_SIZE];
TEST("Create int list") {
lp = createIntList();
assert(lpLength(lp) == 6);
lpFree(lp);
}
TEST("Create list") {
lp = createList();
assert(lpLength(lp) == 4);
lpFree(lp);
}
TEST("Test lpPrepend") {
lp = lpNew(0);
lp = lpPrepend(lp, (unsigned char*)"abc", 3);
lp = lpPrepend(lp, (unsigned char*)"1024", 4);
verifyEntry(lpSeek(lp, 0), (unsigned char*)"1024", 4);
verifyEntry(lpSeek(lp, 1), (unsigned char*)"abc", 3);
lpFree(lp);
}
TEST("Test lpPrependInteger") {
lp = lpNew(0);
lp = lpPrependInteger(lp, 127);
lp = lpPrependInteger(lp, 4095);
lp = lpPrependInteger(lp, 32767);
lp = lpPrependInteger(lp, 8388607);
lp = lpPrependInteger(lp, 2147483647);
lp = lpPrependInteger(lp, 9223372036854775807);
verifyEntry(lpSeek(lp, 0), (unsigned char*)"9223372036854775807", 19);
verifyEntry(lpSeek(lp, -1), (unsigned char*)"127", 3);
lpFree(lp);
}
TEST("Get element at index") {
lp = createList();
verifyEntry(lpSeek(lp, 0), (unsigned char*)"hello", 5);
verifyEntry(lpSeek(lp, 3), (unsigned char*)"1024", 4);
verifyEntry(lpSeek(lp, -1), (unsigned char*)"1024", 4);
verifyEntry(lpSeek(lp, -4), (unsigned char*)"hello", 5);
assert(lpSeek(lp, 4) == NULL);
assert(lpSeek(lp, -5) == NULL);
lpFree(lp);
}
TEST("Pop list") {
lp = createList();
lp = pop(lp, 1);
lp = pop(lp, 0);
lp = pop(lp, 1);
lp = pop(lp, 1);
lpFree(lp);
}
TEST("Get element at index") {
lp = createList();
verifyEntry(lpSeek(lp, 0), (unsigned char*)"hello", 5);
verifyEntry(lpSeek(lp, 3), (unsigned char*)"1024", 4);
verifyEntry(lpSeek(lp, -1), (unsigned char*)"1024", 4);
verifyEntry(lpSeek(lp, -4), (unsigned char*)"hello", 5);
assert(lpSeek(lp, 4) == NULL);
assert(lpSeek(lp, -5) == NULL);
lpFree(lp);
}
TEST("Iterate list from 0 to end") {
lp = createList();
p = lpFirst(lp);
i = 0;
while (p) {
verifyEntry(p, (unsigned char*)mixlist[i], strlen(mixlist[i]));
p = lpNext(lp, p);
i++;
}
lpFree(lp);
}
TEST("Iterate list from 1 to end") {
lp = createList();
i = 1;
p = lpSeek(lp, i);
while (p) {
verifyEntry(p, (unsigned char*)mixlist[i], strlen(mixlist[i]));
p = lpNext(lp, p);
i++;
}
lpFree(lp);
}
TEST("Iterate list from 2 to end") {
lp = createList();
i = 2;
p = lpSeek(lp, i);
while (p) {
verifyEntry(p, (unsigned char*)mixlist[i], strlen(mixlist[i]));
p = lpNext(lp, p);
i++;
}
lpFree(lp);
}
TEST("Iterate from back to front") {
lp = createList();
p = lpLast(lp);
i = 3;
while (p) {
verifyEntry(p, (unsigned char*)mixlist[i], strlen(mixlist[i]));
p = lpPrev(lp, p);
i--;
}
lpFree(lp);
}
TEST("Iterate from back to front, deleting all items") {
lp = createList();
p = lpLast(lp);
i = 3;
while ((p = lpLast(lp))) {
verifyEntry(p, (unsigned char*)mixlist[i], strlen(mixlist[i]));
lp = lpDelete(lp, p, &p);
assert(p == NULL);
i--;
}
lpFree(lp);
}
TEST("Delete foo while iterating") {
lp = createList();
p = lpFirst(lp);
while (p) {
if (lpCompare(p, (unsigned char*)"foo", 3)) {
lp = lpDelete(lp, p, &p);
} else {
p = lpNext(lp, p);
}
}
lpFree(lp);
}
TEST("Replace with same size") {
lp = createList(); /* "hello", "foo", "quux", "1024" */
unsigned char *orig_lp = lp;
p = lpSeek(lp, 0);
lp = lpReplace(lp, &p, (unsigned char*)"zoink", 5);
p = lpSeek(lp, 3);
lp = lpReplace(lp, &p, (unsigned char*)"y", 1);
p = lpSeek(lp, 1);
lp = lpReplace(lp, &p, (unsigned char*)"65536", 5);
p = lpSeek(lp, 0);
assert(!memcmp((char*)p,
"\x85zoink\x06"
"\xf2\x00\x00\x01\x04" /* 65536 as int24 */
"\x84quux\05" "\x81y\x02" "\xff",
22));
assert(lp == orig_lp); /* no reallocations have happened */
lpFree(lp);
}
TEST("Replace with different size") {
lp = createList(); /* "hello", "foo", "quux", "1024" */
p = lpSeek(lp, 1);
lp = lpReplace(lp, &p, (unsigned char*)"squirrel", 8);
p = lpSeek(lp, 0);
assert(!strncmp((char*)p,
"\x85hello\x06" "\x88squirrel\x09" "\x84quux\x05"
"\xc4\x00\x02" "\xff",
27));
lpFree(lp);
}
TEST("Regression test for >255 byte strings") {
char v1[257] = {0}, v2[257] = {0};
memset(v1,'x',256);
memset(v2,'y',256);
lp = lpNew(0);
lp = lpAppend(lp, (unsigned char*)v1 ,strlen(v1));
lp = lpAppend(lp, (unsigned char*)v2 ,strlen(v2));
/* Pop values again and compare their value. */
p = lpFirst(lp);
vstr = lpGet(p, &vlen, NULL);
assert(strncmp(v1, (char*)vstr, vlen) == 0);
p = lpSeek(lp, 1);
vstr = lpGet(p, &vlen, NULL);
assert(strncmp(v2, (char*)vstr, vlen) == 0);
lpFree(lp);
}
TEST("Create long list and check indices") {
lp = lpNew(0);
char buf[32];
int i,len;
for (i = 0; i < 1000; i++) {
len = sprintf(buf, "%d", i);
lp = lpAppend(lp, (unsigned char*)buf, len);
}
for (i = 0; i < 1000; i++) {
p = lpSeek(lp, i);
vstr = lpGet(p, &vlen, NULL);
assert(i == vlen);
p = lpSeek(lp, -i-1);
vstr = lpGet(p, &vlen, NULL);
assert(999-i == vlen);
}
lpFree(lp);
}
TEST("Compare strings with listpack entries") {
lp = createList();
p = lpSeek(lp,0);
assert(lpCompare(p,(unsigned char*)"hello",5));
assert(!lpCompare(p,(unsigned char*)"hella",5));
p = lpSeek(lp,3);
assert(lpCompare(p,(unsigned char*)"1024",4));
assert(!lpCompare(p,(unsigned char*)"1025",4));
lpFree(lp);
}
TEST("Random pair with one element") {
listpackEntry key, val;
unsigned char *lp = lpNew(0);
lp = lpAppend(lp, (unsigned char*)"abc", 3);
lp = lpAppend(lp, (unsigned char*)"123", 3);
lpRandomPair(lp, 1, &key, &val);
assert(memcmp(key.sval, "abc", key.slen) == 0);
assert(val.lval == 123);
lpFree(lp);
}
TEST("Random pair with many elements") {
listpackEntry key, val;
unsigned char *lp = lpNew(0);
lp = lpAppend(lp, (unsigned char*)"abc", 3);
lp = lpAppend(lp, (unsigned char*)"123", 3);
lp = lpAppend(lp, (unsigned char*)"456", 3);
lp = lpAppend(lp, (unsigned char*)"def", 3);
lpRandomPair(lp, 2, &key, &val);
if (key.sval) {
assert(!memcmp(key.sval, "abc", key.slen));
assert(key.slen == 3);
assert(val.lval == 123);
}
if (!key.sval) {
assert(key.lval == 456);
assert(!memcmp(val.sval, "def", val.slen));
}
lpFree(lp);
}
TEST("Random pairs with one element") {
int count = 5;
unsigned char *lp = lpNew(0);
listpackEntry *keys = zmalloc(sizeof(listpackEntry) * count);
listpackEntry *vals = zmalloc(sizeof(listpackEntry) * count);
lp = lpAppend(lp, (unsigned char*)"abc", 3);
lp = lpAppend(lp, (unsigned char*)"123", 3);
lpRandomPairs(lp, count, keys, vals);
assert(memcmp(keys[4].sval, "abc", keys[4].slen) == 0);
assert(vals[4].lval == 123);
zfree(keys);
zfree(vals);
lpFree(lp);
}
TEST("Random pairs with many elements") {
int count = 5;
lp = lpNew(0);
listpackEntry *keys = zmalloc(sizeof(listpackEntry) * count);
listpackEntry *vals = zmalloc(sizeof(listpackEntry) * count);
lp = lpAppend(lp, (unsigned char*)"abc", 3);
lp = lpAppend(lp, (unsigned char*)"123", 3);
lp = lpAppend(lp, (unsigned char*)"456", 3);
lp = lpAppend(lp, (unsigned char*)"def", 3);
lpRandomPairs(lp, count, keys, vals);
for (int i = 0; i < count; i++) {
if (keys[i].sval) {
assert(!memcmp(keys[i].sval, "abc", keys[i].slen));
assert(keys[i].slen == 3);
assert(vals[i].lval == 123);
}
if (!keys[i].sval) {
assert(keys[i].lval == 456);
assert(!memcmp(vals[i].sval, "def", vals[i].slen));
}
}
zfree(keys);
zfree(vals);
lpFree(lp);
}
TEST("Random pairs unique with one element") {
unsigned picked;
int count = 5;
lp = lpNew(0);
listpackEntry *keys = zmalloc(sizeof(listpackEntry) * count);
listpackEntry *vals = zmalloc(sizeof(listpackEntry) * count);
lp = lpAppend(lp, (unsigned char*)"abc", 3);
lp = lpAppend(lp, (unsigned char*)"123", 3);
picked = lpRandomPairsUnique(lp, count, keys, vals);
assert(picked == 1);
assert(memcmp(keys[0].sval, "abc", keys[0].slen) == 0);
assert(vals[0].lval == 123);
zfree(keys);
zfree(vals);
lpFree(lp);
}
TEST("Random pairs unique with many elements") {
unsigned picked;
int count = 5;
lp = lpNew(0);
listpackEntry *keys = zmalloc(sizeof(listpackEntry) * count);
listpackEntry *vals = zmalloc(sizeof(listpackEntry) * count);
lp = lpAppend(lp, (unsigned char*)"abc", 3);
lp = lpAppend(lp, (unsigned char*)"123", 3);
lp = lpAppend(lp, (unsigned char*)"456", 3);
lp = lpAppend(lp, (unsigned char*)"def", 3);
picked = lpRandomPairsUnique(lp, count, keys, vals);
assert(picked == 2);
for (int i = 0; i < 2; i++) {
if (keys[i].sval) {
assert(!memcmp(keys[i].sval, "abc", keys[i].slen));
assert(keys[i].slen == 3);
assert(vals[i].lval == 123);
}
if (!keys[i].sval) {
assert(keys[i].lval == 456);
assert(!memcmp(vals[i].sval, "def", vals[i].slen));
}
}
zfree(keys);
zfree(vals);
lpFree(lp);
}
TEST("push various encodings") {
lp = lpNew(0);
/* Push integer encode element using lpAppend */
lp = lpAppend(lp, (unsigned char*)"127", 3);
assert(LP_ENCODING_IS_7BIT_UINT(lpLast(lp)[0]));
lp = lpAppend(lp, (unsigned char*)"4095", 4);
assert(LP_ENCODING_IS_13BIT_INT(lpLast(lp)[0]));
lp = lpAppend(lp, (unsigned char*)"32767", 5);
assert(LP_ENCODING_IS_16BIT_INT(lpLast(lp)[0]));
lp = lpAppend(lp, (unsigned char*)"8388607", 7);
assert(LP_ENCODING_IS_24BIT_INT(lpLast(lp)[0]));
lp = lpAppend(lp, (unsigned char*)"2147483647", 10);
assert(LP_ENCODING_IS_32BIT_INT(lpLast(lp)[0]));
lp = lpAppend(lp, (unsigned char*)"9223372036854775807", 19);
assert(LP_ENCODING_IS_64BIT_INT(lpLast(lp)[0]));
/* Push integer encode element using lpAppendInteger */
lp = lpAppendInteger(lp, 127);
assert(LP_ENCODING_IS_7BIT_UINT(lpLast(lp)[0]));
verifyEntry(lpLast(lp), (unsigned char*)"127", 3);
lp = lpAppendInteger(lp, 4095);
verifyEntry(lpLast(lp), (unsigned char*)"4095", 4);
assert(LP_ENCODING_IS_13BIT_INT(lpLast(lp)[0]));
lp = lpAppendInteger(lp, 32767);
verifyEntry(lpLast(lp), (unsigned char*)"32767", 5);
assert(LP_ENCODING_IS_16BIT_INT(lpLast(lp)[0]));
lp = lpAppendInteger(lp, 8388607);
verifyEntry(lpLast(lp), (unsigned char*)"8388607", 7);
assert(LP_ENCODING_IS_24BIT_INT(lpLast(lp)[0]));
lp = lpAppendInteger(lp, 2147483647);
verifyEntry(lpLast(lp), (unsigned char*)"2147483647", 10);
assert(LP_ENCODING_IS_32BIT_INT(lpLast(lp)[0]));
lp = lpAppendInteger(lp, 9223372036854775807);
verifyEntry(lpLast(lp), (unsigned char*)"9223372036854775807", 19);
assert(LP_ENCODING_IS_64BIT_INT(lpLast(lp)[0]));
/* string encode */
unsigned char *str = zmalloc(65535);
memset(str, 0, 65535);
lp = lpAppend(lp, (unsigned char*)str, 63);
assert(LP_ENCODING_IS_6BIT_STR(lpLast(lp)[0]));
lp = lpAppend(lp, (unsigned char*)str, 4095);
assert(LP_ENCODING_IS_12BIT_STR(lpLast(lp)[0]));
lp = lpAppend(lp, (unsigned char*)str, 65535);
assert(LP_ENCODING_IS_32BIT_STR(lpLast(lp)[0]));
zfree(str);
lpFree(lp);
}
TEST("Test lpFind") {
lp = createList();
assert(lpFind(lp, lpFirst(lp), (unsigned char*)"abc", 3, 0) == NULL);
verifyEntry(lpFind(lp, lpFirst(lp), (unsigned char*)"hello", 5, 0), (unsigned char*)"hello", 5);
verifyEntry(lpFind(lp, lpFirst(lp), (unsigned char*)"1024", 4, 0), (unsigned char*)"1024", 4);
lpFree(lp);
}
TEST("Test lpValidateIntegrity") {
lp = createList();
long count = 0;
assert(lpValidateIntegrity(lp, lpBytes(lp), 1, lpValidation, &count) == 1);
lpFree(lp);
}
TEST("Stress with random payloads of different encoding") {
unsigned long long start = usec();
int i,j,len,where;
unsigned char *p;
char buf[1024];
int buflen;
list *ref;
listNode *refnode;
int iteration = accurate ? 20000 : 20;
for (i = 0; i < iteration; i++) {
lp = lpNew(0);
ref = listCreate();
listSetFreeMethod(ref,(void (*)(void*))sdsfree);
len = rand() % 256;
/* Create lists */
for (j = 0; j < len; j++) {
where = (rand() & 1) ? 0 : 1;
if (rand() % 2) {
buflen = randstring(buf,1,sizeof(buf)-1);
} else {
switch(rand() % 3) {
case 0:
buflen = sprintf(buf,"%lld",(0LL + rand()) >> 20);
break;
case 1:
buflen = sprintf(buf,"%lld",(0LL + rand()));
break;
case 2:
buflen = sprintf(buf,"%lld",(0LL + rand()) << 20);
break;
default:
assert(NULL);
}
}
/* Add to listpack */
if (where == 0) {
lp = lpPrepend(lp, (unsigned char*)buf, buflen);
} else {
lp = lpAppend(lp, (unsigned char*)buf, buflen);
}
/* Add to reference list */
if (where == 0) {
listAddNodeHead(ref,sdsnewlen(buf, buflen));
} else if (where == 1) {
listAddNodeTail(ref,sdsnewlen(buf, buflen));
} else {
assert(NULL);
}
}
assert(listLength(ref) == lpLength(lp));
for (j = 0; j < len; j++) {
/* Naive way to get elements, but similar to the stresser
* executed from the Tcl test suite. */
p = lpSeek(lp,j);
refnode = listIndex(ref,j);
vstr = lpGet(p, &vlen, intbuf);
assert(memcmp(vstr,listNodeValue(refnode),vlen) == 0);
}
lpFree(lp);
listRelease(ref);
}
printf("Done. usec=%lld\n\n", usec()-start);
}
TEST("Stress with variable listpack size") {
unsigned long long start = usec();
int maxsize = accurate ? 16384 : 16;
stress(0,100000,maxsize,256);
stress(1,100000,maxsize,256);
printf("Done. usec=%lld\n\n", usec()-start);
}
/* Benchmarks */
{
int iteration = accurate ? 100000 : 100;
lp = lpNew(0);
TEST("Benchmark lpAppend") {
unsigned long long start = usec();
for (int i=0; i<iteration; i++) {
char buf[4096] = "asdf";
lp = lpAppend(lp, (unsigned char*)buf, 4);
lp = lpAppend(lp, (unsigned char*)buf, 40);
lp = lpAppend(lp, (unsigned char*)buf, 400);
lp = lpAppend(lp, (unsigned char*)buf, 4000);
lp = lpAppend(lp, (unsigned char*)"1", 1);
lp = lpAppend(lp, (unsigned char*)"10", 2);
lp = lpAppend(lp, (unsigned char*)"100", 3);
lp = lpAppend(lp, (unsigned char*)"1000", 4);
lp = lpAppend(lp, (unsigned char*)"10000", 5);
lp = lpAppend(lp, (unsigned char*)"100000", 6);
}
printf("Done. usec=%lld\n", usec()-start);
}
TEST("Benchmark lpFind string") {
unsigned long long start = usec();
for (int i = 0; i < 2000; i++) {
unsigned char *fptr = lpFirst(lp);
fptr = lpFind(lp, fptr, (unsigned char*)"nothing", 7, 1);
}
printf("Done. usec=%lld\n", usec()-start);
}
TEST("Benchmark lpFind number") {
unsigned long long start = usec();
for (int i = 0; i < 2000; i++) {
unsigned char *fptr = lpFirst(lp);
fptr = lpFind(lp, fptr, (unsigned char*)"99999", 5, 1);
}
printf("Done. usec=%lld\n", usec()-start);
}
TEST("Benchmark lpSeek") {
unsigned long long start = usec();
for (int i = 0; i < 2000; i++) {
lpSeek(lp, 99999);
}
printf("Done. usec=%lld\n", usec()-start);
}
TEST("Benchmark lpValidateIntegrity") {
unsigned long long start = usec();
for (int i = 0; i < 2000; i++) {
lpValidateIntegrity(lp, lpBytes(lp), 1, NULL, NULL);
}
printf("Done. usec=%lld\n", usec()-start);
}
lpFree(lp);
}
return 0;
}
#endif
...@@ -45,22 +45,47 @@ ...@@ -45,22 +45,47 @@
#define LP_AFTER 1 #define LP_AFTER 1
#define LP_REPLACE 2 #define LP_REPLACE 2
/* Each entry in the listpack is either a string or an integer. */
typedef struct {
/* When string is used, it is provided with the length (slen). */
unsigned char *sval;
uint32_t slen;
/* When integer is used, 'sval' is NULL, and lval holds the value. */
long long lval;
} listpackEntry;
unsigned char *lpNew(size_t capacity); unsigned char *lpNew(size_t capacity);
void lpFree(unsigned char *lp); void lpFree(unsigned char *lp);
unsigned char* lpShrinkToFit(unsigned char *lp); unsigned char* lpShrinkToFit(unsigned char *lp);
unsigned char *lpInsert(unsigned char *lp, unsigned char *ele, uint32_t size, unsigned char *p, int where, unsigned char **newp); unsigned char *lpPrepend(unsigned char *lp, unsigned char *s, uint32_t slen);
unsigned char *lpAppend(unsigned char *lp, unsigned char *ele, uint32_t size); unsigned char *lpPrependInteger(unsigned char *lp, long long lval);
unsigned char *lpAppend(unsigned char *lp, unsigned char *s, uint32_t slen);
unsigned char *lpAppendInteger(unsigned char *lp, long long lval);
unsigned char *lpReplace(unsigned char *lp, unsigned char **p, unsigned char *s, uint32_t slen);
unsigned char *lpReplaceInteger(unsigned char *lp, unsigned char **p, long long lval);
unsigned char *lpDelete(unsigned char *lp, unsigned char *p, unsigned char **newp); unsigned char *lpDelete(unsigned char *lp, unsigned char *p, unsigned char **newp);
uint32_t lpLength(unsigned char *lp); unsigned long lpLength(unsigned char *lp);
unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf); unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf);
unsigned char *lpGetValue(unsigned char *p, unsigned int *slen, long long *lval);
unsigned char *lpFind(unsigned char *lp, unsigned char *p, unsigned char *s, uint32_t slen, unsigned int skip);
unsigned char *lpFirst(unsigned char *lp); unsigned char *lpFirst(unsigned char *lp);
unsigned char *lpLast(unsigned char *lp); unsigned char *lpLast(unsigned char *lp);
unsigned char *lpNext(unsigned char *lp, unsigned char *p); unsigned char *lpNext(unsigned char *lp, unsigned char *p);
unsigned char *lpPrev(unsigned char *lp, unsigned char *p); unsigned char *lpPrev(unsigned char *lp, unsigned char *p);
uint32_t lpBytes(unsigned char *lp); size_t lpBytes(unsigned char *lp);
unsigned char *lpSeek(unsigned char *lp, long index); unsigned char *lpSeek(unsigned char *lp, long index);
int lpValidateIntegrity(unsigned char *lp, size_t size, int deep); typedef int (*listpackValidateEntryCB)(unsigned char *p, void *userdata);
int lpValidateIntegrity(unsigned char *lp, size_t size, int deep,
listpackValidateEntryCB entry_cb, void *cb_userdata);
unsigned char *lpValidateFirst(unsigned char *lp); unsigned char *lpValidateFirst(unsigned char *lp);
int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes); int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes);
unsigned int lpCompare(unsigned char *p, unsigned char *s, uint32_t slen);
void lpRandomPair(unsigned char *lp, unsigned long total_count, listpackEntry *key, listpackEntry *val);
void lpRandomPairs(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals);
unsigned int lpRandomPairsUnique(unsigned char *lp, unsigned int count, listpackEntry *keys, listpackEntry *vals);
#ifdef REDIS_TEST
int listpackTest(int argc, char *argv[], int accurate);
#endif
#endif #endif
...@@ -7990,7 +7990,7 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc ...@@ -7990,7 +7990,7 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc
cursor->cursor = 1; cursor->cursor = 1;
cursor->done = 1; cursor->done = 1;
ret = 0; ret = 0;
} else if (o->type == OBJ_HASH || o->type == OBJ_ZSET) { } else if (o->type == OBJ_ZSET) {
unsigned char *p = ziplistIndex(o->ptr,0); unsigned char *p = ziplistIndex(o->ptr,0);
unsigned char *vstr; unsigned char *vstr;
unsigned int vlen; unsigned int vlen;
...@@ -8013,6 +8013,25 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc ...@@ -8013,6 +8013,25 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc
cursor->cursor = 1; cursor->cursor = 1;
cursor->done = 1; cursor->done = 1;
ret = 0; ret = 0;
} else if (o->type == OBJ_HASH) {
unsigned char *p = lpFirst(o->ptr);
unsigned char *vstr;
int64_t vlen;
unsigned char intbuf[LP_INTBUF_SIZE];
while(p) {
vstr = lpGet(p,&vlen,intbuf);
robj *field = createStringObject((char*)vstr,vlen);
p = lpNext(o->ptr,p);
vstr = lpGet(p,&vlen,intbuf);
robj *value = createStringObject((char*)vstr,vlen);
fn(key, field, value, privdata);
p = lpNext(o->ptr,p);
decrRefCount(field);
decrRefCount(value);
}
cursor->cursor = 1;
cursor->done = 1;
ret = 0;
} }
errno = 0; errno = 0;
return ret; return ret;
......
...@@ -255,9 +255,9 @@ robj *createIntsetObject(void) { ...@@ -255,9 +255,9 @@ robj *createIntsetObject(void) {
} }
robj *createHashObject(void) { robj *createHashObject(void) {
unsigned char *zl = ziplistNew(); unsigned char *zl = lpNew(0);
robj *o = createObject(OBJ_HASH, zl); robj *o = createObject(OBJ_HASH, zl);
o->encoding = OBJ_ENCODING_ZIPLIST; o->encoding = OBJ_ENCODING_LISTPACK;
return o; return o;
} }
...@@ -342,8 +342,8 @@ void freeHashObject(robj *o) { ...@@ -342,8 +342,8 @@ void freeHashObject(robj *o) {
case OBJ_ENCODING_HT: case OBJ_ENCODING_HT:
dictRelease((dict*) o->ptr); dictRelease((dict*) o->ptr);
break; break;
case OBJ_ENCODING_ZIPLIST: case OBJ_ENCODING_LISTPACK:
zfree(o->ptr); lpFree(o->ptr);
break; break;
default: default:
serverPanic("Unknown hash encoding type"); serverPanic("Unknown hash encoding type");
...@@ -929,6 +929,7 @@ char *strEncoding(int encoding) { ...@@ -929,6 +929,7 @@ char *strEncoding(int encoding) {
case OBJ_ENCODING_HT: return "hashtable"; case OBJ_ENCODING_HT: return "hashtable";
case OBJ_ENCODING_QUICKLIST: return "quicklist"; case OBJ_ENCODING_QUICKLIST: return "quicklist";
case OBJ_ENCODING_ZIPLIST: return "ziplist"; case OBJ_ENCODING_ZIPLIST: return "ziplist";
case OBJ_ENCODING_LISTPACK: return "listpack";
case OBJ_ENCODING_INTSET: return "intset"; case OBJ_ENCODING_INTSET: return "intset";
case OBJ_ENCODING_SKIPLIST: return "skiplist"; case OBJ_ENCODING_SKIPLIST: return "skiplist";
case OBJ_ENCODING_EMBSTR: return "embstr"; case OBJ_ENCODING_EMBSTR: return "embstr";
...@@ -1038,7 +1039,7 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) { ...@@ -1038,7 +1039,7 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) {
serverPanic("Unknown sorted set encoding"); serverPanic("Unknown sorted set encoding");
} }
} else if (o->type == OBJ_HASH) { } else if (o->type == OBJ_HASH) {
if (o->encoding == OBJ_ENCODING_ZIPLIST) { if (o->encoding == OBJ_ENCODING_LISTPACK) {
asize = sizeof(*o)+zmalloc_size(o->ptr); asize = sizeof(*o)+zmalloc_size(o->ptr);
} else if (o->encoding == OBJ_ENCODING_HT) { } else if (o->encoding == OBJ_ENCODING_HT) {
d = o->ptr; d = o->ptr;
......
...@@ -676,8 +676,8 @@ int rdbSaveObjectType(rio *rdb, robj *o) { ...@@ -676,8 +676,8 @@ int rdbSaveObjectType(rio *rdb, robj *o) {
else else
serverPanic("Unknown sorted set encoding"); serverPanic("Unknown sorted set encoding");
case OBJ_HASH: case OBJ_HASH:
if (o->encoding == OBJ_ENCODING_ZIPLIST) if (o->encoding == OBJ_ENCODING_LISTPACK)
return rdbSaveType(rdb,RDB_TYPE_HASH_ZIPLIST); return rdbSaveType(rdb,RDB_TYPE_HASH_LISTPACK);
else if (o->encoding == OBJ_ENCODING_HT) else if (o->encoding == OBJ_ENCODING_HT)
return rdbSaveType(rdb,RDB_TYPE_HASH); return rdbSaveType(rdb,RDB_TYPE_HASH);
else else
...@@ -897,12 +897,11 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) { ...@@ -897,12 +897,11 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) {
} }
} else if (o->type == OBJ_HASH) { } else if (o->type == OBJ_HASH) {
/* Save a hash value */ /* Save a hash value */
if (o->encoding == OBJ_ENCODING_ZIPLIST) { if (o->encoding == OBJ_ENCODING_LISTPACK) {
size_t l = ziplistBlobLen((unsigned char*)o->ptr); size_t l = lpBytes((unsigned char*)o->ptr);
if ((n = rdbSaveRawString(rdb,o->ptr,l)) == -1) return -1; if ((n = rdbSaveRawString(rdb,o->ptr,l)) == -1) return -1;
nwritten += n; nwritten += n;
} else if (o->encoding == OBJ_ENCODING_HT) { } else if (o->encoding == OBJ_ENCODING_HT) {
dictIterator *di = dictGetIterator(o->ptr); dictIterator *di = dictGetIterator(o->ptr);
dictEntry *de; dictEntry *de;
...@@ -1703,7 +1702,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -1703,7 +1702,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
o = createHashObject(); o = createHashObject();
/* Too many entries? Use a hash table right from the start. */ /* Too many entries? Use a hash table right from the start. */
if (len > server.hash_max_ziplist_entries) if (len > server.hash_max_listpack_entries)
hashTypeConvert(o, OBJ_ENCODING_HT); hashTypeConvert(o, OBJ_ENCODING_HT);
else if (deep_integrity_validation) { else if (deep_integrity_validation) {
/* In this mode, we need to guarantee that the server won't crash /* In this mode, we need to guarantee that the server won't crash
...@@ -1715,7 +1714,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -1715,7 +1714,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
/* Load every field and value into the ziplist */ /* Load every field and value into the ziplist */
while (o->encoding == OBJ_ENCODING_ZIPLIST && len > 0) { while (o->encoding == OBJ_ENCODING_LISTPACK && len > 0) {
len--; len--;
/* Load raw strings */ /* Load raw strings */
if ((field = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) { if ((field = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) {
...@@ -1743,15 +1742,13 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -1743,15 +1742,13 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
} }
} }
/* Add pair to ziplist */ /* Add pair to listpack */
o->ptr = ziplistPush(o->ptr, (unsigned char*)field, o->ptr = lpAppend(o->ptr, (unsigned char*)field, sdslen(field));
sdslen(field), ZIPLIST_TAIL); o->ptr = lpAppend(o->ptr, (unsigned char*)value, sdslen(value));
o->ptr = ziplistPush(o->ptr, (unsigned char*)value,
sdslen(value), ZIPLIST_TAIL);
/* Convert to hash table if size threshold is exceeded */ /* Convert to hash table if size threshold is exceeded */
if (sdslen(field) > server.hash_max_ziplist_value || if (sdslen(field) > server.hash_max_listpack_value ||
sdslen(value) > server.hash_max_ziplist_value) sdslen(value) > server.hash_max_listpack_value)
{ {
sdsfree(field); sdsfree(field);
sdsfree(value); sdsfree(value);
...@@ -1845,7 +1842,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -1845,7 +1842,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
rdbtype == RDB_TYPE_LIST_ZIPLIST || rdbtype == RDB_TYPE_LIST_ZIPLIST ||
rdbtype == RDB_TYPE_SET_INTSET || rdbtype == RDB_TYPE_SET_INTSET ||
rdbtype == RDB_TYPE_ZSET_ZIPLIST || rdbtype == RDB_TYPE_ZSET_ZIPLIST ||
rdbtype == RDB_TYPE_HASH_ZIPLIST) rdbtype == RDB_TYPE_HASH_ZIPLIST ||
rdbtype == RDB_TYPE_HASH_LISTPACK)
{ {
size_t encoded_len; size_t encoded_len;
unsigned char *encoded = unsigned char *encoded =
...@@ -1874,7 +1872,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -1874,7 +1872,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
/* Convert to ziplist encoded hash. This must be deprecated /* Convert to ziplist encoded hash. This must be deprecated
* when loading dumps created by Redis 2.4 gets deprecated. */ * when loading dumps created by Redis 2.4 gets deprecated. */
{ {
unsigned char *zl = ziplistNew(); unsigned char *zl = lpNew(0);
unsigned char *zi = zipmapRewind(o->ptr); unsigned char *zi = zipmapRewind(o->ptr);
unsigned char *fstr, *vstr; unsigned char *fstr, *vstr;
unsigned int flen, vlen; unsigned int flen, vlen;
...@@ -1884,8 +1882,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -1884,8 +1882,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
while ((zi = zipmapNext(zi, &fstr, &flen, &vstr, &vlen)) != NULL) { while ((zi = zipmapNext(zi, &fstr, &flen, &vstr, &vlen)) != NULL) {
if (flen > maxlen) maxlen = flen; if (flen > maxlen) maxlen = flen;
if (vlen > maxlen) maxlen = vlen; if (vlen > maxlen) maxlen = vlen;
zl = ziplistPush(zl, fstr, flen, ZIPLIST_TAIL); zl = lpAppend(zl, fstr, flen);
zl = ziplistPush(zl, vstr, vlen, ZIPLIST_TAIL); zl = lpAppend(zl, vstr, vlen);
/* search for duplicate records */ /* search for duplicate records */
sds field = sdstrynewlen(fstr, flen); sds field = sdstrynewlen(fstr, flen);
...@@ -1904,10 +1902,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -1904,10 +1902,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
zfree(o->ptr); zfree(o->ptr);
o->ptr = zl; o->ptr = zl;
o->type = OBJ_HASH; o->type = OBJ_HASH;
o->encoding = OBJ_ENCODING_ZIPLIST; o->encoding = OBJ_ENCODING_LISTPACK;
if (hashTypeLength(o) > server.hash_max_ziplist_entries || if (hashTypeLength(o) > server.hash_max_listpack_entries ||
maxlen > server.hash_max_ziplist_value) maxlen > server.hash_max_listpack_value)
{ {
hashTypeConvert(o, OBJ_ENCODING_HT); hashTypeConvert(o, OBJ_ENCODING_HT);
} }
...@@ -1970,16 +1968,44 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -1970,16 +1968,44 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
zsetConvert(o,OBJ_ENCODING_SKIPLIST); zsetConvert(o,OBJ_ENCODING_SKIPLIST);
break; break;
case RDB_TYPE_HASH_ZIPLIST: case RDB_TYPE_HASH_ZIPLIST:
{
unsigned char *lp = lpNew(encoded_len);
if (!hashZiplistConvertAndValidateIntegrity(encoded, encoded_len, &lp)) {
rdbReportCorruptRDB("Hash ziplist integrity check failed.");
zfree(lp);
zfree(encoded);
o->ptr = NULL;
decrRefCount(o);
return NULL;
}
zfree(o->ptr);
o->ptr = lp;
o->type = OBJ_HASH;
o->encoding = OBJ_ENCODING_LISTPACK;
if (hashTypeLength(o) == 0) {
zfree(o->ptr);
o->ptr = NULL;
decrRefCount(o);
goto emptykey;
}
if (hashTypeLength(o) > server.hash_max_listpack_entries) {
hashTypeConvert(o, OBJ_ENCODING_HT);
}
break;
}
case RDB_TYPE_HASH_LISTPACK:
if (deep_integrity_validation) server.stat_dump_payload_sanitizations++; if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
if (!hashZiplistValidateIntegrity(encoded, encoded_len, deep_integrity_validation)) { if (!hashListpackValidateIntegrity(encoded, encoded_len, deep_integrity_validation)) {
rdbReportCorruptRDB("Hash ziplist integrity check failed."); rdbReportCorruptRDB("Hash listpack integrity check failed.");
zfree(encoded); zfree(encoded);
o->ptr = NULL; o->ptr = NULL;
decrRefCount(o); decrRefCount(o);
return NULL; return NULL;
} }
o->type = OBJ_HASH; o->type = OBJ_HASH;
o->encoding = OBJ_ENCODING_ZIPLIST; o->encoding = OBJ_ENCODING_LISTPACK;
if (hashTypeLength(o) == 0) { if (hashTypeLength(o) == 0) {
zfree(encoded); zfree(encoded);
o->ptr = NULL; o->ptr = NULL;
...@@ -1987,7 +2013,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -1987,7 +2013,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
goto emptykey; goto emptykey;
} }
if (hashTypeLength(o) > server.hash_max_ziplist_entries) if (hashTypeLength(o) > server.hash_max_listpack_entries)
hashTypeConvert(o, OBJ_ENCODING_HT); hashTypeConvert(o, OBJ_ENCODING_HT);
break; break;
default: default:
......
...@@ -38,7 +38,7 @@ ...@@ -38,7 +38,7 @@
/* The current RDB version. When the format changes in a way that is no longer /* The current RDB version. When the format changes in a way that is no longer
* backward compatible this number gets incremented. */ * backward compatible this number gets incremented. */
#define RDB_VERSION 9 #define RDB_VERSION 10
/* Defines related to the dump file format. To store 32 bits lengths for short /* Defines related to the dump file format. To store 32 bits lengths for short
* keys requires a lot of space, so we check the most significant 2 bits of * keys requires a lot of space, so we check the most significant 2 bits of
...@@ -91,10 +91,11 @@ ...@@ -91,10 +91,11 @@
#define RDB_TYPE_HASH_ZIPLIST 13 #define RDB_TYPE_HASH_ZIPLIST 13
#define RDB_TYPE_LIST_QUICKLIST 14 #define RDB_TYPE_LIST_QUICKLIST 14
#define RDB_TYPE_STREAM_LISTPACKS 15 #define RDB_TYPE_STREAM_LISTPACKS 15
#define RDB_TYPE_HASH_LISTPACK 16
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */ /* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */
/* Test if a type is an object type. */ /* Test if a type is an object type. */
#define rdbIsObjectType(t) ((t >= 0 && t <= 7) || (t >= 9 && t <= 15)) #define rdbIsObjectType(t) ((t >= 0 && t <= 7) || (t >= 9 && t <= 16))
/* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */ /* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */
#define RDB_OPCODE_MODULE_AUX 247 /* Module auxiliary data. */ #define RDB_OPCODE_MODULE_AUX 247 /* Module auxiliary data. */
......
...@@ -90,7 +90,8 @@ char *rdb_type_string[] = { ...@@ -90,7 +90,8 @@ char *rdb_type_string[] = {
"zset-ziplist", "zset-ziplist",
"hash-ziplist", "hash-ziplist",
"quicklist", "quicklist",
"stream" "stream",
"hash-listpack"
}; };
/* Show a few stats collected into 'rdbstate' */ /* Show a few stats collected into 'rdbstate' */
......
...@@ -6241,7 +6241,8 @@ struct redisTest { ...@@ -6241,7 +6241,8 @@ struct redisTest {
{"crc64", crc64Test}, {"crc64", crc64Test},
{"zmalloc", zmalloc_test}, {"zmalloc", zmalloc_test},
{"sds", sdsTest}, {"sds", sdsTest},
{"dict", dictTest} {"dict", dictTest},
{"listpack", listpackTest}
}; };
redisTestProc *getTestProcByName(const char *name) { redisTestProc *getTestProcByName(const char *name) {
int numtests = sizeof(redisTests)/sizeof(struct redisTest); int numtests = sizeof(redisTests)/sizeof(struct redisTest);
......
...@@ -707,6 +707,7 @@ typedef struct RedisModuleDigest { ...@@ -707,6 +707,7 @@ typedef struct RedisModuleDigest {
#define OBJ_ENCODING_EMBSTR 8 /* Embedded sds string encoding */ #define OBJ_ENCODING_EMBSTR 8 /* Embedded sds string encoding */
#define OBJ_ENCODING_QUICKLIST 9 /* Encoded as linked list of ziplists */ #define OBJ_ENCODING_QUICKLIST 9 /* Encoded as linked list of ziplists */
#define OBJ_ENCODING_STREAM 10 /* Encoded as a radix tree of listpacks */ #define OBJ_ENCODING_STREAM 10 /* Encoded as a radix tree of listpacks */
#define OBJ_ENCODING_LISTPACK 11 /* Encoded as a listpack */
#define LRU_BITS 24 #define LRU_BITS 24
#define LRU_CLOCK_MAX ((1<<LRU_BITS)-1) /* Max value of obj->lru */ #define LRU_CLOCK_MAX ((1<<LRU_BITS)-1) /* Max value of obj->lru */
...@@ -1569,8 +1570,8 @@ struct redisServer { ...@@ -1569,8 +1570,8 @@ struct redisServer {
int sort_bypattern; int sort_bypattern;
int sort_store; int sort_store;
/* Zip structure config, see redis.conf for more information */ /* Zip structure config, see redis.conf for more information */
size_t hash_max_ziplist_entries; size_t hash_max_listpack_entries;
size_t hash_max_ziplist_value; size_t hash_max_listpack_value;
size_t set_max_intset_entries; size_t set_max_intset_entries;
size_t zset_max_ziplist_entries; size_t zset_max_ziplist_entries;
size_t zset_max_ziplist_value; size_t zset_max_ziplist_value;
...@@ -2341,10 +2342,10 @@ unsigned long hashTypeLength(const robj *o); ...@@ -2341,10 +2342,10 @@ unsigned long hashTypeLength(const robj *o);
hashTypeIterator *hashTypeInitIterator(robj *subject); hashTypeIterator *hashTypeInitIterator(robj *subject);
void hashTypeReleaseIterator(hashTypeIterator *hi); void hashTypeReleaseIterator(hashTypeIterator *hi);
int hashTypeNext(hashTypeIterator *hi); int hashTypeNext(hashTypeIterator *hi);
void hashTypeCurrentFromZiplist(hashTypeIterator *hi, int what, void hashTypeCurrentFromListpack(hashTypeIterator *hi, int what,
unsigned char **vstr, unsigned char **vstr,
unsigned int *vlen, unsigned int *vlen,
long long *vll); long long *vll);
sds hashTypeCurrentFromHashTable(hashTypeIterator *hi, int what); sds hashTypeCurrentFromHashTable(hashTypeIterator *hi, int what);
void hashTypeCurrentObject(hashTypeIterator *hi, int what, unsigned char **vstr, unsigned int *vlen, long long *vll); void hashTypeCurrentObject(hashTypeIterator *hi, int what, unsigned char **vstr, unsigned int *vlen, long long *vll);
sds hashTypeCurrentObjectNewSds(hashTypeIterator *hi, int what); sds hashTypeCurrentObjectNewSds(hashTypeIterator *hi, int what);
...@@ -2352,7 +2353,8 @@ robj *hashTypeLookupWriteOrCreate(client *c, robj *key); ...@@ -2352,7 +2353,8 @@ robj *hashTypeLookupWriteOrCreate(client *c, robj *key);
robj *hashTypeGetValueObject(robj *o, sds field); robj *hashTypeGetValueObject(robj *o, sds field);
int hashTypeSet(robj *o, sds field, sds value, int flags); int hashTypeSet(robj *o, sds field, sds value, int flags);
robj *hashTypeDup(robj *o); robj *hashTypeDup(robj *o);
int hashZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep); int hashZiplistConvertAndValidateIntegrity(unsigned char *zl, size_t size, unsigned char **lp);
int hashListpackValidateIntegrity(unsigned char *lp, size_t size, int deep);
/* Pub / Sub */ /* Pub / Sub */
int pubsubUnsubscribeAllChannels(client *c, int notify); int pubsubUnsubscribeAllChannels(client *c, int notify);
......
...@@ -35,16 +35,16 @@ ...@@ -35,16 +35,16 @@
*----------------------------------------------------------------------------*/ *----------------------------------------------------------------------------*/
/* Check the length of a number of objects to see if we need to convert a /* Check the length of a number of objects to see if we need to convert a
* ziplist to a real hash. Note that we only check string encoded objects * listpack to a real hash. Note that we only check string encoded objects
* as their string length can be queried in constant time. */ * as their string length can be queried in constant time. */
void hashTypeTryConversion(robj *o, robj **argv, int start, int end) { void hashTypeTryConversion(robj *o, robj **argv, int start, int end) {
int i; int i;
if (o->encoding != OBJ_ENCODING_ZIPLIST) return; if (o->encoding != OBJ_ENCODING_LISTPACK) return;
for (i = start; i <= end; i++) { for (i = start; i <= end; i++) {
if (sdsEncodedObject(argv[i]) && if (sdsEncodedObject(argv[i]) &&
sdslen(argv[i]->ptr) > server.hash_max_ziplist_value) sdslen(argv[i]->ptr) > server.hash_max_listpack_value)
{ {
hashTypeConvert(o, OBJ_ENCODING_HT); hashTypeConvert(o, OBJ_ENCODING_HT);
break; break;
...@@ -52,32 +52,30 @@ void hashTypeTryConversion(robj *o, robj **argv, int start, int end) { ...@@ -52,32 +52,30 @@ void hashTypeTryConversion(robj *o, robj **argv, int start, int end) {
} }
} }
/* Get the value from a ziplist encoded hash, identified by field. /* Get the value from a listpack encoded hash, identified by field.
* Returns -1 when the field cannot be found. */ * Returns -1 when the field cannot be found. */
int hashTypeGetFromZiplist(robj *o, sds field, int hashTypeGetFromListpack(robj *o, sds field,
unsigned char **vstr, unsigned char **vstr,
unsigned int *vlen, unsigned int *vlen,
long long *vll) long long *vll)
{ {
unsigned char *zl, *fptr = NULL, *vptr = NULL; unsigned char *zl, *fptr = NULL, *vptr = NULL;
int ret;
serverAssert(o->encoding == OBJ_ENCODING_ZIPLIST); serverAssert(o->encoding == OBJ_ENCODING_LISTPACK);
zl = o->ptr; zl = o->ptr;
fptr = ziplistIndex(zl, ZIPLIST_HEAD); fptr = lpFirst(zl);
if (fptr != NULL) { if (fptr != NULL) {
fptr = ziplistFind(zl, fptr, (unsigned char*)field, sdslen(field), 1); fptr = lpFind(zl, fptr, (unsigned char*)field, sdslen(field), 1);
if (fptr != NULL) { if (fptr != NULL) {
/* Grab pointer to the value (fptr points to the field) */ /* Grab pointer to the value (fptr points to the field) */
vptr = ziplistNext(zl, fptr); vptr = lpNext(zl, fptr);
serverAssert(vptr != NULL); serverAssert(vptr != NULL);
} }
} }
if (vptr != NULL) { if (vptr != NULL) {
ret = ziplistGet(vptr, vstr, vlen, vll); *vstr = lpGetValue(vptr, vlen, vll);
serverAssert(ret);
return 0; return 0;
} }
...@@ -107,9 +105,9 @@ sds hashTypeGetFromHashTable(robj *o, sds field) { ...@@ -107,9 +105,9 @@ sds hashTypeGetFromHashTable(robj *o, sds field) {
* can always check the function return by checking the return value * can always check the function return by checking the return value
* for C_OK and checking if vll (or vstr) is NULL. */ * for C_OK and checking if vll (or vstr) is NULL. */
int hashTypeGetValue(robj *o, sds field, unsigned char **vstr, unsigned int *vlen, long long *vll) { int hashTypeGetValue(robj *o, sds field, unsigned char **vstr, unsigned int *vlen, long long *vll) {
if (o->encoding == OBJ_ENCODING_ZIPLIST) { if (o->encoding == OBJ_ENCODING_LISTPACK) {
*vstr = NULL; *vstr = NULL;
if (hashTypeGetFromZiplist(o, field, vstr, vlen, vll) == 0) if (hashTypeGetFromListpack(o, field, vstr, vlen, vll) == 0)
return C_OK; return C_OK;
} else if (o->encoding == OBJ_ENCODING_HT) { } else if (o->encoding == OBJ_ENCODING_HT) {
sds value; sds value;
...@@ -143,12 +141,12 @@ robj *hashTypeGetValueObject(robj *o, sds field) { ...@@ -143,12 +141,12 @@ robj *hashTypeGetValueObject(robj *o, sds field) {
* exist. */ * exist. */
size_t hashTypeGetValueLength(robj *o, sds field) { size_t hashTypeGetValueLength(robj *o, sds field) {
size_t len = 0; size_t len = 0;
if (o->encoding == OBJ_ENCODING_ZIPLIST) { if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *vstr = NULL; unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX; unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX; long long vll = LLONG_MAX;
if (hashTypeGetFromZiplist(o, field, &vstr, &vlen, &vll) == 0) if (hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll) == 0)
len = vstr ? vlen : sdigits10(vll); len = vstr ? vlen : sdigits10(vll);
} else if (o->encoding == OBJ_ENCODING_HT) { } else if (o->encoding == OBJ_ENCODING_HT) {
sds aux; sds aux;
...@@ -164,12 +162,12 @@ size_t hashTypeGetValueLength(robj *o, sds field) { ...@@ -164,12 +162,12 @@ size_t hashTypeGetValueLength(robj *o, sds field) {
/* Test if the specified field exists in the given hash. Returns 1 if the field /* Test if the specified field exists in the given hash. Returns 1 if the field
* exists, and 0 when it doesn't. */ * exists, and 0 when it doesn't. */
int hashTypeExists(robj *o, sds field) { int hashTypeExists(robj *o, sds field) {
if (o->encoding == OBJ_ENCODING_ZIPLIST) { if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *vstr = NULL; unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX; unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX; long long vll = LLONG_MAX;
if (hashTypeGetFromZiplist(o, field, &vstr, &vlen, &vll) == 0) return 1; if (hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll) == 0) return 1;
} else if (o->encoding == OBJ_ENCODING_HT) { } else if (o->encoding == OBJ_ENCODING_HT) {
if (hashTypeGetFromHashTable(o, field) != NULL) return 1; if (hashTypeGetFromHashTable(o, field) != NULL) return 1;
} else { } else {
...@@ -202,36 +200,33 @@ int hashTypeExists(robj *o, sds field) { ...@@ -202,36 +200,33 @@ int hashTypeExists(robj *o, sds field) {
int hashTypeSet(robj *o, sds field, sds value, int flags) { int hashTypeSet(robj *o, sds field, sds value, int flags) {
int update = 0; int update = 0;
if (o->encoding == OBJ_ENCODING_ZIPLIST) { if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *zl, *fptr, *vptr; unsigned char *zl, *fptr, *vptr;
zl = o->ptr; zl = o->ptr;
fptr = ziplistIndex(zl, ZIPLIST_HEAD); fptr = lpFirst(zl);
if (fptr != NULL) { if (fptr != NULL) {
fptr = ziplistFind(zl, fptr, (unsigned char*)field, sdslen(field), 1); fptr = lpFind(zl, fptr, (unsigned char*)field, sdslen(field), 1);
if (fptr != NULL) { if (fptr != NULL) {
/* Grab pointer to the value (fptr points to the field) */ /* Grab pointer to the value (fptr points to the field) */
vptr = ziplistNext(zl, fptr); vptr = lpNext(zl, fptr);
serverAssert(vptr != NULL); serverAssert(vptr != NULL);
update = 1; update = 1;
/* Replace value */ /* Replace value */
zl = ziplistReplace(zl, vptr, (unsigned char*)value, zl = lpReplace(zl, &vptr, (unsigned char*)value, sdslen(value));
sdslen(value));
} }
} }
if (!update) { if (!update) {
/* Push new field/value pair onto the tail of the ziplist */ /* Push new field/value pair onto the tail of the listpack */
zl = ziplistPush(zl, (unsigned char*)field, sdslen(field), zl = lpAppend(zl, (unsigned char*)field, sdslen(field));
ZIPLIST_TAIL); zl = lpAppend(zl, (unsigned char*)value, sdslen(value));
zl = ziplistPush(zl, (unsigned char*)value, sdslen(value),
ZIPLIST_TAIL);
} }
o->ptr = zl; o->ptr = zl;
/* Check if the ziplist needs to be converted to a hash table */ /* Check if the listpack needs to be converted to a hash table */
if (hashTypeLength(o) > server.hash_max_ziplist_entries) if (hashTypeLength(o) > server.hash_max_listpack_entries)
hashTypeConvert(o, OBJ_ENCODING_HT); hashTypeConvert(o, OBJ_ENCODING_HT);
} else if (o->encoding == OBJ_ENCODING_HT) { } else if (o->encoding == OBJ_ENCODING_HT) {
dictEntry *de = dictFind(o->ptr,field); dictEntry *de = dictFind(o->ptr,field);
...@@ -276,16 +271,16 @@ int hashTypeSet(robj *o, sds field, sds value, int flags) { ...@@ -276,16 +271,16 @@ int hashTypeSet(robj *o, sds field, sds value, int flags) {
int hashTypeDelete(robj *o, sds field) { int hashTypeDelete(robj *o, sds field) {
int deleted = 0; int deleted = 0;
if (o->encoding == OBJ_ENCODING_ZIPLIST) { if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *zl, *fptr; unsigned char *zl, *fptr;
zl = o->ptr; zl = o->ptr;
fptr = ziplistIndex(zl, ZIPLIST_HEAD); fptr = lpFirst(zl);
if (fptr != NULL) { if (fptr != NULL) {
fptr = ziplistFind(zl, fptr, (unsigned char*)field, sdslen(field), 1); fptr = lpFind(zl, fptr, (unsigned char*)field, sdslen(field), 1);
if (fptr != NULL) { if (fptr != NULL) {
zl = ziplistDelete(zl,&fptr); /* Delete the key. */ zl = lpDelete(zl,fptr,&fptr); /* Delete the key. */
zl = ziplistDelete(zl,&fptr); /* Delete the value. */ zl = lpDelete(zl,fptr,&fptr); /* Delete the value. */
o->ptr = zl; o->ptr = zl;
deleted = 1; deleted = 1;
} }
...@@ -308,8 +303,8 @@ int hashTypeDelete(robj *o, sds field) { ...@@ -308,8 +303,8 @@ int hashTypeDelete(robj *o, sds field) {
unsigned long hashTypeLength(const robj *o) { unsigned long hashTypeLength(const robj *o) {
unsigned long length = ULONG_MAX; unsigned long length = ULONG_MAX;
if (o->encoding == OBJ_ENCODING_ZIPLIST) { if (o->encoding == OBJ_ENCODING_LISTPACK) {
length = ziplistLen(o->ptr) / 2; length = lpLength(o->ptr) / 2;
} else if (o->encoding == OBJ_ENCODING_HT) { } else if (o->encoding == OBJ_ENCODING_HT) {
length = dictSize((const dict*)o->ptr); length = dictSize((const dict*)o->ptr);
} else { } else {
...@@ -323,7 +318,7 @@ hashTypeIterator *hashTypeInitIterator(robj *subject) { ...@@ -323,7 +318,7 @@ hashTypeIterator *hashTypeInitIterator(robj *subject) {
hi->subject = subject; hi->subject = subject;
hi->encoding = subject->encoding; hi->encoding = subject->encoding;
if (hi->encoding == OBJ_ENCODING_ZIPLIST) { if (hi->encoding == OBJ_ENCODING_LISTPACK) {
hi->fptr = NULL; hi->fptr = NULL;
hi->vptr = NULL; hi->vptr = NULL;
} else if (hi->encoding == OBJ_ENCODING_HT) { } else if (hi->encoding == OBJ_ENCODING_HT) {
...@@ -343,7 +338,7 @@ void hashTypeReleaseIterator(hashTypeIterator *hi) { ...@@ -343,7 +338,7 @@ void hashTypeReleaseIterator(hashTypeIterator *hi) {
/* Move to the next entry in the hash. Return C_OK when the next entry /* Move to the next entry in the hash. Return C_OK when the next entry
* could be found and C_ERR when the iterator reaches the end. */ * could be found and C_ERR when the iterator reaches the end. */
int hashTypeNext(hashTypeIterator *hi) { int hashTypeNext(hashTypeIterator *hi) {
if (hi->encoding == OBJ_ENCODING_ZIPLIST) { if (hi->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *zl; unsigned char *zl;
unsigned char *fptr, *vptr; unsigned char *fptr, *vptr;
...@@ -354,16 +349,16 @@ int hashTypeNext(hashTypeIterator *hi) { ...@@ -354,16 +349,16 @@ int hashTypeNext(hashTypeIterator *hi) {
if (fptr == NULL) { if (fptr == NULL) {
/* Initialize cursor */ /* Initialize cursor */
serverAssert(vptr == NULL); serverAssert(vptr == NULL);
fptr = ziplistIndex(zl, 0); fptr = lpFirst(zl);
} else { } else {
/* Advance cursor */ /* Advance cursor */
serverAssert(vptr != NULL); serverAssert(vptr != NULL);
fptr = ziplistNext(zl, vptr); fptr = lpNext(zl, vptr);
} }
if (fptr == NULL) return C_ERR; if (fptr == NULL) return C_ERR;
/* Grab pointer to the value (fptr points to the field) */ /* Grab pointer to the value (fptr points to the field) */
vptr = ziplistNext(zl, fptr); vptr = lpNext(zl, fptr);
serverAssert(vptr != NULL); serverAssert(vptr != NULL);
/* fptr, vptr now point to the first or next pair */ /* fptr, vptr now point to the first or next pair */
...@@ -378,22 +373,18 @@ int hashTypeNext(hashTypeIterator *hi) { ...@@ -378,22 +373,18 @@ int hashTypeNext(hashTypeIterator *hi) {
} }
/* Get the field or value at iterator cursor, for an iterator on a hash value /* Get the field or value at iterator cursor, for an iterator on a hash value
* encoded as a ziplist. Prototype is similar to `hashTypeGetFromZiplist`. */ * encoded as a listpack. Prototype is similar to `hashTypeGetFromListpack`. */
void hashTypeCurrentFromZiplist(hashTypeIterator *hi, int what, void hashTypeCurrentFromListpack(hashTypeIterator *hi, int what,
unsigned char **vstr, unsigned char **vstr,
unsigned int *vlen, unsigned int *vlen,
long long *vll) long long *vll)
{ {
int ret; serverAssert(hi->encoding == OBJ_ENCODING_LISTPACK);
serverAssert(hi->encoding == OBJ_ENCODING_ZIPLIST);
if (what & OBJ_HASH_KEY) { if (what & OBJ_HASH_KEY) {
ret = ziplistGet(hi->fptr, vstr, vlen, vll); *vstr = lpGetValue(hi->fptr, vlen, vll);
serverAssert(ret);
} else { } else {
ret = ziplistGet(hi->vptr, vstr, vlen, vll); *vstr = lpGetValue(hi->vptr, vlen, vll);
serverAssert(ret);
} }
} }
...@@ -421,9 +412,9 @@ sds hashTypeCurrentFromHashTable(hashTypeIterator *hi, int what) { ...@@ -421,9 +412,9 @@ sds hashTypeCurrentFromHashTable(hashTypeIterator *hi, int what) {
* can always check the function return by checking the return value * can always check the function return by checking the return value
* type checking if vstr == NULL. */ * type checking if vstr == NULL. */
void hashTypeCurrentObject(hashTypeIterator *hi, int what, unsigned char **vstr, unsigned int *vlen, long long *vll) { void hashTypeCurrentObject(hashTypeIterator *hi, int what, unsigned char **vstr, unsigned int *vlen, long long *vll) {
if (hi->encoding == OBJ_ENCODING_ZIPLIST) { if (hi->encoding == OBJ_ENCODING_LISTPACK) {
*vstr = NULL; *vstr = NULL;
hashTypeCurrentFromZiplist(hi, what, vstr, vlen, vll); hashTypeCurrentFromListpack(hi, what, vstr, vlen, vll);
} else if (hi->encoding == OBJ_ENCODING_HT) { } else if (hi->encoding == OBJ_ENCODING_HT) {
sds ele = hashTypeCurrentFromHashTable(hi, what); sds ele = hashTypeCurrentFromHashTable(hi, what);
*vstr = (unsigned char*) ele; *vstr = (unsigned char*) ele;
...@@ -456,10 +447,11 @@ robj *hashTypeLookupWriteOrCreate(client *c, robj *key) { ...@@ -456,10 +447,11 @@ robj *hashTypeLookupWriteOrCreate(client *c, robj *key) {
return o; return o;
} }
void hashTypeConvertZiplist(robj *o, int enc) {
serverAssert(o->encoding == OBJ_ENCODING_ZIPLIST);
if (enc == OBJ_ENCODING_ZIPLIST) { void hashTypeConvertListpack(robj *o, int enc) {
serverAssert(o->encoding == OBJ_ENCODING_LISTPACK);
if (enc == OBJ_ENCODING_LISTPACK) {
/* Nothing to do... */ /* Nothing to do... */
} else if (enc == OBJ_ENCODING_HT) { } else if (enc == OBJ_ENCODING_HT) {
...@@ -480,9 +472,9 @@ void hashTypeConvertZiplist(robj *o, int enc) { ...@@ -480,9 +472,9 @@ void hashTypeConvertZiplist(robj *o, int enc) {
value = hashTypeCurrentObjectNewSds(hi,OBJ_HASH_VALUE); value = hashTypeCurrentObjectNewSds(hi,OBJ_HASH_VALUE);
ret = dictAdd(dict, key, value); ret = dictAdd(dict, key, value);
if (ret != DICT_OK) { if (ret != DICT_OK) {
serverLogHexDump(LL_WARNING,"ziplist with dup elements dump", serverLogHexDump(LL_WARNING,"listpack with dup elements dump",
o->ptr,ziplistBlobLen(o->ptr)); o->ptr,lpBytes(o->ptr));
serverPanic("Ziplist corruption detected"); serverPanic("Listpack corruption detected");
} }
} }
hashTypeReleaseIterator(hi); hashTypeReleaseIterator(hi);
...@@ -495,8 +487,8 @@ void hashTypeConvertZiplist(robj *o, int enc) { ...@@ -495,8 +487,8 @@ void hashTypeConvertZiplist(robj *o, int enc) {
} }
void hashTypeConvert(robj *o, int enc) { void hashTypeConvert(robj *o, int enc) {
if (o->encoding == OBJ_ENCODING_ZIPLIST) { if (o->encoding == OBJ_ENCODING_LISTPACK) {
hashTypeConvertZiplist(o, enc); hashTypeConvertListpack(o, enc);
} else if (o->encoding == OBJ_ENCODING_HT) { } else if (o->encoding == OBJ_ENCODING_HT) {
serverPanic("Not implemented"); serverPanic("Not implemented");
} else { } else {
...@@ -515,13 +507,13 @@ robj *hashTypeDup(robj *o) { ...@@ -515,13 +507,13 @@ robj *hashTypeDup(robj *o) {
serverAssert(o->type == OBJ_HASH); serverAssert(o->type == OBJ_HASH);
if(o->encoding == OBJ_ENCODING_ZIPLIST){ if(o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *zl = o->ptr; unsigned char *zl = o->ptr;
size_t sz = ziplistBlobLen(zl); size_t sz = lpBytes(zl);
unsigned char *new_zl = zmalloc(sz); unsigned char *new_zl = zmalloc(sz);
memcpy(new_zl, zl, sz); memcpy(new_zl, zl, sz);
hobj = createObject(OBJ_HASH, new_zl); hobj = createObject(OBJ_HASH, new_zl);
hobj->encoding = OBJ_ENCODING_ZIPLIST; hobj->encoding = OBJ_ENCODING_LISTPACK;
} else if(o->encoding == OBJ_ENCODING_HT){ } else if(o->encoding == OBJ_ENCODING_HT){
dict *d = dictCreate(&hashDictType); dict *d = dictCreate(&hashDictType);
dictExpand(d, dictSize((const dict*)o->ptr)); dictExpand(d, dictSize((const dict*)o->ptr));
...@@ -549,20 +541,25 @@ robj *hashTypeDup(robj *o) { ...@@ -549,20 +541,25 @@ robj *hashTypeDup(robj *o) {
return hobj; return hobj;
} }
/* callback for to check the ziplist doesn't have duplicate records */ /* callback for hashZiplistConvertAndValidateIntegrity.
static int _hashZiplistEntryValidation(unsigned char *p, void *userdata) { * Check that the ziplist doesn't have duplicate hash field names.
* The ziplist element pointed by 'p' will be converted and stored into listpack. */
static int _hashZiplistEntryConvertAndValidation(unsigned char *p, void *userdata) {
unsigned char *str;
unsigned int slen;
long long vll;
struct { struct {
long count; long count;
dict *fields; dict *fields;
unsigned char **lp;
} *data = userdata; } *data = userdata;
if (!ziplistGet(p, &str, &slen, &vll))
return 0;
/* Even records are field names, add to dict and check that's not a dup */ /* Even records are field names, add to dict and check that's not a dup */
if (((data->count) & 1) == 0) { if (((data->count) & 1) == 0) {
unsigned char *str;
unsigned int slen;
long long vll;
if (!ziplistGet(p, &str, &slen, &vll))
return 0;
sds field = str? sdsnewlen(str, slen): sdsfromlonglong(vll); sds field = str? sdsnewlen(str, slen): sdsfromlonglong(vll);
if (dictAdd(data->fields, field, NULL) != DICT_OK) { if (dictAdd(data->fields, field, NULL) != DICT_OK) {
/* Duplicate, return an error */ /* Duplicate, return an error */
...@@ -571,16 +568,70 @@ static int _hashZiplistEntryValidation(unsigned char *p, void *userdata) { ...@@ -571,16 +568,70 @@ static int _hashZiplistEntryValidation(unsigned char *p, void *userdata) {
} }
} }
if (str) {
*(data->lp) = lpAppend(*(data->lp), (unsigned char*)str, slen);
} else {
*(data->lp) = lpAppendInteger(*(data->lp), vll);
}
(data->count)++;
return 1;
}
/* callback for to check the listpack doesn't have duplicate records */
static int _hashListpackEntryValidation(unsigned char *p, void *userdata) {
struct {
long count;
dict *fields;
} *data = userdata;
/* Even records are field names, add to dict and check that's not a dup */
if (((data->count) & 1) == 0) {
unsigned char *str;
int64_t slen;
unsigned char buf[LP_INTBUF_SIZE];
str = lpGet(p, &slen, buf);
sds field = sdsnewlen(str, slen);
if (dictAdd(data->fields, field, NULL) != DICT_OK) {
/* Duplicate, return an error */
sdsfree(field);
return 0;
}
}
(data->count)++; (data->count)++;
return 1; return 1;
} }
/* Validate the integrity of the data structure. /* Validate the integrity of the data structure while converting it to
* listpack and storing it at 'lp'.
* The function is safe to call on non-validated ziplists, it returns 0
* when encounter an integrity validation issue. */
int hashZiplistConvertAndValidateIntegrity(unsigned char *zl, size_t size, unsigned char **lp) {
/* Keep track of the field names to locate duplicate ones */
struct {
long count;
dict *fields;
unsigned char **lp;
} data = {0, dictCreate(&hashDictType), lp};
int ret = ziplistValidateIntegrity(zl, size, 1, _hashZiplistEntryConvertAndValidation, &data);
/* make sure we have an even number of records. */
if (data.count & 1)
ret = 0;
dictRelease(data.fields);
return ret;
}
/* Validate the integrity of the listpack structure.
* when `deep` is 0, only the integrity of the header is validated. * when `deep` is 0, only the integrity of the header is validated.
* when `deep` is 1, we scan all the entries one by one. */ * when `deep` is 1, we scan all the entries one by one. */
int hashZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep) { int hashListpackValidateIntegrity(unsigned char *lp, size_t size, int deep) {
if (!deep) if (!deep)
return ziplistValidateIntegrity(zl, size, 0, NULL, NULL); return lpValidateIntegrity(lp, size, 0, NULL, NULL);
/* Keep track of the field names to locate duplicate ones */ /* Keep track of the field names to locate duplicate ones */
struct { struct {
...@@ -588,7 +639,7 @@ int hashZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep) { ...@@ -588,7 +639,7 @@ int hashZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep) {
dict *fields; dict *fields;
} data = {0, dictCreate(&hashDictType)}; } data = {0, dictCreate(&hashDictType)};
int ret = ziplistValidateIntegrity(zl, size, 1, _hashZiplistEntryValidation, &data); int ret = lpValidateIntegrity(lp, size, 1, _hashListpackEntryValidation, &data);
/* make sure we have an even number of records. */ /* make sure we have an even number of records. */
if (data.count & 1) if (data.count & 1)
...@@ -598,13 +649,13 @@ int hashZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep) { ...@@ -598,13 +649,13 @@ int hashZiplistValidateIntegrity(unsigned char *zl, size_t size, int deep) {
return ret; return ret;
} }
/* Create a new sds string from the ziplist entry. */ /* Create a new sds string from the listpack entry. */
sds hashSdsFromZiplistEntry(ziplistEntry *e) { sds hashSdsFromListpackEntry(listpackEntry *e) {
return e->sval ? sdsnewlen(e->sval, e->slen) : sdsfromlonglong(e->lval); return e->sval ? sdsnewlen(e->sval, e->slen) : sdsfromlonglong(e->lval);
} }
/* Reply with bulk string from the ziplist entry. */ /* Reply with bulk string from the listpack entry. */
void hashReplyFromZiplistEntry(client *c, ziplistEntry *e) { void hashReplyFromListpackEntry(client *c, listpackEntry *e) {
if (e->sval) if (e->sval)
addReplyBulkCBuffer(c, e->sval, e->slen); addReplyBulkCBuffer(c, e->sval, e->slen);
else else
...@@ -615,7 +666,7 @@ void hashReplyFromZiplistEntry(client *c, ziplistEntry *e) { ...@@ -615,7 +666,7 @@ void hashReplyFromZiplistEntry(client *c, ziplistEntry *e) {
* 'key' and 'val' will be set to hold the element. * 'key' and 'val' will be set to hold the element.
* The memory in them is not to be freed or modified by the caller. * The memory in them is not to be freed or modified by the caller.
* 'val' can be NULL in which case it's not extracted. */ * 'val' can be NULL in which case it's not extracted. */
void hashTypeRandomElement(robj *hashobj, unsigned long hashsize, ziplistEntry *key, ziplistEntry *val) { void hashTypeRandomElement(robj *hashobj, unsigned long hashsize, listpackEntry *key, listpackEntry *val) {
if (hashobj->encoding == OBJ_ENCODING_HT) { if (hashobj->encoding == OBJ_ENCODING_HT) {
dictEntry *de = dictGetFairRandomKey(hashobj->ptr); dictEntry *de = dictGetFairRandomKey(hashobj->ptr);
sds s = dictGetKey(de); sds s = dictGetKey(de);
...@@ -626,8 +677,8 @@ void hashTypeRandomElement(robj *hashobj, unsigned long hashsize, ziplistEntry * ...@@ -626,8 +677,8 @@ void hashTypeRandomElement(robj *hashobj, unsigned long hashsize, ziplistEntry *
val->sval = (unsigned char*)s; val->sval = (unsigned char*)s;
val->slen = sdslen(s); val->slen = sdslen(s);
} }
} else if (hashobj->encoding == OBJ_ENCODING_ZIPLIST) { } else if (hashobj->encoding == OBJ_ENCODING_LISTPACK) {
ziplistRandomPair(hashobj->ptr, hashsize, key, val); lpRandomPair(hashobj->ptr, hashsize, key, val);
} else { } else {
serverPanic("Unknown hash encoding"); serverPanic("Unknown hash encoding");
} }
...@@ -774,12 +825,12 @@ static void addHashFieldToReply(client *c, robj *o, sds field) { ...@@ -774,12 +825,12 @@ static void addHashFieldToReply(client *c, robj *o, sds field) {
return; return;
} }
if (o->encoding == OBJ_ENCODING_ZIPLIST) { if (o->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *vstr = NULL; unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX; unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX; long long vll = LLONG_MAX;
ret = hashTypeGetFromZiplist(o, field, &vstr, &vlen, &vll); ret = hashTypeGetFromListpack(o, field, &vstr, &vlen, &vll);
if (ret < 0) { if (ret < 0) {
addReplyNull(c); addReplyNull(c);
} else { } else {
...@@ -871,12 +922,12 @@ void hstrlenCommand(client *c) { ...@@ -871,12 +922,12 @@ void hstrlenCommand(client *c) {
} }
static void addHashIteratorCursorToReply(client *c, hashTypeIterator *hi, int what) { static void addHashIteratorCursorToReply(client *c, hashTypeIterator *hi, int what) {
if (hi->encoding == OBJ_ENCODING_ZIPLIST) { if (hi->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *vstr = NULL; unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX; unsigned int vlen = UINT_MAX;
long long vll = LLONG_MAX; long long vll = LLONG_MAX;
hashTypeCurrentFromZiplist(hi, what, &vstr, &vlen, &vll); hashTypeCurrentFromListpack(hi, what, &vstr, &vlen, &vll);
if (vstr) if (vstr)
addReplyBulkCBuffer(c, vstr, vlen); addReplyBulkCBuffer(c, vstr, vlen);
else else
...@@ -957,7 +1008,7 @@ void hscanCommand(client *c) { ...@@ -957,7 +1008,7 @@ void hscanCommand(client *c) {
scanGenericCommand(c,o,cursor); scanGenericCommand(c,o,cursor);
} }
static void harndfieldReplyWithZiplist(client *c, unsigned int count, ziplistEntry *keys, ziplistEntry *vals) { static void harndfieldReplyWithListpack(client *c, unsigned int count, listpackEntry *keys, listpackEntry *vals) {
for (unsigned long i = 0; i < count; i++) { for (unsigned long i = 0; i < count; i++) {
if (vals && c->resp > 2) if (vals && c->resp > 2)
addReplyArrayLen(c,2); addReplyArrayLen(c,2);
...@@ -1028,18 +1079,19 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { ...@@ -1028,18 +1079,19 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
if (withvalues) if (withvalues)
addReplyBulkCBuffer(c, value, sdslen(value)); addReplyBulkCBuffer(c, value, sdslen(value));
} }
} else if (hash->encoding == OBJ_ENCODING_ZIPLIST) { } else if (hash->encoding == OBJ_ENCODING_LISTPACK) {
ziplistEntry *keys, *vals = NULL; listpackEntry *keys, *vals = NULL;
unsigned long limit, sample_count; unsigned long limit, sample_count;
limit = count > HRANDFIELD_RANDOM_SAMPLE_LIMIT ? HRANDFIELD_RANDOM_SAMPLE_LIMIT : count; limit = count > HRANDFIELD_RANDOM_SAMPLE_LIMIT ? HRANDFIELD_RANDOM_SAMPLE_LIMIT : count;
keys = zmalloc(sizeof(ziplistEntry)*limit); keys = zmalloc(sizeof(listpackEntry)*limit);
if (withvalues) if (withvalues)
vals = zmalloc(sizeof(ziplistEntry)*limit); vals = zmalloc(sizeof(listpackEntry)*limit);
while (count) { while (count) {
sample_count = count > limit ? limit : count; sample_count = count > limit ? limit : count;
count -= sample_count; count -= sample_count;
ziplistRandomPairs(hash->ptr, sample_count, keys, vals); lpRandomPairs(hash->ptr, sample_count, keys, vals);
harndfieldReplyWithZiplist(c, sample_count, keys, vals); harndfieldReplyWithListpack(c, sample_count, keys, vals);
} }
zfree(keys); zfree(keys);
zfree(vals); zfree(vals);
...@@ -1133,15 +1185,15 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { ...@@ -1133,15 +1185,15 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
* to the temporary hash, trying to eventually get enough unique elements * to the temporary hash, trying to eventually get enough unique elements
* to reach the specified count. */ * to reach the specified count. */
else { else {
if (hash->encoding == OBJ_ENCODING_ZIPLIST) { if (hash->encoding == OBJ_ENCODING_LISTPACK) {
/* it is inefficient to repeatedly pick one random element from a /* it is inefficient to repeatedly pick one random element from a
* ziplist. so we use this instead: */ * listpack. so we use this instead: */
ziplistEntry *keys, *vals = NULL; listpackEntry *keys, *vals = NULL;
keys = zmalloc(sizeof(ziplistEntry)*count); keys = zmalloc(sizeof(listpackEntry)*count);
if (withvalues) if (withvalues)
vals = zmalloc(sizeof(ziplistEntry)*count); vals = zmalloc(sizeof(listpackEntry)*count);
serverAssert(ziplistRandomPairsUnique(hash->ptr, count, keys, vals) == count); serverAssert(lpRandomPairsUnique(hash->ptr, count, keys, vals) == count);
harndfieldReplyWithZiplist(c, count, keys, vals); harndfieldReplyWithListpack(c, count, keys, vals);
zfree(keys); zfree(keys);
zfree(vals); zfree(vals);
return; return;
...@@ -1149,7 +1201,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { ...@@ -1149,7 +1201,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
/* Hashtable encoding (generic implementation) */ /* Hashtable encoding (generic implementation) */
unsigned long added = 0; unsigned long added = 0;
ziplistEntry key, value; listpackEntry key, value;
dict *d = dictCreate(&hashDictType); dict *d = dictCreate(&hashDictType);
dictExpand(d, count); dictExpand(d, count);
while(added < count) { while(added < count) {
...@@ -1158,7 +1210,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { ...@@ -1158,7 +1210,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
/* Try to add the object to the dictionary. If it already exists /* Try to add the object to the dictionary. If it already exists
* free it, otherwise increment the number of objects we have * free it, otherwise increment the number of objects we have
* in the result dictionary. */ * in the result dictionary. */
sds skey = hashSdsFromZiplistEntry(&key); sds skey = hashSdsFromListpackEntry(&key);
if (dictAdd(d,skey,NULL) != DICT_OK) { if (dictAdd(d,skey,NULL) != DICT_OK) {
sdsfree(skey); sdsfree(skey);
continue; continue;
...@@ -1168,9 +1220,9 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { ...@@ -1168,9 +1220,9 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
/* We can reply right away, so that we don't need to store the value in the dict. */ /* We can reply right away, so that we don't need to store the value in the dict. */
if (withvalues && c->resp > 2) if (withvalues && c->resp > 2)
addReplyArrayLen(c,2); addReplyArrayLen(c,2);
hashReplyFromZiplistEntry(c, &key); hashReplyFromListpackEntry(c, &key);
if (withvalues) if (withvalues)
hashReplyFromZiplistEntry(c, &value); hashReplyFromListpackEntry(c, &value);
} }
/* Release memory */ /* Release memory */
...@@ -1183,7 +1235,7 @@ void hrandfieldCommand(client *c) { ...@@ -1183,7 +1235,7 @@ void hrandfieldCommand(client *c) {
long l; long l;
int withvalues = 0; int withvalues = 0;
robj *hash; robj *hash;
ziplistEntry ele; listpackEntry ele;
if (c->argc >= 3) { if (c->argc >= 3) {
if (getLongFromObjectOrReply(c,c->argv[2],&l,NULL) != C_OK) return; if (getLongFromObjectOrReply(c,c->argv[2],&l,NULL) != C_OK) return;
...@@ -1203,5 +1255,5 @@ void hrandfieldCommand(client *c) { ...@@ -1203,5 +1255,5 @@ void hrandfieldCommand(client *c) {
} }
hashTypeRandomElement(hash,hashTypeLength(hash),&ele,NULL); hashTypeRandomElement(hash,hashTypeLength(hash),&ele,NULL);
hashReplyFromZiplistEntry(c, &ele); hashReplyFromListpackEntry(c, &ele);
} }
...@@ -241,24 +241,6 @@ robj *streamDup(robj *o) { ...@@ -241,24 +241,6 @@ robj *streamDup(robj *o) {
return sobj; return sobj;
} }
/* This is just a wrapper for lpAppend() to directly use a 64 bit integer
* instead of a string. */
unsigned char *lpAppendInteger(unsigned char *lp, int64_t value) {
char buf[LONG_STR_SIZE];
int slen = ll2string(buf,sizeof(buf),value);
return lpAppend(lp,(unsigned char*)buf,slen);
}
/* This is just a wrapper for lpReplace() to directly use a 64 bit integer
* instead of a string to replace the current element. The function returns
* the new listpack as return value, and also updates the current cursor
* by updating '*pos'. */
unsigned char *lpReplaceInteger(unsigned char *lp, unsigned char **pos, int64_t value) {
char buf[LONG_STR_SIZE];
int slen = ll2string(buf,sizeof(buf),value);
return lpInsert(lp, (unsigned char*)buf, slen, *pos, LP_REPLACE, pos);
}
/* This is a wrapper function for lpGet() to directly get an integer value /* This is a wrapper function for lpGet() to directly get an integer value
* from the listpack (that may store numbers as a string), converting * from the listpack (that may store numbers as a string), converting
* the string if needed. * the string if needed.
...@@ -3577,7 +3559,7 @@ int streamValidateListpackIntegrity(unsigned char *lp, size_t size, int deep) { ...@@ -3577,7 +3559,7 @@ int streamValidateListpackIntegrity(unsigned char *lp, size_t size, int deep) {
/* Since we don't want to run validation of all records twice, we'll /* Since we don't want to run validation of all records twice, we'll
* run the listpack validation of just the header and do the rest here. */ * run the listpack validation of just the header and do the rest here. */
if (!lpValidateIntegrity(lp, size, 0)) if (!lpValidateIntegrity(lp, size, 0, NULL, NULL))
return 0; return 0;
/* In non-deep mode we just validated the listpack header (encoded size) */ /* In non-deep mode we just validated the listpack header (encoded size) */
......
tags {"external:skip"} {
# Copy RDB with ziplist encoded hash to server path
set server_path [tmpdir "server.convert-ziplist-hash-on-load"]
exec cp -f tests/assets/hash-ziplist.rdb $server_path
start_server [list overrides [list "dir" $server_path "dbfilename" "hash-ziplist.rdb"]] {
test "RDB load ziplist hash: converts to listpack when RDB loading" {
r select 0
assert_encoding listpack hash
assert_equal 2 [r hlen hash]
assert_match {v1 v2} [r hmget hash f1 f2]
}
}
exec cp -f tests/assets/hash-ziplist.rdb $server_path
start_server [list overrides [list "dir" $server_path "dbfilename" "hash-ziplist.rdb" "hash-max-ziplist-entries" 1]] {
test "RDB load ziplist hash: converts to hash table when hash-max-ziplist-entries is exceeded" {
r select 0
assert_encoding hashtable hash
assert_equal 2 [r hlen hash]
assert_match {v1 v2} [r hmget hash f1 f2]
}
}
}
...@@ -5,10 +5,10 @@ set server_path [tmpdir "server.convert-zipmap-hash-on-load"] ...@@ -5,10 +5,10 @@ set server_path [tmpdir "server.convert-zipmap-hash-on-load"]
exec cp -f tests/assets/hash-zipmap.rdb $server_path exec cp -f tests/assets/hash-zipmap.rdb $server_path
start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap.rdb"]] { start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap.rdb"]] {
test "RDB load zipmap hash: converts to ziplist" { test "RDB load zipmap hash: converts to listpack" {
r select 0 r select 0
assert_match "*ziplist*" [r debug object hash] assert_match "*listpack*" [r debug object hash]
assert_equal 2 [r hlen hash] assert_equal 2 [r hlen hash]
assert_match {v1 v2} [r hmget hash f1 f2] assert_match {v1 v2} [r hmget hash f1 f2]
} }
......
...@@ -43,32 +43,31 @@ test {corrupt payload: #7445 - without sanitize - 2} { ...@@ -43,32 +43,31 @@ test {corrupt payload: #7445 - without sanitize - 2} {
test {corrupt payload: hash with valid zip list header, invalid entry len} { test {corrupt payload: hash with valid zip list header, invalid entry len} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no catch {
r restore key 0 "\x0D\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x14\x63\x00\x04\x02\x64\x00\xFF\x09\x00\xD9\x10\x54\x92\x15\xF5\x5F\x52" r restore key 0 "\x0D\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x14\x63\x00\x04\x02\x64\x00\xFF\x09\x00\xD9\x10\x54\x92\x15\xF5\x5F\x52"
r config set hash-max-ziplist-entries 1 } err
catch {r hset key b b} assert_match "*Bad data format*" $err
verify_log_message 0 "*zipEntrySafe*" 0 verify_log_message 0 "*integrity check failed*" 0
} }
} }
test {corrupt payload: invalid zlbytes header} { test {corrupt payload: invalid zlbytes header} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no
catch { catch {
r restore key 0 "\x0D\x1B\x25\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x02\x63\x00\x04\x02\x64\x00\xFF\x09\x00\xB7\xF7\x6E\x9F\x43\x43\x14\xC6" r restore key 0 "\x0D\x1B\x25\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x02\x63\x00\x04\x02\x64\x00\xFF\x09\x00\xB7\xF7\x6E\x9F\x43\x43\x14\xC6"
} err } err
assert_match "*Bad data format*" $err assert_match "*Bad data format*" $err
verify_log_message 0 "*integrity check failed*" 0
} }
} }
test {corrupt payload: valid zipped hash header, dup records} { test {corrupt payload: valid zipped hash header, dup records} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no catch {
r restore key 0 "\x0D\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x02\x61\x00\x04\x02\x64\x00\xFF\x09\x00\xA1\x98\x36\x78\xCC\x8E\x93\x2E" r restore key 0 "\x0D\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x02\x61\x00\x04\x02\x64\x00\xFF\x09\x00\xA1\x98\x36\x78\xCC\x8E\x93\x2E"
r config set hash-max-ziplist-entries 1 } err
# cause an assertion when converting to hash table assert_match "*Bad data format*" $err
catch {r hset key b b} verify_log_message 0 "*integrity check failed*" 0
verify_log_message 0 "*ziplist with dup elements dump*" 0
} }
} }
...@@ -235,6 +234,16 @@ test {corrupt payload: hash ziplist with duplicate records} { ...@@ -235,6 +234,16 @@ test {corrupt payload: hash ziplist with duplicate records} {
} }
} }
test {corrupt payload: hash listpack with duplicate records} {
# when we do perform full sanitization, we expect duplicate records to fail the restore
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload yes
r debug set-skip-checksum-validation 1
catch { r RESTORE _hash 0 "\x10\x17\x17\x00\x00\x00\x04\x00\x82a\x00\x03\x82b\x00\x03\x82a\x00\x03\x82d\x00\x03\xff\n\x00\xc0\xcf\xa6\x87\xe5\xa7\xc5\xbe" } err
assert_match "*Bad data format*" $err
}
}
test {corrupt payload: hash ziplist uneven record count} { test {corrupt payload: hash ziplist uneven record count} {
# when we do perform full sanitization, we expect duplicate records to fail the restore # when we do perform full sanitization, we expect duplicate records to fail the restore
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
...@@ -304,16 +313,14 @@ test {corrupt payload: fuzzer findings - NPD in quicklistIndex} { ...@@ -304,16 +313,14 @@ test {corrupt payload: fuzzer findings - NPD in quicklistIndex} {
} }
} }
test {corrupt payload: fuzzer findings - invalid read in ziplistFind} { test {corrupt payload: fuzzer findings - encoded entry header reach outside the allocation} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no
r debug set-skip-checksum-validation 1 r debug set-skip-checksum-validation 1
catch { catch {
r RESTORE key 0 "\x0D\x19\x19\x00\x00\x00\x16\x00\x00\x00\x06\x00\x00\xF1\x02\xF1\x02\xF2\x02\x02\x5F\x31\x04\x99\x02\xF3\xFF\x09\x00\xC5\xB8\x10\xC0\x8A\xF9\x16\xDF" r RESTORE key 0 "\x0D\x19\x19\x00\x00\x00\x16\x00\x00\x00\x06\x00\x00\xF1\x02\xF1\x02\xF2\x02\x02\x5F\x31\x04\x99\x02\xF3\xFF\x09\x00\xC5\xB8\x10\xC0\x8A\xF9\x16\xDF"
r HEXISTS key -688319650333 } err
} assert_match "*Bad data format*" $err
assert_equal [count_log_message 0 "crashed by signal"] 0 verify_log_message 0 "*integrity check failed*" 0
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
} }
} }
...@@ -342,12 +349,12 @@ test {corrupt payload: fuzzer findings - hash crash} { ...@@ -342,12 +349,12 @@ test {corrupt payload: fuzzer findings - hash crash} {
test {corrupt payload: fuzzer findings - uneven entry count in hash} { test {corrupt payload: fuzzer findings - uneven entry count in hash} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no
r debug set-skip-checksum-validation 1 r debug set-skip-checksum-validation 1
r RESTORE _hashbig 0 "\x0D\x3D\x3D\x00\x00\x00\x38\x00\x00\x00\x14\x00\x00\xF2\x02\x02\x5F\x31\x04\x1C\x02\xF7\x02\xF1\x02\xF1\x02\xF5\x02\xF5\x02\xF4\x02\x02\x5F\x33\x04\xF6\x02\x02\x5F\x35\x04\xF8\x02\x02\x5F\x37\x04\xF9\x02\xF9\x02\xF3\x02\xF3\x02\xFA\x02\x02\x5F\x39\xFF\x09\x00\x73\xB7\x68\xC8\x97\x24\x8E\x88" catch {
catch { r HSCAN _hashbig -250 } r RESTORE _hashbig 0 "\x0D\x3D\x3D\x00\x00\x00\x38\x00\x00\x00\x14\x00\x00\xF2\x02\x02\x5F\x31\x04\x1C\x02\xF7\x02\xF1\x02\xF1\x02\xF5\x02\xF5\x02\xF4\x02\x02\x5F\x33\x04\xF6\x02\x02\x5F\x35\x04\xF8\x02\x02\x5F\x37\x04\xF9\x02\xF9\x02\xF3\x02\xF3\x02\xFA\x02\x02\x5F\x39\xFF\x09\x00\x73\xB7\x68\xC8\x97\x24\x8E\x88"
assert_equal [count_log_message 0 "crashed by signal"] 0 } err
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 assert_match "*Bad data format*" $err
verify_log_message 0 "*integrity check failed*" 0
} }
} }
...@@ -474,15 +481,14 @@ test {corrupt payload: fuzzer findings - infinite loop} { ...@@ -474,15 +481,14 @@ test {corrupt payload: fuzzer findings - infinite loop} {
} }
} }
test {corrupt payload: fuzzer findings - hash convert asserts on RESTORE with shallow sanitization} { test {corrupt payload: fuzzer findings - hash ziplist too long entry len} {
# if we don't perform full sanitization, and the next command can assert on converting
# a ziplist to hash records, then we're ok with that happning in RESTORE too
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r config set sanitize-dump-payload no
r debug set-skip-checksum-validation 1 r debug set-skip-checksum-validation 1
catch { r RESTORE _hash 0 "\x0D\x3D\x3D\x00\x00\x00\x3A\x00\x00\x00\x14\x13\x00\xF5\x02\xF5\x02\xF2\x02\x53\x5F\x31\x04\xF3\x02\xF3\x02\xF7\x02\xF7\x02\xF8\x02\x02\x5F\x37\x04\xF1\x02\xF1\x02\xF6\x02\x02\x5F\x35\x04\xF4\x02\x02\x5F\x33\x04\xFA\x02\x02\x5F\x39\x04\xF9\x02\xF9\xFF\x09\x00\xB5\x48\xDE\x62\x31\xD0\xE5\x63" } catch {
assert_equal [count_log_message 0 "crashed by signal"] 0 r RESTORE _hash 0 "\x0D\x3D\x3D\x00\x00\x00\x3A\x00\x00\x00\x14\x13\x00\xF5\x02\xF5\x02\xF2\x02\x53\x5F\x31\x04\xF3\x02\xF3\x02\xF7\x02\xF7\x02\xF8\x02\x02\x5F\x37\x04\xF1\x02\xF1\x02\xF6\x02\x02\x5F\x35\x04\xF4\x02\x02\x5F\x33\x04\xFA\x02\x02\x5F\x39\x04\xF9\x02\xF9\xFF\x09\x00\xB5\x48\xDE\x62\x31\xD0\xE5\x63"
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 } err
assert_match "*Bad data format*" $err
verify_log_message 0 "*integrity check failed*" 0
} }
} }
...@@ -688,5 +694,15 @@ test {corrupt payload: fuzzer findings - hash with len of 0} { ...@@ -688,5 +694,15 @@ test {corrupt payload: fuzzer findings - hash with len of 0} {
} }
} }
test {corrupt payload: fuzzer findings - hash listpack first element too long entry len} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
r debug set-skip-checksum-validation 1
r config set sanitize-dump-payload yes
catch { r restore _hash 0 "\x10\x15\x15\x00\x00\x00\x06\x00\xF0\x01\x00\x01\x01\x01\x82\x5F\x31\x03\x02\x01\x02\x01\xFF\x0A\x00\x94\x21\x0A\xFA\x06\x52\x9F\x44" replace } err
assert_match "*Bad data format*" $err
verify_log_message 0 "*integrity check failed*" 0
}
}
} ;# tags } ;# tags
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment