Unverified Commit f27083a4 authored by perryitay's avatar perryitay Committed by GitHub
Browse files

Add support for list type to store elements larger than 4GB (#9357)



Redis lists are stored in quicklist, which is currently a linked list of ziplists.
Ziplists are limited to storing elements no larger than 4GB, so when bigger
items are added they're getting truncated.
This PR changes quicklists so that they're capable of storing large items
in quicklist nodes that are plain string buffers rather than ziplist.

As part of the PR there were few other changes in redis: 
1. new DEBUG sub-commands: 
   - QUICKLIST-PACKED-THRESHOLD - set the threshold of for the node type to
     be plan or ziplist. default (1GB)
   - QUICKLIST <key> - Shows low level info about the quicklist encoding of <key>
2. rdb format change:
   - A new type was added - RDB_TYPE_LIST_QUICKLIST_2 . 
   - container type (packed / plain) was added to the beginning of the rdb object
     (before the actual node list).
3. testing:
   - Tests that requires over 100MB will be by default skipped. a new flag was
     added to 'runtest' to run the large memory tests (not used by default)
Co-authored-by: default avatarsundb <sundbcn@gmail.com>
Co-authored-by: default avatarOran Agra <oran@redislabs.com>
parent f11a2d4d
...@@ -254,7 +254,7 @@ jobs: ...@@ -254,7 +254,7 @@ jobs:
sudo apt-get install tcl8.6 tclx valgrind -y sudo apt-get install tcl8.6 tclx valgrind -y
- name: test - name: test
if: true && !contains(github.event.inputs.skiptests, 'redis') if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --valgrind --verbose --clients 1 --tags -large-memory --dump-logs ${{github.event.inputs.test_args}} run: ./runtest --valgrind --verbose --clients 1 --dump-logs ${{github.event.inputs.test_args}}
- name: module api test - name: module api test
if: true && !contains(github.event.inputs.skiptests, 'modules') if: true && !contains(github.event.inputs.skiptests, 'modules')
run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 ${{github.event.inputs.test_args}} run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 ${{github.event.inputs.test_args}}
...@@ -285,7 +285,7 @@ jobs: ...@@ -285,7 +285,7 @@ jobs:
sudo apt-get install tcl8.6 tclx valgrind -y sudo apt-get install tcl8.6 tclx valgrind -y
- name: test - name: test
if: true && !contains(github.event.inputs.skiptests, 'redis') if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --valgrind --verbose --clients 1 --tags -large-memory --dump-logs ${{github.event.inputs.test_args}} run: ./runtest --valgrind --verbose --clients 1 --dump-logs ${{github.event.inputs.test_args}}
- name: module api test - name: module api test
if: true && !contains(github.event.inputs.skiptests, 'modules') if: true && !contains(github.event.inputs.skiptests, 'modules')
run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 ${{github.event.inputs.test_args}} run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 ${{github.event.inputs.test_args}}
...@@ -420,7 +420,7 @@ jobs: ...@@ -420,7 +420,7 @@ jobs:
prepare: pkg install -y bash gmake lang/tcl86 lang/tclx prepare: pkg install -y bash gmake lang/tcl86 lang/tclx
run: > run: >
gmake || exit 1 ; gmake || exit 1 ;
if echo "${{github.event.inputs.skiptests}}" | grep -vq redis ; then ./runtest --accurate --verbose --no-latency --tags -large-memory --dump-logs ${{github.event.inputs.test_args}} || exit 1 ; fi ; if echo "${{github.event.inputs.skiptests}}" | grep -vq redis ; then ./runtest --accurate --verbose --no-latency --dump-logs ${{github.event.inputs.test_args}} || exit 1 ; fi ;
if echo "${{github.event.inputs.skiptests}}" | grep -vq modules ; then MAKE=gmake ./runtest-moduleapi --verbose ${{github.event.inputs.test_args}} || exit 1 ; fi ; if echo "${{github.event.inputs.skiptests}}" | grep -vq modules ; then MAKE=gmake ./runtest-moduleapi --verbose ${{github.event.inputs.test_args}} || exit 1 ; fi ;
if echo "${{github.event.inputs.skiptests}}" | grep -vq sentinel ; then ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} || exit 1 ; fi ; if echo "${{github.event.inputs.skiptests}}" | grep -vq sentinel ; then ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} || exit 1 ; fi ;
if echo "${{github.event.inputs.skiptests}}" | grep -vq cluster ; then ./runtest-cluster ${{github.event.inputs.cluster_test_args}} || exit 1 ; fi ; if echo "${{github.event.inputs.skiptests}}" | grep -vq cluster ; then ./runtest-cluster ${{github.event.inputs.cluster_test_args}} || exit 1 ; fi ;
......
...@@ -29,9 +29,11 @@ ...@@ -29,9 +29,11 @@
*/ */
#include "server.h" #include "server.h"
#include "util.h"
#include "sha1.h" /* SHA1 is used for DEBUG DIGEST */ #include "sha1.h" /* SHA1 is used for DEBUG DIGEST */
#include "crc64.h" #include "crc64.h"
#include "bio.h" #include "bio.h"
#include "quicklist.h"
#include <arpa/inet.h> #include <arpa/inet.h>
#include <signal.h> #include <signal.h>
...@@ -459,6 +461,9 @@ void debugCommand(client *c) { ...@@ -459,6 +461,9 @@ void debugCommand(client *c) {
" Setting it to 0 disables expiring keys in background when they are not", " Setting it to 0 disables expiring keys in background when they are not",
" accessed (otherwise the Redis behavior). Setting it to 1 reenables back the", " accessed (otherwise the Redis behavior). Setting it to 1 reenables back the",
" default.", " default.",
"QUICKLIST-PACKED-THRESHOLD <size>",
" Sets the threshold for elements to be inserted as plain vs packed nodes",
" Default value is 1GB, allows values up to 4GB",
"SET-SKIP-CHECKSUM-VALIDATION <0|1>", "SET-SKIP-CHECKSUM-VALIDATION <0|1>",
" Enables or disables checksum checks for RDB files and RESTORE's payload.", " Enables or disables checksum checks for RDB files and RESTORE's payload.",
"SLEEP <seconds>", "SLEEP <seconds>",
...@@ -469,6 +474,9 @@ void debugCommand(client *c) { ...@@ -469,6 +474,9 @@ void debugCommand(client *c) {
" Return the size of different Redis core C structures.", " Return the size of different Redis core C structures.",
"ZIPLIST <key>", "ZIPLIST <key>",
" Show low level info about the ziplist encoding of <key>.", " Show low level info about the ziplist encoding of <key>.",
"QUICKLIST <key> [<0|1>]",
" Show low level info about the quicklist encoding of <key>."
" The optional argument (0 by default) sets the level of detail",
"CLIENT-EVICTION", "CLIENT-EVICTION",
" Show low level client eviction pools info (maxmemory-clients).", " Show low level client eviction pools info (maxmemory-clients).",
"PAUSE-CRON <0|1>", "PAUSE-CRON <0|1>",
...@@ -652,6 +660,21 @@ NULL ...@@ -652,6 +660,21 @@ NULL
ziplistRepr(o->ptr); ziplistRepr(o->ptr);
addReplyStatus(c,"Ziplist structure printed on stdout"); addReplyStatus(c,"Ziplist structure printed on stdout");
} }
} else if (!strcasecmp(c->argv[1]->ptr,"quicklist") && (c->argc == 3 || c->argc == 4)) {
robj *o;
if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.nokeyerr))
== NULL) return;
int full = 0;
if (c->argc == 4)
full = atoi(c->argv[3]->ptr);
if (o->encoding != OBJ_ENCODING_QUICKLIST) {
addReplyError(c,"Not a quicklist encoded object.");
} else {
quicklistRepr(o->ptr, full);
addReplyStatus(c,"Quicklist structure printed on stdout");
}
} else if (!strcasecmp(c->argv[1]->ptr,"populate") && } else if (!strcasecmp(c->argv[1]->ptr,"populate") &&
c->argc >= 3 && c->argc <= 5) { c->argc >= 3 && c->argc <= 5) {
long keys, j; long keys, j;
...@@ -782,6 +805,16 @@ NULL ...@@ -782,6 +805,16 @@ NULL
{ {
server.active_expire_enabled = atoi(c->argv[2]->ptr); server.active_expire_enabled = atoi(c->argv[2]->ptr);
addReply(c,shared.ok); addReply(c,shared.ok);
} else if (!strcasecmp(c->argv[1]->ptr,"quicklist-packed-threshold") &&
c->argc == 3)
{
int memerr;
unsigned long long sz = memtoull((const char *)c->argv[2]->ptr, &memerr);
if (memerr || !quicklistisSetPackedThreshold(sz)) {
addReplyError(c, "argument must be a memory value bigger then 1 and smaller than 4gb");
} else {
addReply(c,shared.ok);
}
} else if (!strcasecmp(c->argv[1]->ptr,"set-skip-checksum-validation") && } else if (!strcasecmp(c->argv[1]->ptr,"set-skip-checksum-validation") &&
c->argc == 3) c->argc == 3)
{ {
......
...@@ -416,8 +416,8 @@ long activeDefragQuickListNode(quicklist *ql, quicklistNode **node_ref) { ...@@ -416,8 +416,8 @@ long activeDefragQuickListNode(quicklist *ql, quicklistNode **node_ref) {
*node_ref = node = newnode; *node_ref = node = newnode;
defragged++; defragged++;
} }
if ((newzl = activeDefragAlloc(node->zl))) if ((newzl = activeDefragAlloc(node->entry)))
defragged++, node->zl = newzl; defragged++, node->entry = newzl;
return defragged; return defragged;
} }
......
...@@ -415,9 +415,9 @@ void dismissListObject(robj *o, size_t size_hint) { ...@@ -415,9 +415,9 @@ void dismissListObject(robj *o, size_t size_hint) {
quicklistNode *node = ql->head; quicklistNode *node = ql->head;
while (node) { while (node) {
if (quicklistNodeIsCompressed(node)) { if (quicklistNodeIsCompressed(node)) {
dismissMemory(node->zl, ((quicklistLZF*)node->zl)->sz); dismissMemory(node->entry, ((quicklistLZF*)node->entry)->sz);
} else { } else {
dismissMemory(node->zl, node->sz); dismissMemory(node->entry, node->sz);
} }
node = node->next; node = node->next;
} }
...@@ -1000,7 +1000,7 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) { ...@@ -1000,7 +1000,7 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) {
quicklistNode *node = ql->head; quicklistNode *node = ql->head;
asize = sizeof(*o)+sizeof(quicklist); asize = sizeof(*o)+sizeof(quicklist);
do { do {
elesize += sizeof(quicklistNode)+zmalloc_size(node->zl); elesize += sizeof(quicklistNode)+zmalloc_size(node->entry);
samples++; samples++;
} while ((node = node->next) && samples < sample_size); } while ((node = node->next) && samples < sample_size);
asize += (double)elesize/samples*ql->len; asize += (double)elesize/samples*ql->len;
......
This diff is collapsed.
...@@ -46,8 +46,8 @@ ...@@ -46,8 +46,8 @@
typedef struct quicklistNode { typedef struct quicklistNode {
struct quicklistNode *prev; struct quicklistNode *prev;
struct quicklistNode *next; struct quicklistNode *next;
unsigned char *zl; unsigned char *entry;
unsigned int sz; /* ziplist size in bytes */ size_t sz; /* entry size in bytes */
unsigned int count : 16; /* count of items in ziplist */ unsigned int count : 16; /* count of items in ziplist */
unsigned int encoding : 2; /* RAW==1 or LZF==2 */ unsigned int encoding : 2; /* RAW==1 or LZF==2 */
unsigned int container : 2; /* NONE==1 or ZIPLIST==2 */ unsigned int container : 2; /* NONE==1 or ZIPLIST==2 */
...@@ -56,13 +56,13 @@ typedef struct quicklistNode { ...@@ -56,13 +56,13 @@ typedef struct quicklistNode {
unsigned int extra : 10; /* more bits to steal for future usage */ unsigned int extra : 10; /* more bits to steal for future usage */
} quicklistNode; } quicklistNode;
/* quicklistLZF is a 4+N byte struct holding 'sz' followed by 'compressed'. /* quicklistLZF is a 8+N byte struct holding 'sz' followed by 'compressed'.
* 'sz' is byte length of 'compressed' field. * 'sz' is byte length of 'compressed' field.
* 'compressed' is LZF data with total (compressed) length 'sz' * 'compressed' is LZF data with total (compressed) length 'sz'
* NOTE: uncompressed length is stored in quicklistNode->sz. * NOTE: uncompressed length is stored in quicklistNode->sz.
* When quicklistNode->zl is compressed, node->zl points to a quicklistLZF */ * When quicklistNode->entry is compressed, node->entry points to a quicklistLZF */
typedef struct quicklistLZF { typedef struct quicklistLZF {
unsigned int sz; /* LZF size in bytes*/ size_t sz; /* LZF size in bytes*/
char compressed[]; char compressed[];
} quicklistLZF; } quicklistLZF;
...@@ -127,7 +127,7 @@ typedef struct quicklistEntry { ...@@ -127,7 +127,7 @@ typedef struct quicklistEntry {
unsigned char *zi; unsigned char *zi;
unsigned char *value; unsigned char *value;
long long longval; long long longval;
unsigned int sz; size_t sz;
int offset; int offset;
} quicklistEntry; } quicklistEntry;
...@@ -142,9 +142,11 @@ typedef struct quicklistEntry { ...@@ -142,9 +142,11 @@ typedef struct quicklistEntry {
#define QUICKLIST_NOCOMPRESS 0 #define QUICKLIST_NOCOMPRESS 0
/* quicklist container formats */ /* quicklist container formats */
#define QUICKLIST_NODE_CONTAINER_NONE 1 #define QUICKLIST_NODE_CONTAINER_PLAIN 1
#define QUICKLIST_NODE_CONTAINER_ZIPLIST 2 #define QUICKLIST_NODE_CONTAINER_ZIPLIST 2
#define QL_NODE_IS_PLAIN(node) ((node)->container == QUICKLIST_NODE_CONTAINER_PLAIN)
#define quicklistNodeIsCompressed(node) \ #define quicklistNodeIsCompressed(node) \
((node)->encoding == QUICKLIST_NODE_ENCODING_LZF) ((node)->encoding == QUICKLIST_NODE_ENCODING_LZF)
...@@ -160,6 +162,7 @@ int quicklistPushTail(quicklist *quicklist, void *value, const size_t sz); ...@@ -160,6 +162,7 @@ int quicklistPushTail(quicklist *quicklist, void *value, const size_t sz);
void quicklistPush(quicklist *quicklist, void *value, const size_t sz, void quicklistPush(quicklist *quicklist, void *value, const size_t sz,
int where); int where);
void quicklistAppendZiplist(quicklist *quicklist, unsigned char *zl); void quicklistAppendZiplist(quicklist *quicklist, unsigned char *zl);
void quicklistAppendPlainNode(quicklist *quicklist, unsigned char *data, size_t sz);
quicklist *quicklistAppendValuesFromZiplist(quicklist *quicklist, quicklist *quicklistAppendValuesFromZiplist(quicklist *quicklist,
unsigned char *zl); unsigned char *zl);
quicklist *quicklistCreateFromZiplist(int fill, int compress, quicklist *quicklistCreateFromZiplist(int fill, int compress,
...@@ -170,9 +173,9 @@ void quicklistInsertBefore(quicklist *quicklist, quicklistEntry *entry, ...@@ -170,9 +173,9 @@ void quicklistInsertBefore(quicklist *quicklist, quicklistEntry *entry,
void *value, const size_t sz); void *value, const size_t sz);
void quicklistDelEntry(quicklistIter *iter, quicklistEntry *entry); void quicklistDelEntry(quicklistIter *iter, quicklistEntry *entry);
void quicklistReplaceEntry(quicklist *quicklist, quicklistEntry *entry, void quicklistReplaceEntry(quicklist *quicklist, quicklistEntry *entry,
void *data, int sz); void *data, size_t sz);
int quicklistReplaceAtIndex(quicklist *quicklist, long index, void *data, int quicklistReplaceAtIndex(quicklist *quicklist, long index, void *data,
int sz); const size_t sz);
int quicklistDelRange(quicklist *quicklist, const long start, const long stop); int quicklistDelRange(quicklist *quicklist, const long start, const long stop);
quicklistIter *quicklistGetIterator(const quicklist *quicklist, int direction); quicklistIter *quicklistGetIterator(const quicklist *quicklist, int direction);
quicklistIter *quicklistGetIteratorAtIdx(const quicklist *quicklist, quicklistIter *quicklistGetIteratorAtIdx(const quicklist *quicklist,
...@@ -185,19 +188,21 @@ int quicklistIndex(const quicklist *quicklist, const long long index, ...@@ -185,19 +188,21 @@ int quicklistIndex(const quicklist *quicklist, const long long index,
quicklistEntry *entry); quicklistEntry *entry);
void quicklistRotate(quicklist *quicklist); void quicklistRotate(quicklist *quicklist);
int quicklistPopCustom(quicklist *quicklist, int where, unsigned char **data, int quicklistPopCustom(quicklist *quicklist, int where, unsigned char **data,
unsigned int *sz, long long *sval, size_t *sz, long long *sval,
void *(*saver)(unsigned char *data, unsigned int sz)); void *(*saver)(unsigned char *data, size_t sz));
int quicklistPop(quicklist *quicklist, int where, unsigned char **data, int quicklistPop(quicklist *quicklist, int where, unsigned char **data,
unsigned int *sz, long long *slong); size_t *sz, long long *slong);
unsigned long quicklistCount(const quicklist *ql); unsigned long quicklistCount(const quicklist *ql);
int quicklistCompare(unsigned char *p1, unsigned char *p2, int p2_len); int quicklistCompare(quicklistEntry *entry, unsigned char *p2, const size_t p2_len);
size_t quicklistGetLzf(const quicklistNode *node, void **data); size_t quicklistGetLzf(const quicklistNode *node, void **data);
void quicklistRepr(unsigned char *ql, int full);
/* bookmarks */ /* bookmarks */
int quicklistBookmarkCreate(quicklist **ql_ref, const char *name, quicklistNode *node); int quicklistBookmarkCreate(quicklist **ql_ref, const char *name, quicklistNode *node);
int quicklistBookmarkDelete(quicklist *ql, const char *name); int quicklistBookmarkDelete(quicklist *ql, const char *name);
quicklistNode *quicklistBookmarkFind(quicklist *ql, const char *name); quicklistNode *quicklistBookmarkFind(quicklist *ql, const char *name);
void quicklistBookmarksClear(quicklist *ql); void quicklistBookmarksClear(quicklist *ql);
int quicklistisSetPackedThreshold(size_t sz);
#ifdef REDIS_TEST #ifdef REDIS_TEST
int quicklistTest(int argc, char *argv[], int accurate); int quicklistTest(int argc, char *argv[], int accurate);
......
...@@ -658,7 +658,7 @@ int rdbSaveObjectType(rio *rdb, robj *o) { ...@@ -658,7 +658,7 @@ int rdbSaveObjectType(rio *rdb, robj *o) {
return rdbSaveType(rdb,RDB_TYPE_STRING); return rdbSaveType(rdb,RDB_TYPE_STRING);
case OBJ_LIST: case OBJ_LIST:
if (o->encoding == OBJ_ENCODING_QUICKLIST) if (o->encoding == OBJ_ENCODING_QUICKLIST)
return rdbSaveType(rdb,RDB_TYPE_LIST_QUICKLIST); return rdbSaveType(rdb, RDB_TYPE_LIST_QUICKLIST_2);
else else
serverPanic("Unknown list encoding"); serverPanic("Unknown list encoding");
case OBJ_SET: case OBJ_SET:
...@@ -813,13 +813,16 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) { ...@@ -813,13 +813,16 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key, int dbid) {
nwritten += n; nwritten += n;
while(node) { while(node) {
if ((n = rdbSaveLen(rdb,node->container)) == -1) return -1;
nwritten += n;
if (quicklistNodeIsCompressed(node)) { if (quicklistNodeIsCompressed(node)) {
void *data; void *data;
size_t compress_len = quicklistGetLzf(node, &data); size_t compress_len = quicklistGetLzf(node, &data);
if ((n = rdbSaveLzfBlob(rdb,data,compress_len,node->sz)) == -1) return -1; if ((n = rdbSaveLzfBlob(rdb,data,compress_len,node->sz)) == -1) return -1;
nwritten += n; nwritten += n;
} else { } else {
if ((n = rdbSaveRawString(rdb,node->zl,node->sz)) == -1) return -1; if ((n = rdbSaveRawString(rdb,node->entry,node->sz)) == -1) return -1;
nwritten += n; nwritten += n;
} }
node = node->next; node = node->next;
...@@ -1934,36 +1937,58 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) { ...@@ -1934,36 +1937,58 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int dbid, int *error) {
/* All pairs should be read by now */ /* All pairs should be read by now */
serverAssert(len == 0); serverAssert(len == 0);
} else if (rdbtype == RDB_TYPE_LIST_QUICKLIST) { } else if (rdbtype == RDB_TYPE_LIST_QUICKLIST || rdbtype == RDB_TYPE_LIST_QUICKLIST_2) {
if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL; if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL;
if (len == 0) goto emptykey; if (len == 0) goto emptykey;
o = createQuicklistObject(); o = createQuicklistObject();
quicklistSetOptions(o->ptr, server.list_max_ziplist_size, quicklistSetOptions(o->ptr, server.list_max_ziplist_size,
server.list_compress_depth); server.list_compress_depth);
uint64_t container = QUICKLIST_NODE_CONTAINER_ZIPLIST;
while (len--) { while (len--) {
size_t encoded_len; size_t encoded_len;
unsigned char *zl =
if (rdbtype == RDB_TYPE_LIST_QUICKLIST_2) {
if ((container = rdbLoadLen(rdb,NULL)) == RDB_LENERR) {
decrRefCount(o);
return NULL;
}
if (container != QUICKLIST_NODE_CONTAINER_ZIPLIST && container != QUICKLIST_NODE_CONTAINER_PLAIN) {
rdbReportCorruptRDB("Quicklist integrity check failed.");
decrRefCount(o);
return NULL;
}
}
unsigned char *data =
rdbGenericLoadStringObject(rdb,RDB_LOAD_PLAIN,&encoded_len); rdbGenericLoadStringObject(rdb,RDB_LOAD_PLAIN,&encoded_len);
if (zl == NULL) { if (data == NULL || (encoded_len == 0)) {
zfree(data);
decrRefCount(o); decrRefCount(o);
return NULL; return NULL;
} }
if (container == QUICKLIST_NODE_CONTAINER_PLAIN) {
quicklistAppendPlainNode(o->ptr, data, encoded_len);
zfree(data);
continue;
}
if (deep_integrity_validation) server.stat_dump_payload_sanitizations++; if (deep_integrity_validation) server.stat_dump_payload_sanitizations++;
if (!ziplistValidateIntegrity(zl, encoded_len, deep_integrity_validation, NULL, NULL)) { if (!ziplistValidateIntegrity(data, encoded_len, deep_integrity_validation, NULL, NULL)) {
rdbReportCorruptRDB("Ziplist integrity check failed."); rdbReportCorruptRDB("Ziplist integrity check failed.");
decrRefCount(o); decrRefCount(o);
zfree(zl); zfree(data);
return NULL; return NULL;
} }
/* Silently skip empty ziplists, if we'll end up with empty quicklist we'll fail later. */ /* Silently skip empty ziplists, if we'll end up with empty quicklist we'll fail later. */
if (ziplistLen(zl) == 0) { if (ziplistLen(data) == 0) {
zfree(zl); zfree(data);
continue; continue;
} else { } else {
quicklistAppendZiplist(o->ptr, zl); quicklistAppendZiplist(o->ptr, data);
} }
} }
......
...@@ -93,10 +93,11 @@ ...@@ -93,10 +93,11 @@
#define RDB_TYPE_STREAM_LISTPACKS 15 #define RDB_TYPE_STREAM_LISTPACKS 15
#define RDB_TYPE_HASH_LISTPACK 16 #define RDB_TYPE_HASH_LISTPACK 16
#define RDB_TYPE_ZSET_LISTPACK 17 #define RDB_TYPE_ZSET_LISTPACK 17
#define RDB_TYPE_LIST_QUICKLIST_2 18
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */ /* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */
/* Test if a type is an object type. */ /* Test if a type is an object type. */
#define rdbIsObjectType(t) ((t >= 0 && t <= 7) || (t >= 9 && t <= 17)) #define rdbIsObjectType(t) ((t >= 0 && t <= 7) || (t >= 9 && t <= 18))
/* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */ /* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */
#define RDB_OPCODE_MODULE_AUX 247 /* Module auxiliary data. */ #define RDB_OPCODE_MODULE_AUX 247 /* Module auxiliary data. */
......
...@@ -92,7 +92,8 @@ char *rdb_type_string[] = { ...@@ -92,7 +92,8 @@ char *rdb_type_string[] = {
"quicklist", "quicklist",
"stream", "stream",
"hash-listpack", "hash-listpack",
"zset-listpack" "zset-listpack",
"quicklist-v2"
}; };
/* Show a few stats collected into 'rdbstate' */ /* Show a few stats collected into 'rdbstate' */
......
...@@ -29,8 +29,6 @@ ...@@ -29,8 +29,6 @@
#include "server.h" #include "server.h"
#define LIST_MAX_ITEM_SIZE ((1ull<<32)-1024)
/*----------------------------------------------------------------------------- /*-----------------------------------------------------------------------------
* List API * List API
*----------------------------------------------------------------------------*/ *----------------------------------------------------------------------------*/
...@@ -55,7 +53,7 @@ void listTypePush(robj *subject, robj *value, int where) { ...@@ -55,7 +53,7 @@ void listTypePush(robj *subject, robj *value, int where) {
} }
} }
void *listPopSaver(unsigned char *data, unsigned int sz) { void *listPopSaver(unsigned char *data, size_t sz) {
return createStringObject((char*)data,sz); return createStringObject((char*)data,sz);
} }
...@@ -186,7 +184,7 @@ void listTypeReplace(listTypeEntry *entry, robj *value) { ...@@ -186,7 +184,7 @@ void listTypeReplace(listTypeEntry *entry, robj *value) {
int listTypeEqual(listTypeEntry *entry, robj *o) { int listTypeEqual(listTypeEntry *entry, robj *o) {
if (entry->li->encoding == OBJ_ENCODING_QUICKLIST) { if (entry->li->encoding == OBJ_ENCODING_QUICKLIST) {
serverAssertWithInfo(NULL,o,sdsEncodedObject(o)); serverAssertWithInfo(NULL,o,sdsEncodedObject(o));
return quicklistCompare(entry->entry.zi,o->ptr,sdslen(o->ptr)); return quicklistCompare(&entry->entry,o->ptr,sdslen(o->ptr));
} else { } else {
serverPanic("Unknown list encoding"); serverPanic("Unknown list encoding");
} }
...@@ -210,7 +208,7 @@ void listTypeConvert(robj *subject, int enc) { ...@@ -210,7 +208,7 @@ void listTypeConvert(robj *subject, int enc) {
size_t zlen = server.list_max_ziplist_size; size_t zlen = server.list_max_ziplist_size;
int depth = server.list_compress_depth; int depth = server.list_compress_depth;
subject->ptr = quicklistCreateFromZiplist(zlen, depth, subject->ptr); subject->ptr = quicklistCreateFromZiplist(zlen, depth, subject->ptr);
subject->encoding = OBJ_ENCODING_QUICKLIST; subject->encoding = enc;
} else { } else {
serverPanic("Unsupported list conversion"); serverPanic("Unsupported list conversion");
} }
...@@ -229,7 +227,7 @@ robj *listTypeDup(robj *o) { ...@@ -229,7 +227,7 @@ robj *listTypeDup(robj *o) {
switch (o->encoding) { switch (o->encoding) {
case OBJ_ENCODING_QUICKLIST: case OBJ_ENCODING_QUICKLIST:
lobj = createObject(OBJ_LIST, quicklistDup(o->ptr)); lobj = createObject(OBJ_LIST, quicklistDup(o->ptr));
lobj->encoding = OBJ_ENCODING_QUICKLIST; lobj->encoding = o->encoding;
break; break;
default: default:
serverPanic("Unknown list encoding"); serverPanic("Unknown list encoding");
...@@ -256,13 +254,6 @@ int listTypeDelRange(robj *subject, long start, long count) { ...@@ -256,13 +254,6 @@ int listTypeDelRange(robj *subject, long start, long count) {
void pushGenericCommand(client *c, int where, int xx) { void pushGenericCommand(client *c, int where, int xx) {
int j; int j;
for (j = 2; j < c->argc; j++) {
if (sdslen(c->argv[j]->ptr) > LIST_MAX_ITEM_SIZE) {
addReplyError(c, "Element too large");
return;
}
}
robj *lobj = lookupKeyWrite(c->db, c->argv[1]); robj *lobj = lookupKeyWrite(c->db, c->argv[1]);
if (checkType(c,lobj,OBJ_LIST)) return; if (checkType(c,lobj,OBJ_LIST)) return;
if (!lobj) { if (!lobj) {
...@@ -326,11 +317,6 @@ void linsertCommand(client *c) { ...@@ -326,11 +317,6 @@ void linsertCommand(client *c) {
return; return;
} }
if (sdslen(c->argv[4]->ptr) > LIST_MAX_ITEM_SIZE) {
addReplyError(c, "Element too large");
return;
}
if ((subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL || if ((subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL ||
checkType(c,subject,OBJ_LIST)) return; checkType(c,subject,OBJ_LIST)) return;
...@@ -398,11 +384,6 @@ void lsetCommand(client *c) { ...@@ -398,11 +384,6 @@ void lsetCommand(client *c) {
long index; long index;
robj *value = c->argv[3]; robj *value = c->argv[3];
if (sdslen(value->ptr) > LIST_MAX_ITEM_SIZE) {
addReplyError(c, "Element too large");
return;
}
if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != C_OK)) if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != C_OK))
return; return;
...@@ -702,11 +683,6 @@ void lposCommand(client *c) { ...@@ -702,11 +683,6 @@ void lposCommand(client *c) {
int direction = LIST_TAIL; int direction = LIST_TAIL;
long rank = 1, count = -1, maxlen = 0; /* Count -1: option not given. */ long rank = 1, count = -1, maxlen = 0; /* Count -1: option not given. */
if (sdslen(ele->ptr) > LIST_MAX_ITEM_SIZE) {
addReplyError(c, "Element too large");
return;
}
/* Parse the optional arguments. */ /* Parse the optional arguments. */
for (int j = 3; j < c->argc; j++) { for (int j = 3; j < c->argc; j++) {
char *opt = c->argv[j]->ptr; char *opt = c->argv[j]->ptr;
...@@ -802,11 +778,6 @@ void lremCommand(client *c) { ...@@ -802,11 +778,6 @@ void lremCommand(client *c) {
long toremove; long toremove;
long removed = 0; long removed = 0;
if (sdslen(obj->ptr) > LIST_MAX_ITEM_SIZE) {
addReplyError(c, "Element too large");
return;
}
if ((getLongFromObjectOrReply(c, c->argv[2], &toremove, NULL) != C_OK)) if ((getLongFromObjectOrReply(c, c->argv[2], &toremove, NULL) != C_OK))
return; return;
......
...@@ -17,6 +17,7 @@ match different external server configurations: ...@@ -17,6 +17,7 @@ match different external server configurations:
| `--ignore-encoding` | Skip all checks for specific encoding. | | `--ignore-encoding` | Skip all checks for specific encoding. |
| `--ignore-digest` | Skip key value digest validations. | | `--ignore-digest` | Skip key value digest validations. |
| `--cluster-mode` | Run in strict Redis Cluster compatibility mode. | | `--cluster-mode` | Run in strict Redis Cluster compatibility mode. |
| `--large-memory` | Enables tests that consume more than 100mb |
Tags Tags
---- ----
...@@ -36,6 +37,7 @@ The following compatibility and capability tags are currently used: ...@@ -36,6 +37,7 @@ The following compatibility and capability tags are currently used:
| --------------------- | --------- | | --------------------- | --------- |
| `external:skip` | Not compatible with external servers. | | `external:skip` | Not compatible with external servers. |
| `cluster:skip` | Not compatible with `--cluster-mode`. | | `cluster:skip` | Not compatible with `--cluster-mode`. |
| `large-memory` | Test that requires more than 100mb |
| `tls:skip` | Not campatible with `--tls`. | | `tls:skip` | Not campatible with `--tls`. |
| `needs:repl` | Uses replication and needs to be able to `SYNC` from server. | | `needs:repl` | Uses replication and needs to be able to `SYNC` from server. |
| `needs:debug` | Uses the `DEBUG` command or other debugging focused commands (like `OBJECT`). | | `needs:debug` | Uses the `DEBUG` command or other debugging focused commands (like `OBJECT`). |
...@@ -51,6 +53,9 @@ When using an external server (`--host` and `--port`), filtering using the ...@@ -51,6 +53,9 @@ When using an external server (`--host` and `--port`), filtering using the
When using `--cluster-mode`, filtering using the `cluster:skip` tag is done When using `--cluster-mode`, filtering using the `cluster:skip` tag is done
automatically. automatically.
When not using `--large-memory`, filtering using the `largemem:skip` tag is done
automatically.
In addition, it is possible to specify additional configuration. For example, to In addition, it is possible to specify additional configuration. For example, to
run tests on a server that does not permit `SYNC` use: run tests on a server that does not permit `SYNC` use:
......
...@@ -135,6 +135,14 @@ test {corrupt payload: quicklist with empty ziplist} { ...@@ -135,6 +135,14 @@ test {corrupt payload: quicklist with empty ziplist} {
} }
} }
test {corrupt payload: quicklist encoded_len is 0} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
catch { r restore _list 0 "\x12\x01\x01\x00\n\x00\x8f\xc6\xc0W\x1c\n\xb3<" replace } err
assert_match "*Bad data format*" $err
r ping
}
}
test {corrupt payload: #3080 - ziplist} { test {corrupt payload: #3080 - ziplist} {
start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] {
# shallow sanitization is enough for restore to safely reject the payload with wrong size # shallow sanitization is enough for restore to safely reject the payload with wrong size
......
...@@ -4,6 +4,15 @@ set server_path [tmpdir "server.rdb-encoding-test"] ...@@ -4,6 +4,15 @@ set server_path [tmpdir "server.rdb-encoding-test"]
# Copy RDB with different encodings in server path # Copy RDB with different encodings in server path
exec cp tests/assets/encodings.rdb $server_path exec cp tests/assets/encodings.rdb $server_path
exec cp tests/assets/list-quicklist.rdb $server_path
start_server [list overrides [list "dir" $server_path "dbfilename" "list-quicklist.rdb"]] {
test "test old version rdb file" {
r select 0
assert_equal [r get x] 7
r lpop list
} {7}
}
start_server [list overrides [list "dir" $server_path "dbfilename" "encodings.rdb"]] { start_server [list overrides [list "dir" $server_path "dbfilename" "encodings.rdb"]] {
test "RDB encoding loading test" { test "RDB encoding loading test" {
......
...@@ -146,6 +146,10 @@ proc ::redis::__method__read {id fd} { ...@@ -146,6 +146,10 @@ proc ::redis::__method__read {id fd} {
::redis::redis_read_reply $id $fd ::redis::redis_read_reply $id $fd
} }
proc ::redis::__method__rawread {id fd len} {
return [read $fd $len]
}
proc ::redis::__method__write {id fd buf} { proc ::redis::__method__write {id fd buf} {
::redis::redis_write $fd $buf ::redis::redis_write $fd $buf
} }
......
...@@ -210,6 +210,11 @@ proc tags_acceptable {tags err_return} { ...@@ -210,6 +210,11 @@ proc tags_acceptable {tags err_return} {
return 0 return 0
} }
if {!$::large_memory && [lsearch $tags "large-memory"] >= 0} {
set err "large memory flag not provided"
return 0
}
return 1 return 1
} }
......
...@@ -127,6 +127,7 @@ set ::singledb 0 ...@@ -127,6 +127,7 @@ set ::singledb 0
set ::cluster_mode 0 set ::cluster_mode 0
set ::ignoreencoding 0 set ::ignoreencoding 0
set ::ignoredigest 0 set ::ignoredigest 0
set ::large_memory 0
# Set to 1 when we are running in client mode. The Redis test uses a # Set to 1 when we are running in client mode. The Redis test uses a
# server-client model to run tests simultaneously. The server instance # server-client model to run tests simultaneously. The server instance
...@@ -592,6 +593,7 @@ proc print_help_screen {} { ...@@ -592,6 +593,7 @@ proc print_help_screen {} {
"--cluster-mode Run tests in cluster protocol compatible mode." "--cluster-mode Run tests in cluster protocol compatible mode."
"--ignore-encoding Don't validate object encoding." "--ignore-encoding Don't validate object encoding."
"--ignore-digest Don't use debug digest validations." "--ignore-digest Don't use debug digest validations."
"--large-memory Run tests using over 100mb."
"--help Print this help screen." "--help Print this help screen."
} "\n"] } "\n"]
} }
...@@ -703,6 +705,8 @@ for {set j 0} {$j < [llength $argv]} {incr j} { ...@@ -703,6 +705,8 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
} elseif {$opt eq {--cluster-mode}} { } elseif {$opt eq {--cluster-mode}} {
set ::cluster_mode 1 set ::cluster_mode 1
set ::singledb 1 set ::singledb 1
} elseif {$opt eq {--large-memory}} {
set ::large_memory 1
} elseif {$opt eq {--ignore-encoding}} { } elseif {$opt eq {--ignore-encoding}} {
set ::ignoreencoding 1 set ::ignoreencoding 1
} elseif {$opt eq {--ignore-digest}} { } elseif {$opt eq {--ignore-digest}} {
......
set ::str500 [string repeat x 500000000] ;# 500mb
# Utility function to write big argument into redis client connection
proc write_big_bulk {size} {
r write "\$$size\r\n"
while {$size >= 500000000} {
r write $::str500
incr size -500000000
}
if {$size > 0} {
r write [string repeat x $size]
}
r write "\r\n"
r flush
r read
}
# Utility to read big bulk response (work around Tcl limitations)
proc read_big_bulk {code} {
r readraw 1
set resp_len [uplevel 1 $code] ;# get the first line of the RESP response
assert_equal [string range $resp_len 0 0] "$"
set resp_len [string range $resp_len 1 end]
set remaining $resp_len
while {$remaining > 0} {
set l $remaining
if {$l > 2147483647} {set l 2147483647}
set nbytes [string length [r rawread $l]]
incr remaining [expr {- $nbytes}]
}
assert_equal [r rawread 2] "\r\n"
r readraw 0
return $resp_len
}
# check functionality compression of plain and zipped nodes
start_server [list overrides [list save ""] ] {
r config set list-compress-depth 2
r config set list-max-ziplist-size 1
# 3 test to check compression with regular ziplist nodes
# 1. using push + insert
# 2. using push + insert + trim
# 3. using push + insert + set
test {reg node check compression with insert and pop} {
r lpush list1 [string repeat a 500]
r lpush list1 [string repeat b 500]
r lpush list1 [string repeat c 500]
r lpush list1 [string repeat d 500]
r linsert list1 after [string repeat d 500] [string repeat e 500]
r linsert list1 after [string repeat d 500] [string repeat f 500]
r linsert list1 after [string repeat d 500] [string repeat g 500]
r linsert list1 after [string repeat d 500] [string repeat j 500]
assert_equal [r lpop list1] [string repeat d 500]
assert_equal [r lpop list1] [string repeat j 500]
assert_equal [r lpop list1] [string repeat g 500]
assert_equal [r lpop list1] [string repeat f 500]
assert_equal [r lpop list1] [string repeat e 500]
assert_equal [r lpop list1] [string repeat c 500]
assert_equal [r lpop list1] [string repeat b 500]
assert_equal [r lpop list1] [string repeat a 500]
};
test {reg node check compression combined with trim} {
r lpush list2 [string repeat a 500]
r linsert list2 after [string repeat a 500] [string repeat b 500]
r rpush list2 [string repeat c 500]
assert_equal [string repeat b 500] [r lindex list2 1]
r LTRIM list2 1 -1
r llen list2
} {2}
test {reg node check compression with lset} {
r lpush list3 [string repeat a 500]
r LSET list3 0 [string repeat b 500]
assert_equal [string repeat b 500] [r lindex list3 0]
r lpush list3 [string repeat c 500]
r LSET list3 0 [string repeat d 500]
assert_equal [string repeat d 500] [r lindex list3 0]
}
# repeating the 3 tests with plain nodes
# (by adjusting quicklist-packed-threshold)
test {plain node check compression} {
r debug quicklist-packed-threshold 1b
r lpush list4 [string repeat a 500]
r lpush list4 [string repeat b 500]
r lpush list4 [string repeat c 500]
r lpush list4 [string repeat d 500]
r linsert list4 after [string repeat d 500] [string repeat e 500]
r linsert list4 after [string repeat d 500] [string repeat f 500]
r linsert list4 after [string repeat d 500] [string repeat g 500]
r linsert list4 after [string repeat d 500] [string repeat j 500]
assert_equal [r lpop list4] [string repeat d 500]
assert_equal [r lpop list4] [string repeat j 500]
assert_equal [r lpop list4] [string repeat g 500]
assert_equal [r lpop list4] [string repeat f 500]
assert_equal [r lpop list4] [string repeat e 500]
assert_equal [r lpop list4] [string repeat c 500]
assert_equal [r lpop list4] [string repeat b 500]
assert_equal [r lpop list4] [string repeat a 500]
} {} {needs:debug}
test {plain node check compression with ltrim} {
r debug quicklist-packed-threshold 1b
r lpush list5 [string repeat a 500]
r linsert list5 after [string repeat a 500] [string repeat b 500]
r rpush list5 [string repeat c 500]
assert_equal [string repeat b 500] [r lindex list5 1]
r LTRIM list5 1 -1
r llen list5
} {2} {needs:debug}
test {plain node check compression using lset} {
r debug quicklist-packed-threshold 1b
r lpush list6 [string repeat a 500]
r LSET list6 0 [string repeat b 500]
assert_equal [string repeat b 500] [r lindex list6 0]
r lpush list6 [string repeat c 500]
r LSET list6 0 [string repeat d 500]
assert_equal [string repeat d 500] [r lindex list6 0]
} {} {needs:debug}
}
# check functionality of plain nodes using low packed-threshold
start_server [list overrides [list save ""] ] {
# basic command check for plain nodes - "LPUSH & LPOP"
test {Test LPUSH and LPOP on plain nodes} {
r flushdb
r debug quicklist-packed-threshold 1b
r lpush lst 9
r lpush lst xxxxxxxxxx
r lpush lst xxxxxxxxxx
set s0 [s used_memory]
assert {$s0 > 10}
assert {[r llen lst] == 3}
set s0 [r rpop lst]
set s1 [r rpop lst]
assert {$s0 eq "9"}
assert {[r llen lst] == 1}
r lpop lst
assert {[string length $s1] == 10}
# check rdb
r lpush lst xxxxxxxxxx
r lpush lst bb
r debug reload
assert_equal [r rpop lst] "xxxxxxxxxx"
} {} {needs:debug}
# basic command check for plain nodes - "LINDEX & LINSERT"
test {Test LINDEX and LINSERT on plain nodes} {
r flushdb
r debug quicklist-packed-threshold 1b
r lpush lst xxxxxxxxxxx
r lpush lst 9
r lpush lst xxxxxxxxxxx
r linsert lst before "9" "8"
assert {[r lindex lst 1] eq "8"}
r linsert lst BEFORE "9" "7"
r linsert lst BEFORE "9" "xxxxxxxxxxx"
assert {[r lindex lst 3] eq "xxxxxxxxxxx"}
} {} {needs:debug}
# basic command check for plain nodes - "LTRIM"
test {Test LTRIM on plain nodes} {
r flushdb
r debug quicklist-packed-threshold 1b
r lpush lst1 9
r lpush lst1 xxxxxxxxxxx
r lpush lst1 9
r LTRIM lst1 1 -1
assert_equal [r llen lst1] 2
} {} {needs:debug}
# basic command check for plain nodes - "LREM"
test {Test LREM on plain nodes} {
r flushdb
r debug quicklist-packed-threshold 1b
r lpush lst one
r lpush lst xxxxxxxxxxx
set s0 [s used_memory]
assert {$s0 > 10}
r lpush lst 9
r LREM lst -2 "one"
assert_equal [r llen lst] 2
} {} {needs:debug}
# basic command check for plain nodes - "LPOS"
test {Test LPOS on plain nodes} {
r flushdb
r debug quicklist-packed-threshold 1b
r RPUSH lst "aa"
r RPUSH lst "bb"
r RPUSH lst "cc"
r LSET lst 0 "xxxxxxxxxxx"
assert_equal [r LPOS lst "xxxxxxxxxxx"] 0
} {} {needs:debug}
# basic command check for plain nodes - "LMOVE"
test {Test LMOVE on plain nodes} {
r flushdb
r debug quicklist-packed-threshold 1b
r RPUSH lst2{t} "aa"
r RPUSH lst2{t} "bb"
r LSET lst2{t} 0 xxxxxxxxxxx
r RPUSH lst2{t} "cc"
r RPUSH lst2{t} "dd"
r LMOVE lst2{t} lst{t} RIGHT LEFT
r LMOVE lst2{t} lst{t} LEFT RIGHT
assert_equal [r llen lst{t}] 2
assert_equal [r llen lst2{t}] 2
assert_equal [r lpop lst2{t}] "bb"
assert_equal [r lpop lst2{t}] "cc"
assert_equal [r lpop lst{t}] "dd"
assert_equal [r lpop lst{t}] "xxxxxxxxxxx"
} {} {needs:debug}
# testing LSET with combinations of node types
# plain->packed , packed->plain, plain->plain, packed->packed
test {Test LSET with packed / plain combinations} {
r debug quicklist-packed-threshold 5b
r RPUSH lst "aa"
r RPUSH lst "bb"
r lset lst 0 [string repeat d 50001]
set s1 [r lpop lst]
assert_equal $s1 [string repeat d 50001]
r RPUSH lst [string repeat f 50001]
r lset lst 0 [string repeat e 50001]
set s1 [r lpop lst]
assert_equal $s1 [string repeat e 50001]
r RPUSH lst [string repeat m 50001]
r lset lst 0 "bb"
set s1 [r lpop lst]
assert_equal $s1 "bb"
r RPUSH lst "bb"
r lset lst 0 "cc"
set s1 [r lpop lst]
assert_equal $s1 "cc"
} {} {needs:debug}
# checking LSET in case ziplist needs to be split
test {Test LSET with packed is split in the middle} {
r flushdb
r debug quicklist-packed-threshold 5b
r RPUSH lst "aa"
r RPUSH lst "bb"
r RPUSH lst "cc"
r RPUSH lst "dd"
r RPUSH lst "ee"
r lset lst 2 [string repeat e 10]
assert_equal [r lpop lst] "aa"
assert_equal [r lpop lst] "bb"
assert_equal [r lpop lst] [string repeat e 10]
assert_equal [r lpop lst] "dd"
assert_equal [r lpop lst] "ee"
} {} {needs:debug}
# repeating "plain check LSET with combinations"
# but now with single item in each ziplist
test {Test LSET with packed consist only one item} {
r flushdb
r config set list-max-ziplist-size 1
r debug quicklist-packed-threshold 1b
r RPUSH lst "aa"
r RPUSH lst "bb"
r lset lst 0 [string repeat d 50001]
set s1 [r lpop lst]
assert_equal $s1 [string repeat d 50001]
r RPUSH lst [string repeat f 50001]
r lset lst 0 [string repeat e 50001]
set s1 [r lpop lst]
assert_equal $s1 [string repeat e 50001]
r RPUSH lst [string repeat m 50001]
r lset lst 0 "bb"
set s1 [r lpop lst]
assert_equal $s1 "bb"
r RPUSH lst "bb"
r lset lst 0 "cc"
set s1 [r lpop lst]
assert_equal $s1 "cc"
} {} {needs:debug}
}
start_server [list overrides [list save ""] ] {
# test if the server supports such large configs (avoid 32 bit builds)
catch {
r config set proto-max-bulk-len 10000000000 ;#10gb
r config set client-query-buffer-limit 10000000000 ;#10gb
}
if {[lindex [r config get proto-max-bulk-len] 1] == 10000000000} {
set str_length 5000000000
# repeating all the plain nodes basic checks with 5gb values
test {Test LPUSH and LPOP on plain nodes over 4GB} {
r flushdb
r lpush lst 9
r write "*3\r\n\$5\r\nLPUSH\r\n\$3\r\nlst\r\n"
write_big_bulk $str_length;
r write "*3\r\n\$5\r\nLPUSH\r\n\$3\r\nlst\r\n"
write_big_bulk $str_length;
set s0 [s used_memory]
assert {$s0 > $str_length}
assert {[r llen lst] == 3}
assert_equal [r rpop lst] "9"
assert_equal [read_big_bulk {r rpop lst}] $str_length
assert {[r llen lst] == 1}
assert_equal [read_big_bulk {r rpop lst}] $str_length
} {} {large-memory}
test {Test LINDEX and LINSERT on plain nodes over 4GB} {
r flushdb
r write "*3\r\n\$5\r\nLPUSH\r\n\$3\r\nlst\r\n"
write_big_bulk $str_length;
r lpush lst 9
r write "*3\r\n\$5\r\nLPUSH\r\n\$3\r\nlst\r\n"
write_big_bulk $str_length;
r linsert lst before "9" "8"
assert_equal [r lindex lst 1] "8"
r LINSERT lst BEFORE "9" "7"
r write "*5\r\n\$7\r\nLINSERT\r\n\$3\r\nlst\r\n\$6\r\nBEFORE\r\n\$3\r\n\"9\"\r\n"
write_big_bulk 10;
assert_equal [read_big_bulk {r rpop lst}] $str_length
} {} {large-memory}
test {Test LTRIM on plain nodes over 4GB} {
r flushdb
r lpush lst 9
r write "*3\r\n\$5\r\nLPUSH\r\n\$3\r\nlst\r\n"
write_big_bulk $str_length;
r lpush lst 9
r LTRIM lst 1 -1
assert_equal [r llen lst] 2
assert_equal [r rpop lst] 9
assert_equal [read_big_bulk {r rpop lst}] $str_length
} {} {large-memory}
test {Test LREM on plain nodes over 4GB} {
r flushdb
r lpush lst one
r write "*3\r\n\$5\r\nLPUSH\r\n\$3\r\nlst\r\n"
write_big_bulk $str_length;
r lpush lst 9
r LREM lst -2 "one"
assert_equal [read_big_bulk {r rpop lst}] $str_length
r llen lst
} {1} {large-memory}
test {Test LSET on plain nodes over 4GB} {
r flushdb
r RPUSH lst "aa"
r RPUSH lst "bb"
r RPUSH lst "cc"
r write "*4\r\n\$4\r\nLSET\r\n\$3\r\nlst\r\n\$1\r\n0\r\n"
write_big_bulk $str_length;
assert_equal [r rpop lst] "cc"
assert_equal [r rpop lst] "bb"
assert_equal [read_big_bulk {r rpop lst}] $str_length
} {} {large-memory}
test {Test LMOVE on plain nodes over 4GB} {
r flushdb
r RPUSH lst2{t} "aa"
r RPUSH lst2{t} "bb"
r write "*4\r\n\$4\r\nLSET\r\n\$7\r\nlst2{t}\r\n\$1\r\n0\r\n"
write_big_bulk $str_length;
r RPUSH lst2{t} "cc"
r RPUSH lst2{t} "dd"
r LMOVE lst2{t} lst{t} RIGHT LEFT
assert_equal [read_big_bulk {r LMOVE lst2{t} lst{t} LEFT RIGHT}] $str_length
assert_equal [r llen lst{t}] 2
assert_equal [r llen lst2{t}] 2
assert_equal [r lpop lst2{t}] "bb"
assert_equal [r lpop lst2{t}] "cc"
assert_equal [r lpop lst{t}] "dd"
assert_equal [read_big_bulk {r rpop lst{t}}] $str_length
} {} {large-memory}
} ;# skip 32bit builds
}
start_server { start_server {
tags {"list"} tags {"list"}
overrides { overrides {
...@@ -1405,5 +1789,4 @@ foreach {pop} {BLPOP BLMPOP_RIGHT} { ...@@ -1405,5 +1789,4 @@ foreach {pop} {BLPOP BLMPOP_RIGHT} {
assert_equal [lpop k] [string repeat x 31] assert_equal [lpop k] [string repeat x 31]
set _ $k set _ $k
} {12 0 9223372036854775808 2147483647 32767 127} } {12 0 9223372036854775808 2147483647 32767 127}
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment