Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
71a8df6a
Commit
71a8df6a
authored
Mar 02, 2017
by
Guy Benoish
Browse files
Merge branch 'unstable' of
https://github.com/antirez/redis
into unstable
parents
56c01c95
9cc83d2a
Changes
182
Show whitespace changes
Inline
Side-by-side
src/dict.h
View file @
71a8df6a
...
...
@@ -56,7 +56,7 @@ typedef struct dictEntry {
}
dictEntry
;
typedef
struct
dictType
{
u
nsigned
in
t
(
*
hashFunction
)(
const
void
*
key
);
u
int64_
t
(
*
hashFunction
)(
const
void
*
key
);
void
*
(
*
keyDup
)(
void
*
privdata
,
const
void
*
key
);
void
*
(
*
valDup
)(
void
*
privdata
,
const
void
*
obj
);
int
(
*
keyCompare
)(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
...
...
@@ -168,15 +168,15 @@ void dictReleaseIterator(dictIterator *iter);
dictEntry
*
dictGetRandomKey
(
dict
*
d
);
unsigned
int
dictGetSomeKeys
(
dict
*
d
,
dictEntry
**
des
,
unsigned
int
count
);
void
dictGetStats
(
char
*
buf
,
size_t
bufsize
,
dict
*
d
);
u
nsigned
in
t
dictGenHashFunction
(
const
void
*
key
,
int
len
);
u
nsigned
in
t
dictGenCaseHashFunction
(
const
unsigned
char
*
buf
,
int
len
);
u
int64_
t
dictGenHashFunction
(
const
void
*
key
,
int
len
);
u
int64_
t
dictGenCaseHashFunction
(
const
unsigned
char
*
buf
,
int
len
);
void
dictEmpty
(
dict
*
d
,
void
(
callback
)(
void
*
));
void
dictEnableResize
(
void
);
void
dictDisableResize
(
void
);
int
dictRehash
(
dict
*
d
,
int
n
);
int
dictRehashMilliseconds
(
dict
*
d
,
int
ms
);
void
dictSetHashFunctionSeed
(
u
nsigned
int
initval
);
u
nsigned
in
t
dictGetHashFunctionSeed
(
void
);
void
dictSetHashFunctionSeed
(
u
int8_t
*
seed
);
u
int8_
t
*
dictGetHashFunctionSeed
(
void
);
unsigned
long
dictScan
(
dict
*
d
,
unsigned
long
v
,
dictScanFunction
*
fn
,
dictScanBucketFunction
*
bucketfn
,
void
*
privdata
);
unsigned
int
dictGetHash
(
dict
*
d
,
const
void
*
key
);
dictEntry
**
dictFindEntryRefByPtrAndHash
(
dict
*
d
,
const
void
*
oldptr
,
unsigned
int
hash
);
...
...
src/evict.c
View file @
71a8df6a
...
...
@@ -336,20 +336,13 @@ unsigned long LFUDecrAndReturn(robj *o) {
* server when there is data to add in order to make space if needed.
* --------------------------------------------------------------------------*/
int
freeMemoryIfNeeded
(
void
)
{
size_t
mem_reported
,
mem_used
,
mem_tofree
,
mem_freed
;
/* We don't want to count AOF buffers and slaves output buffers as
* used memory: the eviction should use mostly data size. This function
* returns the sum of AOF and slaves buffer. */
size_t
freeMemoryGetNotCountedMemory
(
void
)
{
size_t
overhead
=
0
;
int
slaves
=
listLength
(
server
.
slaves
);
mstime_t
latency
,
eviction_latency
;
long
long
delta
;
/* Check if we are over the memory usage limit. If we are not, no need
* to subtract the slaves output buffers. We can just return ASAP. */
mem_reported
=
zmalloc_used_memory
();
if
(
mem_reported
<=
server
.
maxmemory
)
return
C_OK
;
/* Remove the size of slaves output buffers and AOF buffer from the
* count of used memory. */
mem_used
=
mem_reported
;
if
(
slaves
)
{
listIter
li
;
listNode
*
ln
;
...
...
@@ -357,17 +350,31 @@ int freeMemoryIfNeeded(void) {
listRewind
(
server
.
slaves
,
&
li
);
while
((
ln
=
listNext
(
&
li
)))
{
client
*
slave
=
listNodeValue
(
ln
);
unsigned
long
obuf_bytes
=
getClientOutputBufferMemoryUsage
(
slave
);
if
(
obuf_bytes
>
mem_used
)
mem_used
=
0
;
else
mem_used
-=
obuf_bytes
;
overhead
+=
getClientOutputBufferMemoryUsage
(
slave
);
}
}
if
(
server
.
aof_state
!=
AOF_OFF
)
{
mem_used
-=
sdslen
(
server
.
aof_buf
);
mem_used
-=
aofRewriteBufferSize
();
overhead
+=
sdslen
(
server
.
aof_buf
)
+
aofRewriteBufferSize
();
}
return
overhead
;
}
int
freeMemoryIfNeeded
(
void
)
{
size_t
mem_reported
,
mem_used
,
mem_tofree
,
mem_freed
;
mstime_t
latency
,
eviction_latency
;
long
long
delta
;
int
slaves
=
listLength
(
server
.
slaves
);
/* Check if we are over the memory usage limit. If we are not, no need
* to subtract the slaves output buffers. We can just return ASAP. */
mem_reported
=
zmalloc_used_memory
();
if
(
mem_reported
<=
server
.
maxmemory
)
return
C_OK
;
/* Remove the size of slaves output buffers and AOF buffer from the
* count of used memory. */
mem_used
=
mem_reported
;
size_t
overhead
=
freeMemoryGetNotCountedMemory
();
mem_used
=
(
mem_used
>
overhead
)
?
mem_used
-
overhead
:
0
;
/* Check if we are still over the memory limit. */
if
(
mem_used
<=
server
.
maxmemory
)
return
C_OK
;
...
...
@@ -498,6 +505,22 @@ int freeMemoryIfNeeded(void) {
* deliver data to the slaves fast enough, so we force the
* transmission here inside the loop. */
if
(
slaves
)
flushSlavesOutputBuffers
();
/* Normally our stop condition is the ability to release
* a fixed, pre-computed amount of memory. However when we
* are deleting objects in another thread, it's better to
* check, from time to time, if we already reached our target
* memory, since the "mem_freed" amount is computed only
* across the dbAsyncDelete() call, while the thread can
* release the memory all the time. */
if
(
server
.
lazyfree_lazy_eviction
&&
!
(
keys_freed
%
16
))
{
overhead
=
freeMemoryGetNotCountedMemory
();
mem_used
=
zmalloc_used_memory
();
mem_used
=
(
mem_used
>
overhead
)
?
mem_used
-
overhead
:
0
;
if
(
mem_used
<=
server
.
maxmemory
)
{
mem_freed
=
mem_tofree
;
}
}
}
if
(
!
keys_freed
)
{
...
...
src/hyperloglog.c
View file @
71a8df6a
...
...
@@ -401,7 +401,11 @@ uint64_t MurmurHash64A (const void * key, int len, unsigned int seed) {
uint64_t
k
;
#if (BYTE_ORDER == LITTLE_ENDIAN)
#ifdef USE_ALIGNED_ACCESS
memcpy
(
&
k
,
data
,
sizeof
(
uint64_t
));
#else
k
=
*
((
uint64_t
*
)
data
);
#endif
#else
k
=
(
uint64_t
)
data
[
0
];
k
|=
(
uint64_t
)
data
[
1
]
<<
8
;
...
...
src/latency.c
View file @
71a8df6a
...
...
@@ -41,7 +41,7 @@ int dictStringKeyCompare(void *privdata, const void *key1, const void *key2) {
return
strcmp
(
key1
,
key2
)
==
0
;
}
u
nsigned
in
t
dictStringHash
(
const
void
*
key
)
{
u
int64_
t
dictStringHash
(
const
void
*
key
)
{
return
dictGenHashFunction
(
key
,
strlen
(
key
));
}
...
...
src/module.c
View file @
71a8df6a
...
...
@@ -2743,8 +2743,8 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver,
moduleTypeLoadFunc
rdb_load
;
moduleTypeSaveFunc
rdb_save
;
moduleTypeRewriteFunc
aof_rewrite
;
moduleTypeDigestFunc
digest
;
moduleTypeMemUsageFunc
mem_usage
;
moduleTypeDigestFunc
digest
;
moduleTypeFreeFunc
free
;
}
*
tms
=
(
struct
typemethods
*
)
typemethods_ptr
;
...
...
@@ -3264,7 +3264,7 @@ void *RM_GetBlockedClientPrivateData(RedisModuleCtx *ctx) {
/* server.moduleapi dictionary type. Only uses plain C strings since
* this gets queries from modules. */
u
nsigned
in
t
dictCStringKeyHash
(
const
void
*
key
)
{
u
int64_
t
dictCStringKeyHash
(
const
void
*
key
)
{
return
dictGenHashFunction
((
unsigned
char
*
)
key
,
strlen
((
char
*
)
key
));
}
...
...
src/modules/hellotype.c
View file @
71a8df6a
...
...
@@ -226,10 +226,12 @@ void HelloTypeAofRewrite(RedisModuleIO *aof, RedisModuleString *key, void *value
}
}
void
HelloTypeDigest
(
RedisModuleDigest
*
digest
,
void
*
value
)
{
REDISMODULE_NOT_USED
(
digest
);
REDISMODULE_NOT_USED
(
value
);
/* TODO: The DIGEST module interface is yet not implemented. */
/* The goal of this function is to return the amount of memory used by
* the HelloType value. */
size_t
HelloTypeMemUsage
(
const
void
*
value
)
{
const
struct
HelloTypeObject
*
hto
=
value
;
struct
HelloTypeNode
*
node
=
hto
->
head
;
return
sizeof
(
*
hto
)
+
sizeof
(
*
node
)
*
hto
->
len
;
}
void
HelloTypeFree
(
void
*
value
)
{
...
...
@@ -250,6 +252,7 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
.
rdb_load
=
HelloTypeRdbLoad
,
.
rdb_save
=
HelloTypeRdbSave
,
.
aof_rewrite
=
HelloTypeAofRewrite
,
.
mem_usage
=
HelloTypeMemUsage
,
.
free
=
HelloTypeFree
};
...
...
src/object.c
View file @
71a8df6a
...
...
@@ -246,11 +246,9 @@ void freeStringObject(robj *o) {
}
void
freeListObject
(
robj
*
o
)
{
switch
(
o
->
encoding
)
{
case
OBJ_ENCODING_QUICKLIST
:
if
(
o
->
encoding
==
OBJ_ENCODING_QUICKLIST
)
{
quicklistRelease
(
o
->
ptr
);
break
;
default:
}
else
{
serverPanic
(
"Unknown list encoding type"
);
}
}
...
...
@@ -786,6 +784,14 @@ size_t objectComputeSize(robj *o, size_t sample_size) {
}
else
{
serverPanic
(
"Unknown hash encoding"
);
}
}
else
if
(
o
->
type
==
OBJ_MODULE
)
{
moduleValue
*
mv
=
o
->
ptr
;
moduleType
*
mt
=
mv
->
type
;
if
(
mt
->
mem_usage
!=
NULL
)
{
asize
=
mt
->
mem_usage
(
mv
->
value
);
}
else
{
asize
=
0
;
}
}
else
{
serverPanic
(
"Unknown object type"
);
}
...
...
@@ -945,7 +951,7 @@ sds getMemoryDoctorReport(void) {
}
/* Slaves using more than 10 MB each? */
if
(
mh
->
clients_slaves
/
numslaves
>
(
1024
*
1024
*
10
))
{
if
(
numslaves
>
0
&&
mh
->
clients_slaves
/
numslaves
>
(
1024
*
1024
*
10
))
{
big_slave_buf
=
1
;
num_reports
++
;
}
...
...
src/redis-cli.c
View file @
71a8df6a
...
...
@@ -1273,6 +1273,11 @@ static void repl(void) {
int
argc
;
sds
*
argv
;
/* Initialize the help and, if possible, use the COMMAND command in order
* to retrieve missing entries. */
cliInitHelp
();
cliIntegrateHelp
();
config
.
interactive
=
1
;
linenoiseSetMultiLine
(
1
);
linenoiseSetCompletionCallback
(
completionCallback
);
...
...
@@ -2606,11 +2611,6 @@ int main(int argc, char **argv) {
argc
-=
firstarg
;
argv
+=
firstarg
;
/* Initialize the help and, if possible, use the COMMAND command in order
* to retrieve missing entries. */
cliInitHelp
();
cliIntegrateHelp
();
/* Latency mode */
if
(
config
.
latency_mode
)
{
if
(
cliConnect
(
0
)
==
REDIS_ERR
)
exit
(
1
);
...
...
src/redisassert.h
View file @
71a8df6a
...
...
@@ -41,7 +41,9 @@
#include <unistd.h>
/* for _exit() */
#define assert(_e) ((_e)?(void)0 : (_serverAssert(#_e,__FILE__,__LINE__),_exit(1)))
#define panic(...) _serverPanic(__FILE__,__LINE__,__VA_ARGS__),_exit(1)
void
_serverAssert
(
char
*
estr
,
char
*
file
,
int
line
);
void
_serverPanic
(
const
char
*
file
,
int
line
,
const
char
*
msg
,
...);
#endif
src/redismodule.h
View file @
71a8df6a
...
...
@@ -91,7 +91,7 @@ typedef int (*RedisModuleCmdFunc) (RedisModuleCtx *ctx, RedisModuleString **argv
typedef
void
*
(
*
RedisModuleTypeLoadFunc
)(
RedisModuleIO
*
rdb
,
int
encver
);
typedef
void
(
*
RedisModuleTypeSaveFunc
)(
RedisModuleIO
*
rdb
,
void
*
value
);
typedef
void
(
*
RedisModuleTypeRewriteFunc
)(
RedisModuleIO
*
aof
,
RedisModuleString
*
key
,
void
*
value
);
typedef
size_t
(
*
RedisModuleTypeMemUsageFunc
)(
void
*
value
);
typedef
size_t
(
*
RedisModuleTypeMemUsageFunc
)(
const
void
*
value
);
typedef
void
(
*
RedisModuleTypeDigestFunc
)(
RedisModuleDigest
*
digest
,
void
*
value
);
typedef
void
(
*
RedisModuleTypeFreeFunc
)(
void
*
value
);
...
...
src/replication.c
View file @
71a8df6a
...
...
@@ -1568,7 +1568,7 @@ int slaveTryPartialResynchronization(int fd, int read_reply) {
* establish a connection with the master. */
void
syncWithMaster
(
aeEventLoop
*
el
,
int
fd
,
void
*
privdata
,
int
mask
)
{
char
tmpfile
[
256
],
*
err
=
NULL
;
int
dfd
,
maxtries
=
5
;
int
dfd
=
-
1
,
maxtries
=
5
;
int
sockerr
=
0
,
psync_result
;
socklen_t
errlen
=
sizeof
(
sockerr
);
UNUSED
(
el
);
...
...
@@ -1832,6 +1832,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) {
error:
aeDeleteFileEvent
(
server
.
el
,
fd
,
AE_READABLE
|
AE_WRITABLE
);
if
(
dfd
!=
-
1
)
close
(
dfd
);
close
(
fd
);
server
.
repl_transfer_s
=
-
1
;
server
.
repl_state
=
REPL_STATE_CONNECT
;
...
...
src/sentinel.c
View file @
71a8df6a
...
...
@@ -379,7 +379,7 @@ void sentinelSimFailureCrash(void);
/* ========================= Dictionary types =============================== */
u
nsigned
in
t
dictSdsHash
(
const
void
*
key
);
u
int64_
t
dictSdsHash
(
const
void
*
key
);
int
dictSdsKeyCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
void
releaseSentinelRedisInstance
(
sentinelRedisInstance
*
ri
);
...
...
src/server.c
View file @
71a8df6a
...
...
@@ -482,16 +482,16 @@ int dictObjKeyCompare(void *privdata, const void *key1,
return
dictSdsKeyCompare
(
privdata
,
o1
->
ptr
,
o2
->
ptr
);
}
u
nsigned
in
t
dictObjHash
(
const
void
*
key
)
{
u
int64_
t
dictObjHash
(
const
void
*
key
)
{
const
robj
*
o
=
key
;
return
dictGenHashFunction
(
o
->
ptr
,
sdslen
((
sds
)
o
->
ptr
));
}
u
nsigned
in
t
dictSdsHash
(
const
void
*
key
)
{
u
int64_
t
dictSdsHash
(
const
void
*
key
)
{
return
dictGenHashFunction
((
unsigned
char
*
)
key
,
sdslen
((
char
*
)
key
));
}
u
nsigned
in
t
dictSdsCaseHash
(
const
void
*
key
)
{
u
int64_
t
dictSdsCaseHash
(
const
void
*
key
)
{
return
dictGenCaseHashFunction
((
unsigned
char
*
)
key
,
sdslen
((
char
*
)
key
));
}
...
...
@@ -513,7 +513,7 @@ int dictEncObjKeyCompare(void *privdata, const void *key1,
return
cmp
;
}
u
nsigned
in
t
dictEncObjHash
(
const
void
*
key
)
{
u
int64_
t
dictEncObjHash
(
const
void
*
key
)
{
robj
*
o
=
(
robj
*
)
key
;
if
(
sdsEncodedObject
(
o
))
{
...
...
@@ -526,7 +526,7 @@ unsigned int dictEncObjHash(const void *key) {
len
=
ll2string
(
buf
,
32
,(
long
)
o
->
ptr
);
return
dictGenHashFunction
((
unsigned
char
*
)
buf
,
len
);
}
else
{
u
nsigned
in
t
hash
;
u
int64_
t
hash
;
o
=
getDecodedObject
(
o
);
hash
=
dictGenHashFunction
(
o
->
ptr
,
sdslen
((
sds
)
o
->
ptr
));
...
...
@@ -3639,7 +3639,9 @@ int main(int argc, char **argv) {
zmalloc_set_oom_handler
(
redisOutOfMemoryHandler
);
srand
(
time
(
NULL
)
^
getpid
());
gettimeofday
(
&
tv
,
NULL
);
dictSetHashFunctionSeed
(
tv
.
tv_sec
^
tv
.
tv_usec
^
getpid
());
char
hashseed
[
16
];
getRandomHexChars
(
hashseed
,
sizeof
(
hashseed
));
dictSetHashFunctionSeed
((
uint8_t
*
)
hashseed
);
server
.
sentinel_mode
=
checkForSentinelMode
(
argc
,
argv
);
initServerConfig
();
moduleInitModulesSystem
();
...
...
src/server.h
View file @
71a8df6a
...
...
@@ -435,7 +435,7 @@ typedef long long mstime_t; /* millisecond time type. */
/* We can print the stacktrace, so our assert is defined this way: */
#define serverAssertWithInfo(_c,_o,_e) ((_e)?(void)0 : (_serverAssertWithInfo(_c,_o,#_e,__FILE__,__LINE__),_exit(1)))
#define serverAssert(_e) ((_e)?(void)0 : (_serverAssert(#_e,__FILE__,__LINE__),_exit(1)))
#define serverPanic(
_e
) _serverPanic(
#_e,
__FILE__,__LINE__),_exit(1)
#define serverPanic(
...
) _serverPanic(__FILE__,__LINE__
,__VA_ARGS__
),_exit(1)
/*-----------------------------------------------------------------------------
* Data types
...
...
@@ -483,7 +483,7 @@ typedef void *(*moduleTypeLoadFunc)(struct RedisModuleIO *io, int encver);
typedef
void
(
*
moduleTypeSaveFunc
)(
struct
RedisModuleIO
*
io
,
void
*
value
);
typedef
void
(
*
moduleTypeRewriteFunc
)(
struct
RedisModuleIO
*
io
,
struct
redisObject
*
key
,
void
*
value
);
typedef
void
(
*
moduleTypeDigestFunc
)(
struct
RedisModuleDigest
*
digest
,
void
*
value
);
typedef
size_t
(
*
moduleTypeMemUsageFunc
)(
void
*
value
);
typedef
size_t
(
*
moduleTypeMemUsageFunc
)(
const
void
*
value
);
typedef
void
(
*
moduleTypeFreeFunc
)(
void
*
value
);
/* The module type, which is referenced in each value of a given type, defines
...
...
@@ -1765,7 +1765,7 @@ unsigned long LFUGetTimeInMinutes(void);
uint8_t
LFULogIncr
(
uint8_t
value
);
/* Keys hashing / comparison functions for dict.c hash tables. */
u
nsigned
in
t
dictSdsHash
(
const
void
*
key
);
u
int64_
t
dictSdsHash
(
const
void
*
key
);
int
dictSdsKeyCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
void
dictSdsDestructor
(
void
*
privdata
,
void
*
val
);
...
...
@@ -1960,7 +1960,7 @@ void *realloc(void *ptr, size_t size) __attribute__ ((deprecated));
/* Debugging stuff */
void
_serverAssertWithInfo
(
const
client
*
c
,
const
robj
*
o
,
const
char
*
estr
,
const
char
*
file
,
int
line
);
void
_serverAssert
(
const
char
*
estr
,
const
char
*
file
,
int
line
);
void
_serverPanic
(
const
char
*
msg
,
const
char
*
file
,
int
line
);
void
_serverPanic
(
const
char
*
file
,
int
line
,
const
char
*
msg
,
...
);
void
bugReportStart
(
void
);
void
serverLogObjectDebugInfo
(
const
robj
*
o
);
void
sigsegvHandler
(
int
sig
,
siginfo_t
*
info
,
void
*
secret
);
...
...
src/siphash.c
0 → 100644
View file @
71a8df6a
/*
SipHash reference C implementation
Copyright (c) 2012-2016 Jean-Philippe Aumasson
<jeanphilippe.aumasson@gmail.com>
Copyright (c) 2012-2014 Daniel J. Bernstein <djb@cr.yp.to>
Copyright (c) 2017 Salvatore Sanfilippo <antirez@gmail.com>
To the extent possible under law, the author(s) have dedicated all copyright
and related and neighboring rights to this software to the public domain
worldwide. This software is distributed without any warranty.
You should have received a copy of the CC0 Public Domain Dedication along
with this software. If not, see
<http://creativecommons.org/publicdomain/zero/1.0/>.
----------------------------------------------------------------------------
This version was modified by Salvatore Sanfilippo <antirez@gmail.com>
in the following ways:
1. We use SipHash 1-2. This is not believed to be as strong as the
suggested 2-4 variant, but AFAIK there are not trivial attacks
against this reduced-rounds version, and it runs at the same speed
as Murmurhash2 that we used previously, why the 2-4 variant slowed
down Redis by a 4% figure more or less.
2. Hard-code rounds in the hope the compiler can optimize it more
in this raw from. Anyway we always want the standard 2-4 variant.
3. Modify the prototype and implementation so that the function directly
returns an uint64_t value, the hash itself, instead of receiving an
output buffer. This also means that the output size is set to 8 bytes
and the 16 bytes output code handling was removed.
4. Provide a case insensitive variant to be used when hashing strings that
must be considered identical by the hash table regardless of the case.
If we don't have directly a case insensitive hash function, we need to
perform a text transformation in some temporary buffer, which is costly.
5. Remove debugging code.
6. Modified the original test.c file to be a stand-alone function testing
the function in the new form (returing an uint64_t) using just the
relevant test vector.
*/
#include <assert.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
/* Fast tolower() alike function that does not care about locale
* but just returns a-z insetad of A-Z. */
int
siptlw
(
int
c
)
{
if
(
c
>=
'A'
&&
c
<=
'Z'
)
{
return
c
+
(
'a'
-
'A'
);
}
else
{
return
c
;
}
}
/* Test of the CPU is Little Endian and supports not aligned accesses.
* Two interesting conditions to speedup the function that happen to be
* in most of x86 servers. */
#if defined(__X86_64__) || defined(__x86_64__) || defined (__i386__)
#define UNALIGNED_LE_CPU
#endif
#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
#define U32TO8_LE(p, v) \
(p)[0] = (uint8_t)((v)); \
(p)[1] = (uint8_t)((v) >> 8); \
(p)[2] = (uint8_t)((v) >> 16); \
(p)[3] = (uint8_t)((v) >> 24);
#define U64TO8_LE(p, v) \
U32TO8_LE((p), (uint32_t)((v))); \
U32TO8_LE((p) + 4, (uint32_t)((v) >> 32));
#ifdef UNALIGNED_LE_CPU
#define U8TO64_LE(p) (*((uint64_t*)(p)))
#else
#define U8TO64_LE(p) \
(((uint64_t)((p)[0])) | ((uint64_t)((p)[1]) << 8) | \
((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24) | \
((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) | \
((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56))
#endif
#define U8TO64_LE_NOCASE(p) \
(((uint64_t)(siptlw((p)[0]))) | \
((uint64_t)(siptlw((p)[1])) << 8) | \
((uint64_t)(siptlw((p)[2])) << 16) | \
((uint64_t)(siptlw((p)[3])) << 24) | \
((uint64_t)(siptlw((p)[4])) << 32) | \
((uint64_t)(siptlw((p)[5])) << 40) | \
((uint64_t)(siptlw((p)[6])) << 48) | \
((uint64_t)(siptlw((p)[7])) << 56))
#define SIPROUND \
do { \
v0 += v1; \
v1 = ROTL(v1, 13); \
v1 ^= v0; \
v0 = ROTL(v0, 32); \
v2 += v3; \
v3 = ROTL(v3, 16); \
v3 ^= v2; \
v0 += v3; \
v3 = ROTL(v3, 21); \
v3 ^= v0; \
v2 += v1; \
v1 = ROTL(v1, 17); \
v1 ^= v2; \
v2 = ROTL(v2, 32); \
} while (0)
uint64_t
siphash
(
const
uint8_t
*
in
,
const
size_t
inlen
,
const
uint8_t
*
k
)
{
#ifndef UNALIGNED_LE_CPU
uint64_t
hash
;
uint8_t
*
out
=
(
uint8_t
*
)
&
hash
;
#endif
uint64_t
v0
=
0x736f6d6570736575ULL
;
uint64_t
v1
=
0x646f72616e646f6dULL
;
uint64_t
v2
=
0x6c7967656e657261ULL
;
uint64_t
v3
=
0x7465646279746573ULL
;
uint64_t
k0
=
U8TO64_LE
(
k
);
uint64_t
k1
=
U8TO64_LE
(
k
+
8
);
uint64_t
m
;
const
uint8_t
*
end
=
in
+
inlen
-
(
inlen
%
sizeof
(
uint64_t
));
const
int
left
=
inlen
&
7
;
uint64_t
b
=
((
uint64_t
)
inlen
)
<<
56
;
v3
^=
k1
;
v2
^=
k0
;
v1
^=
k1
;
v0
^=
k0
;
for
(;
in
!=
end
;
in
+=
8
)
{
m
=
U8TO64_LE
(
in
);
v3
^=
m
;
SIPROUND
;
v0
^=
m
;
}
switch
(
left
)
{
case
7
:
b
|=
((
uint64_t
)
in
[
6
])
<<
48
;
case
6
:
b
|=
((
uint64_t
)
in
[
5
])
<<
40
;
case
5
:
b
|=
((
uint64_t
)
in
[
4
])
<<
32
;
case
4
:
b
|=
((
uint64_t
)
in
[
3
])
<<
24
;
case
3
:
b
|=
((
uint64_t
)
in
[
2
])
<<
16
;
case
2
:
b
|=
((
uint64_t
)
in
[
1
])
<<
8
;
case
1
:
b
|=
((
uint64_t
)
in
[
0
]);
break
;
case
0
:
break
;
}
v3
^=
b
;
SIPROUND
;
v0
^=
b
;
v2
^=
0xff
;
SIPROUND
;
SIPROUND
;
b
=
v0
^
v1
^
v2
^
v3
;
#ifndef UNALIGNED_LE_CPU
U64TO8_LE
(
out
,
b
);
return
hash
;
#else
return
b
;
#endif
}
uint64_t
siphash_nocase
(
const
uint8_t
*
in
,
const
size_t
inlen
,
const
uint8_t
*
k
)
{
#ifndef UNALIGNED_LE_CPU
uint64_t
hash
;
uint8_t
*
out
=
(
uint8_t
*
)
&
hash
;
#endif
uint64_t
v0
=
0x736f6d6570736575ULL
;
uint64_t
v1
=
0x646f72616e646f6dULL
;
uint64_t
v2
=
0x6c7967656e657261ULL
;
uint64_t
v3
=
0x7465646279746573ULL
;
uint64_t
k0
=
U8TO64_LE
(
k
);
uint64_t
k1
=
U8TO64_LE
(
k
+
8
);
uint64_t
m
;
const
uint8_t
*
end
=
in
+
inlen
-
(
inlen
%
sizeof
(
uint64_t
));
const
int
left
=
inlen
&
7
;
uint64_t
b
=
((
uint64_t
)
inlen
)
<<
56
;
v3
^=
k1
;
v2
^=
k0
;
v1
^=
k1
;
v0
^=
k0
;
for
(;
in
!=
end
;
in
+=
8
)
{
m
=
U8TO64_LE_NOCASE
(
in
);
v3
^=
m
;
SIPROUND
;
v0
^=
m
;
}
switch
(
left
)
{
case
7
:
b
|=
((
uint64_t
)
siptlw
(
in
[
6
]))
<<
48
;
case
6
:
b
|=
((
uint64_t
)
siptlw
(
in
[
5
]))
<<
40
;
case
5
:
b
|=
((
uint64_t
)
siptlw
(
in
[
4
]))
<<
32
;
case
4
:
b
|=
((
uint64_t
)
siptlw
(
in
[
3
]))
<<
24
;
case
3
:
b
|=
((
uint64_t
)
siptlw
(
in
[
2
]))
<<
16
;
case
2
:
b
|=
((
uint64_t
)
siptlw
(
in
[
1
]))
<<
8
;
case
1
:
b
|=
((
uint64_t
)
siptlw
(
in
[
0
]));
break
;
case
0
:
break
;
}
v3
^=
b
;
SIPROUND
;
v0
^=
b
;
v2
^=
0xff
;
SIPROUND
;
SIPROUND
;
b
=
v0
^
v1
^
v2
^
v3
;
#ifndef UNALIGNED_LE_CPU
U64TO8_LE
(
out
,
b
);
return
hash
;
#else
return
b
;
#endif
}
/* --------------------------------- TEST ------------------------------------ */
#ifdef SIPHASH_TEST
const
uint8_t
vectors_sip64
[
64
][
8
]
=
{
{
0x31
,
0x0e
,
0x0e
,
0xdd
,
0x47
,
0xdb
,
0x6f
,
0x72
,
},
{
0xfd
,
0x67
,
0xdc
,
0x93
,
0xc5
,
0x39
,
0xf8
,
0x74
,
},
{
0x5a
,
0x4f
,
0xa9
,
0xd9
,
0x09
,
0x80
,
0x6c
,
0x0d
,
},
{
0x2d
,
0x7e
,
0xfb
,
0xd7
,
0x96
,
0x66
,
0x67
,
0x85
,
},
{
0xb7
,
0x87
,
0x71
,
0x27
,
0xe0
,
0x94
,
0x27
,
0xcf
,
},
{
0x8d
,
0xa6
,
0x99
,
0xcd
,
0x64
,
0x55
,
0x76
,
0x18
,
},
{
0xce
,
0xe3
,
0xfe
,
0x58
,
0x6e
,
0x46
,
0xc9
,
0xcb
,
},
{
0x37
,
0xd1
,
0x01
,
0x8b
,
0xf5
,
0x00
,
0x02
,
0xab
,
},
{
0x62
,
0x24
,
0x93
,
0x9a
,
0x79
,
0xf5
,
0xf5
,
0x93
,
},
{
0xb0
,
0xe4
,
0xa9
,
0x0b
,
0xdf
,
0x82
,
0x00
,
0x9e
,
},
{
0xf3
,
0xb9
,
0xdd
,
0x94
,
0xc5
,
0xbb
,
0x5d
,
0x7a
,
},
{
0xa7
,
0xad
,
0x6b
,
0x22
,
0x46
,
0x2f
,
0xb3
,
0xf4
,
},
{
0xfb
,
0xe5
,
0x0e
,
0x86
,
0xbc
,
0x8f
,
0x1e
,
0x75
,
},
{
0x90
,
0x3d
,
0x84
,
0xc0
,
0x27
,
0x56
,
0xea
,
0x14
,
},
{
0xee
,
0xf2
,
0x7a
,
0x8e
,
0x90
,
0xca
,
0x23
,
0xf7
,
},
{
0xe5
,
0x45
,
0xbe
,
0x49
,
0x61
,
0xca
,
0x29
,
0xa1
,
},
{
0xdb
,
0x9b
,
0xc2
,
0x57
,
0x7f
,
0xcc
,
0x2a
,
0x3f
,
},
{
0x94
,
0x47
,
0xbe
,
0x2c
,
0xf5
,
0xe9
,
0x9a
,
0x69
,
},
{
0x9c
,
0xd3
,
0x8d
,
0x96
,
0xf0
,
0xb3
,
0xc1
,
0x4b
,
},
{
0xbd
,
0x61
,
0x79
,
0xa7
,
0x1d
,
0xc9
,
0x6d
,
0xbb
,
},
{
0x98
,
0xee
,
0xa2
,
0x1a
,
0xf2
,
0x5c
,
0xd6
,
0xbe
,
},
{
0xc7
,
0x67
,
0x3b
,
0x2e
,
0xb0
,
0xcb
,
0xf2
,
0xd0
,
},
{
0x88
,
0x3e
,
0xa3
,
0xe3
,
0x95
,
0x67
,
0x53
,
0x93
,
},
{
0xc8
,
0xce
,
0x5c
,
0xcd
,
0x8c
,
0x03
,
0x0c
,
0xa8
,
},
{
0x94
,
0xaf
,
0x49
,
0xf6
,
0xc6
,
0x50
,
0xad
,
0xb8
,
},
{
0xea
,
0xb8
,
0x85
,
0x8a
,
0xde
,
0x92
,
0xe1
,
0xbc
,
},
{
0xf3
,
0x15
,
0xbb
,
0x5b
,
0xb8
,
0x35
,
0xd8
,
0x17
,
},
{
0xad
,
0xcf
,
0x6b
,
0x07
,
0x63
,
0x61
,
0x2e
,
0x2f
,
},
{
0xa5
,
0xc9
,
0x1d
,
0xa7
,
0xac
,
0xaa
,
0x4d
,
0xde
,
},
{
0x71
,
0x65
,
0x95
,
0x87
,
0x66
,
0x50
,
0xa2
,
0xa6
,
},
{
0x28
,
0xef
,
0x49
,
0x5c
,
0x53
,
0xa3
,
0x87
,
0xad
,
},
{
0x42
,
0xc3
,
0x41
,
0xd8
,
0xfa
,
0x92
,
0xd8
,
0x32
,
},
{
0xce
,
0x7c
,
0xf2
,
0x72
,
0x2f
,
0x51
,
0x27
,
0x71
,
},
{
0xe3
,
0x78
,
0x59
,
0xf9
,
0x46
,
0x23
,
0xf3
,
0xa7
,
},
{
0x38
,
0x12
,
0x05
,
0xbb
,
0x1a
,
0xb0
,
0xe0
,
0x12
,
},
{
0xae
,
0x97
,
0xa1
,
0x0f
,
0xd4
,
0x34
,
0xe0
,
0x15
,
},
{
0xb4
,
0xa3
,
0x15
,
0x08
,
0xbe
,
0xff
,
0x4d
,
0x31
,
},
{
0x81
,
0x39
,
0x62
,
0x29
,
0xf0
,
0x90
,
0x79
,
0x02
,
},
{
0x4d
,
0x0c
,
0xf4
,
0x9e
,
0xe5
,
0xd4
,
0xdc
,
0xca
,
},
{
0x5c
,
0x73
,
0x33
,
0x6a
,
0x76
,
0xd8
,
0xbf
,
0x9a
,
},
{
0xd0
,
0xa7
,
0x04
,
0x53
,
0x6b
,
0xa9
,
0x3e
,
0x0e
,
},
{
0x92
,
0x59
,
0x58
,
0xfc
,
0xd6
,
0x42
,
0x0c
,
0xad
,
},
{
0xa9
,
0x15
,
0xc2
,
0x9b
,
0xc8
,
0x06
,
0x73
,
0x18
,
},
{
0x95
,
0x2b
,
0x79
,
0xf3
,
0xbc
,
0x0a
,
0xa6
,
0xd4
,
},
{
0xf2
,
0x1d
,
0xf2
,
0xe4
,
0x1d
,
0x45
,
0x35
,
0xf9
,
},
{
0x87
,
0x57
,
0x75
,
0x19
,
0x04
,
0x8f
,
0x53
,
0xa9
,
},
{
0x10
,
0xa5
,
0x6c
,
0xf5
,
0xdf
,
0xcd
,
0x9a
,
0xdb
,
},
{
0xeb
,
0x75
,
0x09
,
0x5c
,
0xcd
,
0x98
,
0x6c
,
0xd0
,
},
{
0x51
,
0xa9
,
0xcb
,
0x9e
,
0xcb
,
0xa3
,
0x12
,
0xe6
,
},
{
0x96
,
0xaf
,
0xad
,
0xfc
,
0x2c
,
0xe6
,
0x66
,
0xc7
,
},
{
0x72
,
0xfe
,
0x52
,
0x97
,
0x5a
,
0x43
,
0x64
,
0xee
,
},
{
0x5a
,
0x16
,
0x45
,
0xb2
,
0x76
,
0xd5
,
0x92
,
0xa1
,
},
{
0xb2
,
0x74
,
0xcb
,
0x8e
,
0xbf
,
0x87
,
0x87
,
0x0a
,
},
{
0x6f
,
0x9b
,
0xb4
,
0x20
,
0x3d
,
0xe7
,
0xb3
,
0x81
,
},
{
0xea
,
0xec
,
0xb2
,
0xa3
,
0x0b
,
0x22
,
0xa8
,
0x7f
,
},
{
0x99
,
0x24
,
0xa4
,
0x3c
,
0xc1
,
0x31
,
0x57
,
0x24
,
},
{
0xbd
,
0x83
,
0x8d
,
0x3a
,
0xaf
,
0xbf
,
0x8d
,
0xb7
,
},
{
0x0b
,
0x1a
,
0x2a
,
0x32
,
0x65
,
0xd5
,
0x1a
,
0xea
,
},
{
0x13
,
0x50
,
0x79
,
0xa3
,
0x23
,
0x1c
,
0xe6
,
0x60
,
},
{
0x93
,
0x2b
,
0x28
,
0x46
,
0xe4
,
0xd7
,
0x06
,
0x66
,
},
{
0xe1
,
0x91
,
0x5f
,
0x5c
,
0xb1
,
0xec
,
0xa4
,
0x6c
,
},
{
0xf3
,
0x25
,
0x96
,
0x5c
,
0xa1
,
0x6d
,
0x62
,
0x9f
,
},
{
0x57
,
0x5f
,
0xf2
,
0x8e
,
0x60
,
0x38
,
0x1b
,
0xe5
,
},
{
0x72
,
0x45
,
0x06
,
0xeb
,
0x4c
,
0x32
,
0x8a
,
0x95
,
},
};
/* Test siphash using a test vector. Returns 0 if the function passed
* all the tests, otherwise 1 is returned.
*
* IMPORTANT: The test vector is for SipHash 2-4. Before running
* the test revert back the siphash() function to 2-4 rounds since
* now it uses 1-2 rounds. */
int
siphash_test
(
void
)
{
uint8_t
in
[
64
],
k
[
16
];
int
i
;
int
fails
=
0
;
for
(
i
=
0
;
i
<
16
;
++
i
)
k
[
i
]
=
i
;
for
(
i
=
0
;
i
<
64
;
++
i
)
{
in
[
i
]
=
i
;
uint64_t
hash
=
siphash
(
in
,
i
,
k
);
const
uint8_t
*
v
=
NULL
;
v
=
(
uint8_t
*
)
vectors_sip64
;
if
(
memcmp
(
&
hash
,
v
+
(
i
*
8
),
8
))
{
/* printf("fail for %d bytes\n", i); */
fails
++
;
}
}
/* Run a few basic tests with the case insensitive version. */
uint64_t
h1
,
h2
;
h1
=
siphash
((
uint8_t
*
)
"hello world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
h2
=
siphash_nocase
((
uint8_t
*
)
"hello world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
if
(
h1
!=
h2
)
fails
++
;
h1
=
siphash
((
uint8_t
*
)
"hello world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
h2
=
siphash_nocase
((
uint8_t
*
)
"HELLO world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
if
(
h1
!=
h2
)
fails
++
;
h1
=
siphash
((
uint8_t
*
)
"HELLO world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
h2
=
siphash_nocase
((
uint8_t
*
)
"HELLO world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
if
(
h1
==
h2
)
fails
++
;
if
(
!
fails
)
return
0
;
return
1
;
}
int
main
(
void
)
{
if
(
siphash_test
()
==
0
)
{
printf
(
"SipHash test: OK
\n
"
);
return
0
;
}
else
{
printf
(
"SipHash test: FAILED
\n
"
);
return
1
;
}
}
#endif
src/t_zset.c
View file @
71a8df6a
...
...
@@ -1521,7 +1521,7 @@ void zaddGenericCommand(client *c, int flags) {
/* After the options, we expect to have an even number of args, since
* we expect any number of score-element pairs. */
elements
=
c
->
argc
-
scoreidx
;
if
(
elements
%
2
)
{
if
(
elements
%
2
||
!
elements
)
{
addReply
(
c
,
shared
.
syntaxerr
);
return
;
}
...
...
@@ -2110,7 +2110,7 @@ inline static void zunionInterAggregate(double *target, double val, int aggregat
}
}
u
nsigned
in
t
dictSdsHash
(
const
void
*
key
);
u
int64_
t
dictSdsHash
(
const
void
*
key
);
int
dictSdsKeyCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
dictType
setAccumulatorDictType
=
{
...
...
src/ziplist.c
View file @
71a8df6a
...
...
@@ -9,78 +9,149 @@
* ----------------------------------------------------------------------------
*
* ZIPLIST OVERALL LAYOUT
* ======================
*
* The general layout of the ziplist is as follows:
*
* <zlbytes> <zltail> <zllen> <entry> <entry> ... <entry> <zlend>
*
*
A
ll fields are stored in little endian.
*
NOTE: a
ll fields are stored in little endian
, if not specified otherwise
.
*
* <uint32_t zlbytes> is an unsigned integer to hold the number of bytes that
* the ziplist occupies. This value needs to be stored to be able to resize the
* entire structure without the need to traverse it first.
* the ziplist occupies, including the four bytes of the zlbytes field itself.
* This value needs to be stored to be able to resize the entire structure
* without the need to traverse it first.
*
* <uint32_t zltail> is the offset to the last entry in the list. This allows
* a pop operation on the far side of the list without the need for full
* traversal.
*
* <uint16_t zllen> is the number of entries. When th
is value is larger
*
than
2^16-2
, we need to traverse the entir
e
l
ist to
know how many items it
* holds.
* <uint16_t zllen> is the number of entries. When th
ere are more than
* 2^16-2
entires, this valu
e is
se
t to
2^16-1 and we need to traverse the
*
entire list to know how many items it
holds.
*
* <uint8_t zlend> is a single byte special value, equal to 255, which
* indicates the end of the list.
* <uint8_t zlend> is a special entry representing the end of the ziplist.
* Is encoded as a single byte equal to 255. No other normal entry starts
* with a byte set to the value of 255.
*
* ZIPLIST ENTRIES
* ===============
*
* Every entry in the ziplist is prefixed by
a header
that contains two pieces
* Every entry in the ziplist is prefixed by
metadata
that contains two pieces
* of information. First, the length of the previous entry is stored to be
* able to traverse the list from back to front. Second, the encoding with an
* optional string length of the entry itself is stored.
* able to traverse the list from back to front. Second, the entry encoding is
* provided. It represents the entry type, integer or string, and in the case
* of strings it also represents the length of the string payload.
* So a complete entry is stored like this:
*
* The length of the previous entry is encoded in the following way:
* If this length is smaller than 254 bytes, it will only consume a single
* byte that takes the length as value. When the length is greater than or
* equal to 254, it will consume 5 bytes. The first byte is set to 254 to
* indicate a larger value is following. The remaining 4 bytes take the
* length of the previous entry as value.
* <prevlen> <encoding> <entry-data>
*
* The other header field of the entry itself depends on the contents of the
* entry. When the entry is a string, the first 2 bits of this header will hold
* the type of encoding used to store the length of the string, followed by the
* actual length of the string. When the entry is an integer the first 2 bits
* are both set to 1. The following 2 bits are used to specify what kind of
* integer will be stored after this header. An overview of the different
* types and encodings is as follows:
* Sometimes the encoding represents the entry itself, like for small integers
* as we'll see later. In such a case the <entry-data> part is missing, and we
* could have just:
*
* <prevlen> <encoding>
*
* The length of the previous entry, <prevlen>, is encoded in the following way:
* If this length is smaller than 255 bytes, it will only consume a single
* byte representing the length as an unsinged 8 bit integer. When the length
* is greater than or equal to 255, it will consume 5 bytes. The first byte is
* set to 255 (FF) to indicate a larger value is following. The remaining 4
* bytes take the length of the previous entry as value.
*
* So practically an entry is encoded in the following way:
*
* <prevlen from 0 to 254> <encoding> <entry>
*
* Or alternatively if the previous entry length is greater than 254 bytes
* the following encoding is used:
*
* 0xFF <4 bytes unsigned little endian prevlen> <encoding> <entry>
*
* The encoding field of the entry depends on the content of the
* entry. When the entry is a string, the first 2 bits of the encoding first
* byte will hold the type of encoding used to store the length of the string,
* followed by the actual length of the string. When the entry is an integer
* the first 2 bits are both set to 1. The following 2 bits are used to specify
* what kind of integer will be stored after this header. An overview of the
* different types and encodings is as follows. The first byte is always enough
* to determine the kind of entry.
*
* |00pppppp| - 1 byte
* String value with length less than or equal to 63 bytes (6 bits).
* "pppppp" represents the unsigned 6 bit length.
* |01pppppp|qqqqqqqq| - 2 bytes
* String value with length less than or equal to 16383 bytes (14 bits).
* |10______|qqqqqqqq|rrrrrrrr|ssssssss|tttttttt| - 5 bytes
* IMPORTANT: The 14 bit number is stored in big endian.
* |10000000|qqqqqqqq|rrrrrrrr|ssssssss|tttttttt| - 5 bytes
* String value with length greater than or equal to 16384 bytes.
* |11000000| - 1 byte
* Only the 4 bytes following the first byte represents the length
* up to 32^2-1. The 6 lower bits of the first byte are not used and
* are set to zero.
* IMPORTANT: The 32 bit number is stored in big endian.
* |11000000| - 3 bytes
* Integer encoded as int16_t (2 bytes).
* |11010000| -
1
byte
* |11010000| -
5
byte
s
* Integer encoded as int32_t (4 bytes).
* |11100000| -
1
byte
* |11100000| -
9
byte
s
* Integer encoded as int64_t (8 bytes).
* |11110000| -
1
byte
* |11110000| -
4
byte
s
* Integer encoded as 24 bit signed (3 bytes).
* |11111110| -
1
byte
* |11111110| -
2
byte
s
* Integer encoded as 8 bit signed (1 byte).
* |1111xxxx| - (with xxxx between 0000 and 1101) immediate 4 bit integer.
* Unsigned integer from 0 to 12. The encoded value is actually from
* 1 to 13 because 0000 and 1111 can not be used, so 1 should be
* subtracted from the encoded 4 bit value to obtain the right value.
* |11111111| - End of ziplist.
* |11111111| - End of ziplist special entry.
*
* Like for the ziplist header, all the integers are represented in little
* endian byte order, even when this code is compiled in big endian systems.
*
* EXAMPLES OF ACTUAL ZIPLISTS
* ===========================
*
* The following is a ziplist containing the two elements representing
* the strings "2" and "5". It is composed of 15 bytes, that we visually
* split into sections:
*
* [0f 00 00 00] [0c 00 00 00] [02 00] [00 f3] [02 f6] [ff]
* | | | | | |
* zlbytes zltail entries "2" "5" end
*
* The first 4 bytes represent the number 15, that is the number of bytes
* the whole ziplist is composed of. The second 4 bytes are the offset
* at which the last ziplist entry is found, that is 12, in fact the
* last entry, that is "5", is at offset 12 inside the ziplist.
* The next 16 bit integer represents the number of elements inside the
* ziplist, its value is 2 since there are just two elements inside.
* Finally "00 f3" is the first entry representing the number 2. It is
* composed of the previous entry length, which is zero because this is
* our first entry, and the byte F3 which corresponds to the encoding
* |1111xxxx| with xxxx between 0001 and 1101. We need to remove the "F"
* higher order bits 1111, and subtract 1 from the "3", so the entry value
* is "2". The next entry has a prevlen of 02, since the first entry is
* composed of exactly two bytes. The entry itself, F6, is encoded exactly
* like the first entry, and 6-1 = 5, so the value of the entry is 5.
* Finally the special entry FF signals the end of the ziplist.
*
* Adding another element to the above string with the value "Hello World"
* allows us to show how the ziplist encodes small strings. We'll just show
* the hex dump of the entry itself. Imagine the bytes as following the
* entry that stores "5" in the ziplist above:
*
* [02] [0b] [48 65 6c 6c 6f 20 57 6f 72 6c 64]
*
* All the integers are represented in little endian byte order.
* The first byte, 02, is the length of the previous entry. The next
* byte represents the encoding in the pattern |00pppppp| that means
* that the entry is a string of length <pppppp>, so 0B means that
* an 11 bytes string follows. From the third byte (48) to the last (64)
* there are just the ASCII characters for "Hello World".
*
* ----------------------------------------------------------------------------
*
* Copyright (c) 2009-2012, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2009-201
2
, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2009-201
7
, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
...
...
@@ -119,8 +190,13 @@
#include "endianconv.h"
#include "redisassert.h"
#define ZIP_END 255
#define ZIP_BIGLEN 254
#define ZIP_END 255
/* Special "end of ziplist" entry. */
#define ZIP_BIG_PREVLEN 254
/* Max number of bytes of the previous entry, for
the "prevlen" field prefixing each entry, to be
represented with just a single byte. Otherwise
it is represented as FF AA BB CC DD, where
AA BB CC DD are a 4 bytes unsigned integer
representing the previous entry len. */
/* Different encoding/length possibilities */
#define ZIP_STR_MASK 0xc0
...
...
@@ -133,41 +209,83 @@
#define ZIP_INT_64B (0xc0 | 2<<4)
#define ZIP_INT_24B (0xc0 | 3<<4)
#define ZIP_INT_8B 0xfe
/* 4 bit integer immediate encoding */
#define ZIP_INT_IMM_MASK 0x0f
/* 4 bit integer immediate encoding |1111xxxx| with xxxx between
* 0001 and 1101. */
#define ZIP_INT_IMM_MASK 0x0f
/* Mask to extract the 4 bits value. To add
one is needed to reconstruct the value. */
#define ZIP_INT_IMM_MIN 0xf1
/* 11110001 */
#define ZIP_INT_IMM_MAX 0xfd
/* 11111101 */
#define ZIP_INT_IMM_VAL(v) (v & ZIP_INT_IMM_MASK)
#define INT24_MAX 0x7fffff
#define INT24_MIN (-INT24_MAX - 1)
/* Macro to determine type */
/* Macro to determine if the entry is a string. String entries never start
* with "11" as most significant bits of the first byte. */
#define ZIP_IS_STR(enc) (((enc) & ZIP_STR_MASK) < ZIP_STR_MASK)
/* Utility macros */
/* Utility macros.*/
/* Return total bytes a ziplist is composed of. */
#define ZIPLIST_BYTES(zl) (*((uint32_t*)(zl)))
/* Return the offset of the last item inside the ziplist. */
#define ZIPLIST_TAIL_OFFSET(zl) (*((uint32_t*)((zl)+sizeof(uint32_t))))
/* Return the length of a ziplist, or UINT16_MAX if the length cannot be
* determined without scanning the whole ziplist. */
#define ZIPLIST_LENGTH(zl) (*((uint16_t*)((zl)+sizeof(uint32_t)*2)))
/* The size of a ziplist header: two 32 bit integers for the total
* bytes count and last item offset. One 16 bit integer for the number
* of items field. */
#define ZIPLIST_HEADER_SIZE (sizeof(uint32_t)*2+sizeof(uint16_t))
/* Size of the "end of ziplist" entry. Just one byte. */
#define ZIPLIST_END_SIZE (sizeof(uint8_t))
/* Return the pointer to the first entry of a ziplist. */
#define ZIPLIST_ENTRY_HEAD(zl) ((zl)+ZIPLIST_HEADER_SIZE)
/* Return the pointer to the last entry of a ziplist, using the
* last entry offset inside the ziplist header. */
#define ZIPLIST_ENTRY_TAIL(zl) ((zl)+intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl)))
/* Return the pointer to the last byte of a ziplist, which is, the
* end of ziplist FF entry. */
#define ZIPLIST_ENTRY_END(zl) ((zl)+intrev32ifbe(ZIPLIST_BYTES(zl))-1)
/* We know a positive increment can only be 1 because entries can only be
* pushed one at a time. */
/* Increment the number of items field in the ziplist header. Note that this
* macro should never overflow the unsigned 16 bit integer, since entires are
* always pushed one at a time. When UINT16_MAX is reached we want the count
* to stay there to signal that a full scan is needed to get the number of
* items inside the ziplist. */
#define ZIPLIST_INCR_LENGTH(zl,incr) { \
if (ZIPLIST_LENGTH(zl) < UINT16_MAX) \
ZIPLIST_LENGTH(zl) = intrev16ifbe(intrev16ifbe(ZIPLIST_LENGTH(zl))+incr); \
}
/* We use this function to receive information about a ziplist entry.
* Note that this is not how the data is actually encoded, is just what we
* get filled by a function in order to operate more easily. */
typedef
struct
zlentry
{
unsigned
int
prevrawlensize
,
prevrawlen
;
unsigned
int
lensize
,
len
;
unsigned
int
headersize
;
unsigned
char
encoding
;
unsigned
char
*
p
;
unsigned
int
prevrawlensize
;
/* Bytes used to encode the previos entry len*/
unsigned
int
prevrawlen
;
/* Previous entry len. */
unsigned
int
lensize
;
/* Bytes used to encode this entry type/len.
For example strings have a 1, 2 or 5 bytes
header. Integers always use a single byte.*/
unsigned
int
len
;
/* Bytes used to represent the actual entry.
For strings this is just the string length
while for integers it is 1, 2, 3, 4, 8 or
0 (for 4 bit immediate) depending on the
number range. */
unsigned
int
headersize
;
/* prevrawlensize + lensize. */
unsigned
char
encoding
;
/* Set to ZIP_STR_* or ZIP_INT_* depending on
the entry encoding. However for 4 bits
immediate integers this can assume a range
of values and must be range-checked. */
unsigned
char
*
p
;
/* Pointer to the very start of the entry, that
is, this points to prev-entry-len field. */
}
zlentry
;
#define ZIPLIST_ENTRY_ZERO(zle) { \
...
...
@@ -178,31 +296,40 @@ typedef struct zlentry {
}
/* Extract the encoding from the byte pointed by 'ptr' and set it into
* 'encoding'. */
* 'encoding'
field of the zlentry structure
. */
#define ZIP_ENTRY_ENCODING(ptr, encoding) do { \
(encoding) = (ptr[0]); \
if ((encoding) < ZIP_STR_MASK) (encoding) &= ZIP_STR_MASK; \
} while(0)
void
ziplistRepr
(
unsigned
char
*
zl
);
/* Return bytes needed to store integer encoded by 'encoding' */
static
unsigned
int
zipIntSize
(
unsigned
char
encoding
)
{
/* Return bytes needed to store integer encoded by 'encoding'. */
unsigned
int
zipIntSize
(
unsigned
char
encoding
)
{
switch
(
encoding
)
{
case
ZIP_INT_8B
:
return
1
;
case
ZIP_INT_16B
:
return
2
;
case
ZIP_INT_24B
:
return
3
;
case
ZIP_INT_32B
:
return
4
;
case
ZIP_INT_64B
:
return
8
;
default:
return
0
;
/* 4 bit immediate */
}
assert
(
NULL
);
if
(
encoding
>=
ZIP_INT_IMM_MIN
&&
encoding
<=
ZIP_INT_IMM_MAX
)
return
0
;
/* 4 bit immediate */
panic
(
"Invalid integer encoding 0x%02X"
,
encoding
);
return
0
;
}
/* Encode the length 'rawlen' writing it in 'p'. If p is NULL it just returns
* the amount of bytes required to encode such a length. */
static
unsigned
int
zipEncodeLength
(
unsigned
char
*
p
,
unsigned
char
encoding
,
unsigned
int
rawlen
)
{
/* Write the encoidng header of the entry in 'p'. If p is NULL it just returns
* the amount of bytes required to encode such a length. Arguments:
*
* 'encoding' is the encoding we are using for the entry. It could be
* ZIP_INT_* or ZIP_STR_* or between ZIP_INT_IMM_MIN and ZIP_INT_IMM_MAX
* for single-byte small immediate integers.
*
* 'rawlen' is only used for ZIP_STR_* encodings and is the length of the
* srting that this entry represents.
*
* The function returns the number of bytes used by the encoding/length
* header stored in 'p'. */
unsigned
int
zipStoreEntryEncoding
(
unsigned
char
*
p
,
unsigned
char
encoding
,
unsigned
int
rawlen
)
{
unsigned
char
len
=
1
,
buf
[
5
];
if
(
ZIP_IS_STR
(
encoding
))
{
...
...
@@ -231,15 +358,16 @@ static unsigned int zipEncodeLength(unsigned char *p, unsigned char encoding, un
buf
[
0
]
=
encoding
;
}
/* Store this length at p */
/* Store this length at p
.
*/
memcpy
(
p
,
buf
,
len
);
return
len
;
}
/* Decode the length encoded in 'ptr'. The 'encoding' variable will hold the
* entries encoding, the 'lensize' variable will hold the number of bytes
* required to encode the entries length, and the 'len' variable will hold the
* entries length. */
/* Decode the entry encoding type and data length (string length for strings,
* number of bytes used for the integer for integer entries) encoded in 'ptr'.
* The 'encoding' variable will hold the entry encoding, the 'lensize'
* variable will hold the number of bytes required to encode the entry
* length, and the 'len' variable will hold the entry length. */
#define ZIP_DECODE_LENGTH(ptr, encoding, lensize, len) do { \
ZIP_ENTRY_ENCODING((ptr), (encoding)); \
if ((encoding) < ZIP_STR_MASK) { \
...
...
@@ -256,7 +384,7 @@ static unsigned int zipEncodeLength(unsigned char *p, unsigned char encoding, un
((ptr)[3] << 8) | \
((ptr)[4]); \
} else { \
assert(NULL);
\
panic("Invalid string encoding 0x%02X", (encoding));
\
} \
} else { \
(lensize) = 1; \
...
...
@@ -264,45 +392,49 @@ static unsigned int zipEncodeLength(unsigned char *p, unsigned char encoding, un
} \
} while(0);
/* Encode the length of the previous entry and write it to "p". This only
* uses the larger encoding (required in __ziplistCascadeUpdate). */
int
zipStorePrevEntryLengthLarge
(
unsigned
char
*
p
,
unsigned
int
len
)
{
if
(
p
!=
NULL
)
{
p
[
0
]
=
ZIP_BIG_PREVLEN
;
memcpy
(
p
+
1
,
&
len
,
sizeof
(
len
));
memrev32ifbe
(
p
+
1
);
}
return
1
+
sizeof
(
len
);
}
/* Encode the length of the previous entry and write it to "p". Return the
* number of bytes needed to encode this length if "p" is NULL. */
static
unsigned
int
zipPrevEn
code
Length
(
unsigned
char
*
p
,
unsigned
int
len
)
{
unsigned
int
zip
Store
PrevEn
try
Length
(
unsigned
char
*
p
,
unsigned
int
len
)
{
if
(
p
==
NULL
)
{
return
(
len
<
ZIP_BIGLEN
)
?
1
:
sizeof
(
len
)
+
1
;
return
(
len
<
ZIP_BIG
_PREV
LEN
)
?
1
:
sizeof
(
len
)
+
1
;
}
else
{
if
(
len
<
ZIP_BIGLEN
)
{
if
(
len
<
ZIP_BIG
_PREV
LEN
)
{
p
[
0
]
=
len
;
return
1
;
}
else
{
p
[
0
]
=
ZIP_BIGLEN
;
memcpy
(
p
+
1
,
&
len
,
sizeof
(
len
));
memrev32ifbe
(
p
+
1
);
return
1
+
sizeof
(
len
);
return
zipStorePrevEntryLengthLarge
(
p
,
len
);
}
}
}
/* Encode the length of the previous entry and write it to "p". This only
* uses the larger encoding (required in __ziplistCascadeUpdate). */
static
void
zipPrevEncodeLengthForceLarge
(
unsigned
char
*
p
,
unsigned
int
len
)
{
if
(
p
==
NULL
)
return
;
p
[
0
]
=
ZIP_BIGLEN
;
memcpy
(
p
+
1
,
&
len
,
sizeof
(
len
));
memrev32ifbe
(
p
+
1
);
}
/* Decode the number of bytes required to store the length of the previous
* element, from the perspective of the entry pointed to by 'ptr'. */
/* Return the number of bytes used to encode the length of the previous
* entry. The length is returned by setting the var 'prevlensize'. */
#define ZIP_DECODE_PREVLENSIZE(ptr, prevlensize) do { \
if ((ptr)[0] < ZIP_BIGLEN) {
\
if ((ptr)[0] < ZIP_BIG
_PREV
LEN) { \
(prevlensize) = 1; \
} else { \
(prevlensize) = 5; \
} \
} while(0);
/* Decode the length of the previous element, from the perspective of the entry
* pointed to by 'ptr'. */
/* Return the length of the previous element, and the number of bytes that
* are used in order to encode the previous element length.
* 'ptr' must point to the prevlen prefix of an entry (that encodes the
* length of the previos entry in order to navigate the elements backward).
* The length of the previous entry is stored in 'prevlen', the number of
* bytes needed to encode the previous entry length are stored in
* 'prevlensize'. */
#define ZIP_DECODE_PREVLEN(ptr, prevlensize, prevlen) do { \
ZIP_DECODE_PREVLENSIZE(ptr, prevlensize); \
if ((prevlensize) == 1) { \
...
...
@@ -314,16 +446,29 @@ static void zipPrevEncodeLengthForceLarge(unsigned char *p, unsigned int len) {
} \
} while(0);
/* Return the difference in number of bytes needed to store the length of the
* previous element 'len', in the entry pointed to by 'p'. */
static
int
zipPrevLenByteDiff
(
unsigned
char
*
p
,
unsigned
int
len
)
{
/* Given a pointer 'p' to the prevlen info that prefixes an entry, this
* function returns the difference in number of bytes needed to encode
* the prevlen if the previous entry changes of size.
*
* So if A is the number of bytes used right now to encode the 'prevlen'
* field.
*
* And B is the number of bytes that are needed in order to encode the
* 'prevlen' if the previous element will be updated to one of size 'len'.
*
* Then the function returns B - A
*
* So the function returns a positive number if more space is needed,
* a negative number if less space is needed, or zero if the same space
* is needed. */
int
zipPrevLenByteDiff
(
unsigned
char
*
p
,
unsigned
int
len
)
{
unsigned
int
prevlensize
;
ZIP_DECODE_PREVLENSIZE
(
p
,
prevlensize
);
return
zipPrevEn
code
Length
(
NULL
,
len
)
-
prevlensize
;
return
zip
Store
PrevEn
try
Length
(
NULL
,
len
)
-
prevlensize
;
}
/* Return the total number of bytes used by the entry pointed to by 'p'. */
static
unsigned
int
zipRawEntryLength
(
unsigned
char
*
p
)
{
unsigned
int
zipRawEntryLength
(
unsigned
char
*
p
)
{
unsigned
int
prevlensize
,
encoding
,
lensize
,
len
;
ZIP_DECODE_PREVLENSIZE
(
p
,
prevlensize
);
ZIP_DECODE_LENGTH
(
p
+
prevlensize
,
encoding
,
lensize
,
len
);
...
...
@@ -332,7 +477,7 @@ static unsigned int zipRawEntryLength(unsigned char *p) {
/* Check if string pointed to by 'entry' can be encoded as an integer.
* Stores the integer value in 'v' and its encoding in 'encoding'. */
static
int
zipTryEncoding
(
unsigned
char
*
entry
,
unsigned
int
entrylen
,
long
long
*
v
,
unsigned
char
*
encoding
)
{
int
zipTryEncoding
(
unsigned
char
*
entry
,
unsigned
int
entrylen
,
long
long
*
v
,
unsigned
char
*
encoding
)
{
long
long
value
;
if
(
entrylen
>=
32
||
entrylen
==
0
)
return
0
;
...
...
@@ -359,7 +504,7 @@ static int zipTryEncoding(unsigned char *entry, unsigned int entrylen, long long
}
/* Store integer 'value' at 'p', encoded as 'encoding' */
static
void
zipSaveInteger
(
unsigned
char
*
p
,
int64_t
value
,
unsigned
char
encoding
)
{
void
zipSaveInteger
(
unsigned
char
*
p
,
int64_t
value
,
unsigned
char
encoding
)
{
int16_t
i16
;
int32_t
i32
;
int64_t
i64
;
...
...
@@ -389,7 +534,7 @@ static void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encodi
}
/* Read integer encoded as 'encoding' from 'p' */
static
int64_t
zipLoadInteger
(
unsigned
char
*
p
,
unsigned
char
encoding
)
{
int64_t
zipLoadInteger
(
unsigned
char
*
p
,
unsigned
char
encoding
)
{
int16_t
i16
;
int32_t
i32
;
int64_t
i64
,
ret
=
0
;
...
...
@@ -421,7 +566,7 @@ static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) {
}
/* Return a struct with all information about an entry. */
static
void
zipEntry
(
unsigned
char
*
p
,
zlentry
*
e
)
{
void
zipEntry
(
unsigned
char
*
p
,
zlentry
*
e
)
{
ZIP_DECODE_PREVLEN
(
p
,
e
->
prevrawlensize
,
e
->
prevrawlen
);
ZIP_DECODE_LENGTH
(
p
+
e
->
prevrawlensize
,
e
->
encoding
,
e
->
lensize
,
e
->
len
);
...
...
@@ -441,7 +586,7 @@ unsigned char *ziplistNew(void) {
}
/* Resize the ziplist. */
static
unsigned
char
*
ziplistResize
(
unsigned
char
*
zl
,
unsigned
int
len
)
{
unsigned
char
*
ziplistResize
(
unsigned
char
*
zl
,
unsigned
int
len
)
{
zl
=
zrealloc
(
zl
,
len
);
ZIPLIST_BYTES
(
zl
)
=
intrev32ifbe
(
len
);
zl
[
len
-
1
]
=
ZIP_END
;
...
...
@@ -456,8 +601,8 @@ static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
* causes a realloc and memmove). However, encoding the prevlen may require
* that this entry is grown as well. This effect may cascade throughout
* the ziplist when there are consecutive entries with a size close to
* ZIP_BIGLEN, so we need to check that the prevlen can be encoded in
every
* consecutive entry.
* ZIP_BIG
_PREV
LEN, so we need to check that the prevlen can be encoded in
*
every
consecutive entry.
*
* Note that this effect can also happen in reverse, where the bytes required
* to encode the prevlen field can shrink. This effect is deliberately ignored,
...
...
@@ -468,7 +613,7 @@ static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
*
* The pointer "p" points to the first entry that does NOT need to be
* updated, i.e. consecutive fields MAY need an update. */
static
unsigned
char
*
__ziplistCascadeUpdate
(
unsigned
char
*
zl
,
unsigned
char
*
p
)
{
unsigned
char
*
__ziplistCascadeUpdate
(
unsigned
char
*
zl
,
unsigned
char
*
p
)
{
size_t
curlen
=
intrev32ifbe
(
ZIPLIST_BYTES
(
zl
)),
rawlen
,
rawlensize
;
size_t
offset
,
noffset
,
extra
;
unsigned
char
*
np
;
...
...
@@ -477,7 +622,7 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p
while
(
p
[
0
]
!=
ZIP_END
)
{
zipEntry
(
p
,
&
cur
);
rawlen
=
cur
.
headersize
+
cur
.
len
;
rawlensize
=
zipPrevEn
code
Length
(
NULL
,
rawlen
);
rawlensize
=
zip
Store
PrevEn
try
Length
(
NULL
,
rawlen
);
/* Abort if there is no next entry. */
if
(
p
[
rawlen
]
==
ZIP_END
)
break
;
...
...
@@ -508,7 +653,7 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p
memmove
(
np
+
rawlensize
,
np
+
next
.
prevrawlensize
,
curlen
-
noffset
-
next
.
prevrawlensize
-
1
);
zipPrevEn
code
Length
(
np
,
rawlen
);
zip
Store
PrevEn
try
Length
(
np
,
rawlen
);
/* Advance the cursor */
p
+=
rawlen
;
...
...
@@ -517,9 +662,9 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p
if
(
next
.
prevrawlensize
>
rawlensize
)
{
/* This would result in shrinking, which we want to avoid.
* So, set "rawlen" in the available bytes. */
zipPrevEn
code
Length
Force
Large
(
p
+
rawlen
,
rawlen
);
zip
Store
PrevEn
try
LengthLarge
(
p
+
rawlen
,
rawlen
);
}
else
{
zipPrevEn
code
Length
(
p
+
rawlen
,
rawlen
);
zip
Store
PrevEn
try
Length
(
p
+
rawlen
,
rawlen
);
}
/* Stop here, as the raw length of "next" has not changed. */
...
...
@@ -530,7 +675,7 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p
}
/* Delete "num" entries, starting at "p". Returns pointer to the ziplist. */
static
unsigned
char
*
__ziplistDelete
(
unsigned
char
*
zl
,
unsigned
char
*
p
,
unsigned
int
num
)
{
unsigned
char
*
__ziplistDelete
(
unsigned
char
*
zl
,
unsigned
char
*
p
,
unsigned
int
num
)
{
unsigned
int
i
,
totlen
,
deleted
=
0
;
size_t
offset
;
int
nextdiff
=
0
;
...
...
@@ -542,7 +687,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
deleted
++
;
}
totlen
=
p
-
first
.
p
;
totlen
=
p
-
first
.
p
;
/* Bytes taken by the element(s) to delete. */
if
(
totlen
>
0
)
{
if
(
p
[
0
]
!=
ZIP_END
)
{
/* Storing `prevrawlen` in this entry may increase or decrease the
...
...
@@ -550,8 +695,13 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
* There always is room to store this, because it was previously
* stored by an entry that is now being deleted. */
nextdiff
=
zipPrevLenByteDiff
(
p
,
first
.
prevrawlen
);
/* Note that there is always space when p jumps backward: if
* the new previous entry is large, one of the deleted elements
* had a 5 bytes prevlen header, so there is for sure at least
* 5 bytes free and we need just 4. */
p
-=
nextdiff
;
zipPrevEn
code
Length
(
p
,
first
.
prevrawlen
);
zip
Store
PrevEn
try
Length
(
p
,
first
.
prevrawlen
);
/* Update offset for tail */
ZIPLIST_TAIL_OFFSET
(
zl
)
=
...
...
@@ -590,7 +740,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
}
/* Insert item at "p". */
static
unsigned
char
*
__ziplistInsert
(
unsigned
char
*
zl
,
unsigned
char
*
p
,
unsigned
char
*
s
,
unsigned
int
slen
)
{
unsigned
char
*
__ziplistInsert
(
unsigned
char
*
zl
,
unsigned
char
*
p
,
unsigned
char
*
s
,
unsigned
int
slen
)
{
size_t
curlen
=
intrev32ifbe
(
ZIPLIST_BYTES
(
zl
)),
reqlen
;
unsigned
int
prevlensize
,
prevlen
=
0
;
size_t
offset
;
...
...
@@ -616,19 +766,24 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
/* 'encoding' is set to the appropriate integer encoding */
reqlen
=
zipIntSize
(
encoding
);
}
else
{
/* 'encoding' is untouched, however zip
EncodeLength
will use the
/* 'encoding' is untouched, however zip
StoreEntryEncoding
will use the
* string length to figure out how to encode it. */
reqlen
=
slen
;
}
/* We need space for both the length of the previous entry and
* the length of the payload. */
reqlen
+=
zipPrevEn
code
Length
(
NULL
,
prevlen
);
reqlen
+=
zip
EncodeLength
(
NULL
,
encoding
,
slen
);
reqlen
+=
zip
Store
PrevEn
try
Length
(
NULL
,
prevlen
);
reqlen
+=
zip
StoreEntryEncoding
(
NULL
,
encoding
,
slen
);
/* When the insert position is not equal to the tail, we need to
* make sure that the next entry can hold this entry's length in
* its prevlen field. */
int
forcelarge
=
0
;
nextdiff
=
(
p
[
0
]
!=
ZIP_END
)
?
zipPrevLenByteDiff
(
p
,
reqlen
)
:
0
;
if
(
nextdiff
==
-
4
&&
reqlen
<
4
)
{
nextdiff
=
0
;
forcelarge
=
1
;
}
/* Store offset because a realloc may change the address of zl. */
offset
=
p
-
zl
;
...
...
@@ -641,7 +796,10 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
memmove
(
p
+
reqlen
,
p
-
nextdiff
,
curlen
-
offset
-
1
+
nextdiff
);
/* Encode this entry's raw length in the next entry. */
zipPrevEncodeLength
(
p
+
reqlen
,
reqlen
);
if
(
forcelarge
)
zipStorePrevEntryLengthLarge
(
p
+
reqlen
,
reqlen
);
else
zipStorePrevEntryLength
(
p
+
reqlen
,
reqlen
);
/* Update offset for tail */
ZIPLIST_TAIL_OFFSET
(
zl
)
=
...
...
@@ -669,8 +827,8 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
}
/* Write the entry */
p
+=
zipPrevEn
code
Length
(
p
,
prevlen
);
p
+=
zip
EncodeLength
(
p
,
encoding
,
slen
);
p
+=
zip
Store
PrevEn
try
Length
(
p
,
prevlen
);
p
+=
zip
StoreEntryEncoding
(
p
,
encoding
,
slen
);
if
(
ZIP_IS_STR
(
encoding
))
{
memcpy
(
p
,
s
,
slen
);
}
else
{
...
...
tests/integration/aof.tcl
View file @
71a8df6a
...
...
@@ -88,7 +88,7 @@ tags {"aof"} {
set pattern
"*Bad file format reading the append only file*"
set retry 10
while
{
$retry
}
{
set result
[
exec tail -
n
1 <
[
dict get $srv stdout
]]
set result
[
exec tail -1 <
[
dict get $srv stdout
]]
if
{[
string match $pattern $result
]}
{
break
}
...
...
@@ -113,7 +113,7 @@ tags {"aof"} {
set pattern
"*Unexpected end of file reading the append only file*"
set retry 10
while
{
$retry
}
{
set result
[
exec tail -
n
1 <
[
dict get $srv stdout
]]
set result
[
exec tail -1 <
[
dict get $srv stdout
]]
if
{[
string match $pattern $result
]}
{
break
}
...
...
@@ -137,7 +137,7 @@ tags {"aof"} {
set pattern
"*Unexpected end of file reading the append only file*"
set retry 10
while
{
$retry
}
{
set result
[
exec tail -
n
1 <
[
dict get $srv stdout
]]
set result
[
exec tail -1 <
[
dict get $srv stdout
]]
if
{[
string match $pattern $result
]}
{
break
}
...
...
tests/integration/rdb.tcl
View file @
71a8df6a
...
...
@@ -66,7 +66,7 @@ if {!$isroot} {
test
{
Server should not start if RDB file can't be open
}
{
wait_for_condition 50 100
{
[
string match
{
*Fatal error loading*
}
\
[
exec tail -
n
1 <
[
dict get $srv stdout
]]]
[
exec tail -1 <
[
dict get $srv stdout
]]]
}
else
{
fail
"Server started even if RDB was unreadable!"
}
...
...
@@ -90,7 +90,7 @@ start_server_and_kill_it [list "dir" $server_path] {
test
{
Server should not start if RDB is corrupted
}
{
wait_for_condition 50 100
{
[
string match
{
*CRC error*
}
\
[
exec tail -
n
10 <
[
dict get $srv stdout
]]]
[
exec tail -10 <
[
dict get $srv stdout
]]]
}
else
{
fail
"Server started even if RDB was corrupted!"
}
...
...
tests/integration/replication-psync.tcl
View file @
71a8df6a
...
...
@@ -47,7 +47,7 @@ proc test_psync {descr duration backlog_size backlog_ttl delay cond diskless rec
# Check that the background clients are actually writing.
test
{
Detect write load to master
}
{
wait_for_condition 50 100
{
wait_for_condition 50 100
0
{
[
$master
dbsize
]
> 100
}
else
{
fail
"Can't detect write load from background clients."
...
...
Prev
1
…
5
6
7
8
9
10
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment