Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
71a8df6a
Commit
71a8df6a
authored
Mar 02, 2017
by
Guy Benoish
Browse files
Merge branch 'unstable' of
https://github.com/antirez/redis
into unstable
parents
56c01c95
9cc83d2a
Changes
182
Hide whitespace changes
Inline
Side-by-side
src/dict.h
View file @
71a8df6a
...
@@ -56,7 +56,7 @@ typedef struct dictEntry {
...
@@ -56,7 +56,7 @@ typedef struct dictEntry {
}
dictEntry
;
}
dictEntry
;
typedef
struct
dictType
{
typedef
struct
dictType
{
u
nsigned
in
t
(
*
hashFunction
)(
const
void
*
key
);
u
int64_
t
(
*
hashFunction
)(
const
void
*
key
);
void
*
(
*
keyDup
)(
void
*
privdata
,
const
void
*
key
);
void
*
(
*
keyDup
)(
void
*
privdata
,
const
void
*
key
);
void
*
(
*
valDup
)(
void
*
privdata
,
const
void
*
obj
);
void
*
(
*
valDup
)(
void
*
privdata
,
const
void
*
obj
);
int
(
*
keyCompare
)(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
int
(
*
keyCompare
)(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
...
@@ -168,15 +168,15 @@ void dictReleaseIterator(dictIterator *iter);
...
@@ -168,15 +168,15 @@ void dictReleaseIterator(dictIterator *iter);
dictEntry
*
dictGetRandomKey
(
dict
*
d
);
dictEntry
*
dictGetRandomKey
(
dict
*
d
);
unsigned
int
dictGetSomeKeys
(
dict
*
d
,
dictEntry
**
des
,
unsigned
int
count
);
unsigned
int
dictGetSomeKeys
(
dict
*
d
,
dictEntry
**
des
,
unsigned
int
count
);
void
dictGetStats
(
char
*
buf
,
size_t
bufsize
,
dict
*
d
);
void
dictGetStats
(
char
*
buf
,
size_t
bufsize
,
dict
*
d
);
u
nsigned
in
t
dictGenHashFunction
(
const
void
*
key
,
int
len
);
u
int64_
t
dictGenHashFunction
(
const
void
*
key
,
int
len
);
u
nsigned
in
t
dictGenCaseHashFunction
(
const
unsigned
char
*
buf
,
int
len
);
u
int64_
t
dictGenCaseHashFunction
(
const
unsigned
char
*
buf
,
int
len
);
void
dictEmpty
(
dict
*
d
,
void
(
callback
)(
void
*
));
void
dictEmpty
(
dict
*
d
,
void
(
callback
)(
void
*
));
void
dictEnableResize
(
void
);
void
dictEnableResize
(
void
);
void
dictDisableResize
(
void
);
void
dictDisableResize
(
void
);
int
dictRehash
(
dict
*
d
,
int
n
);
int
dictRehash
(
dict
*
d
,
int
n
);
int
dictRehashMilliseconds
(
dict
*
d
,
int
ms
);
int
dictRehashMilliseconds
(
dict
*
d
,
int
ms
);
void
dictSetHashFunctionSeed
(
u
nsigned
int
initval
);
void
dictSetHashFunctionSeed
(
u
int8_t
*
seed
);
u
nsigned
in
t
dictGetHashFunctionSeed
(
void
);
u
int8_
t
*
dictGetHashFunctionSeed
(
void
);
unsigned
long
dictScan
(
dict
*
d
,
unsigned
long
v
,
dictScanFunction
*
fn
,
dictScanBucketFunction
*
bucketfn
,
void
*
privdata
);
unsigned
long
dictScan
(
dict
*
d
,
unsigned
long
v
,
dictScanFunction
*
fn
,
dictScanBucketFunction
*
bucketfn
,
void
*
privdata
);
unsigned
int
dictGetHash
(
dict
*
d
,
const
void
*
key
);
unsigned
int
dictGetHash
(
dict
*
d
,
const
void
*
key
);
dictEntry
**
dictFindEntryRefByPtrAndHash
(
dict
*
d
,
const
void
*
oldptr
,
unsigned
int
hash
);
dictEntry
**
dictFindEntryRefByPtrAndHash
(
dict
*
d
,
const
void
*
oldptr
,
unsigned
int
hash
);
...
...
src/evict.c
View file @
71a8df6a
...
@@ -336,20 +336,13 @@ unsigned long LFUDecrAndReturn(robj *o) {
...
@@ -336,20 +336,13 @@ unsigned long LFUDecrAndReturn(robj *o) {
* server when there is data to add in order to make space if needed.
* server when there is data to add in order to make space if needed.
* --------------------------------------------------------------------------*/
* --------------------------------------------------------------------------*/
int
freeMemoryIfNeeded
(
void
)
{
/* We don't want to count AOF buffers and slaves output buffers as
size_t
mem_reported
,
mem_used
,
mem_tofree
,
mem_freed
;
* used memory: the eviction should use mostly data size. This function
* returns the sum of AOF and slaves buffer. */
size_t
freeMemoryGetNotCountedMemory
(
void
)
{
size_t
overhead
=
0
;
int
slaves
=
listLength
(
server
.
slaves
);
int
slaves
=
listLength
(
server
.
slaves
);
mstime_t
latency
,
eviction_latency
;
long
long
delta
;
/* Check if we are over the memory usage limit. If we are not, no need
* to subtract the slaves output buffers. We can just return ASAP. */
mem_reported
=
zmalloc_used_memory
();
if
(
mem_reported
<=
server
.
maxmemory
)
return
C_OK
;
/* Remove the size of slaves output buffers and AOF buffer from the
* count of used memory. */
mem_used
=
mem_reported
;
if
(
slaves
)
{
if
(
slaves
)
{
listIter
li
;
listIter
li
;
listNode
*
ln
;
listNode
*
ln
;
...
@@ -357,17 +350,31 @@ int freeMemoryIfNeeded(void) {
...
@@ -357,17 +350,31 @@ int freeMemoryIfNeeded(void) {
listRewind
(
server
.
slaves
,
&
li
);
listRewind
(
server
.
slaves
,
&
li
);
while
((
ln
=
listNext
(
&
li
)))
{
while
((
ln
=
listNext
(
&
li
)))
{
client
*
slave
=
listNodeValue
(
ln
);
client
*
slave
=
listNodeValue
(
ln
);
unsigned
long
obuf_bytes
=
getClientOutputBufferMemoryUsage
(
slave
);
overhead
+=
getClientOutputBufferMemoryUsage
(
slave
);
if
(
obuf_bytes
>
mem_used
)
mem_used
=
0
;
else
mem_used
-=
obuf_bytes
;
}
}
}
}
if
(
server
.
aof_state
!=
AOF_OFF
)
{
if
(
server
.
aof_state
!=
AOF_OFF
)
{
mem_used
-=
sdslen
(
server
.
aof_buf
);
overhead
+=
sdslen
(
server
.
aof_buf
)
+
aofRewriteBufferSize
();
mem_used
-=
aofRewriteBufferSize
();
}
}
return
overhead
;
}
int
freeMemoryIfNeeded
(
void
)
{
size_t
mem_reported
,
mem_used
,
mem_tofree
,
mem_freed
;
mstime_t
latency
,
eviction_latency
;
long
long
delta
;
int
slaves
=
listLength
(
server
.
slaves
);
/* Check if we are over the memory usage limit. If we are not, no need
* to subtract the slaves output buffers. We can just return ASAP. */
mem_reported
=
zmalloc_used_memory
();
if
(
mem_reported
<=
server
.
maxmemory
)
return
C_OK
;
/* Remove the size of slaves output buffers and AOF buffer from the
* count of used memory. */
mem_used
=
mem_reported
;
size_t
overhead
=
freeMemoryGetNotCountedMemory
();
mem_used
=
(
mem_used
>
overhead
)
?
mem_used
-
overhead
:
0
;
/* Check if we are still over the memory limit. */
/* Check if we are still over the memory limit. */
if
(
mem_used
<=
server
.
maxmemory
)
return
C_OK
;
if
(
mem_used
<=
server
.
maxmemory
)
return
C_OK
;
...
@@ -498,6 +505,22 @@ int freeMemoryIfNeeded(void) {
...
@@ -498,6 +505,22 @@ int freeMemoryIfNeeded(void) {
* deliver data to the slaves fast enough, so we force the
* deliver data to the slaves fast enough, so we force the
* transmission here inside the loop. */
* transmission here inside the loop. */
if
(
slaves
)
flushSlavesOutputBuffers
();
if
(
slaves
)
flushSlavesOutputBuffers
();
/* Normally our stop condition is the ability to release
* a fixed, pre-computed amount of memory. However when we
* are deleting objects in another thread, it's better to
* check, from time to time, if we already reached our target
* memory, since the "mem_freed" amount is computed only
* across the dbAsyncDelete() call, while the thread can
* release the memory all the time. */
if
(
server
.
lazyfree_lazy_eviction
&&
!
(
keys_freed
%
16
))
{
overhead
=
freeMemoryGetNotCountedMemory
();
mem_used
=
zmalloc_used_memory
();
mem_used
=
(
mem_used
>
overhead
)
?
mem_used
-
overhead
:
0
;
if
(
mem_used
<=
server
.
maxmemory
)
{
mem_freed
=
mem_tofree
;
}
}
}
}
if
(
!
keys_freed
)
{
if
(
!
keys_freed
)
{
...
...
src/hyperloglog.c
View file @
71a8df6a
...
@@ -401,7 +401,11 @@ uint64_t MurmurHash64A (const void * key, int len, unsigned int seed) {
...
@@ -401,7 +401,11 @@ uint64_t MurmurHash64A (const void * key, int len, unsigned int seed) {
uint64_t
k
;
uint64_t
k
;
#if (BYTE_ORDER == LITTLE_ENDIAN)
#if (BYTE_ORDER == LITTLE_ENDIAN)
#ifdef USE_ALIGNED_ACCESS
memcpy
(
&
k
,
data
,
sizeof
(
uint64_t
));
#else
k
=
*
((
uint64_t
*
)
data
);
k
=
*
((
uint64_t
*
)
data
);
#endif
#else
#else
k
=
(
uint64_t
)
data
[
0
];
k
=
(
uint64_t
)
data
[
0
];
k
|=
(
uint64_t
)
data
[
1
]
<<
8
;
k
|=
(
uint64_t
)
data
[
1
]
<<
8
;
...
...
src/latency.c
View file @
71a8df6a
...
@@ -41,7 +41,7 @@ int dictStringKeyCompare(void *privdata, const void *key1, const void *key2) {
...
@@ -41,7 +41,7 @@ int dictStringKeyCompare(void *privdata, const void *key1, const void *key2) {
return
strcmp
(
key1
,
key2
)
==
0
;
return
strcmp
(
key1
,
key2
)
==
0
;
}
}
u
nsigned
in
t
dictStringHash
(
const
void
*
key
)
{
u
int64_
t
dictStringHash
(
const
void
*
key
)
{
return
dictGenHashFunction
(
key
,
strlen
(
key
));
return
dictGenHashFunction
(
key
,
strlen
(
key
));
}
}
...
...
src/module.c
View file @
71a8df6a
...
@@ -2743,8 +2743,8 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver,
...
@@ -2743,8 +2743,8 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver,
moduleTypeLoadFunc
rdb_load
;
moduleTypeLoadFunc
rdb_load
;
moduleTypeSaveFunc
rdb_save
;
moduleTypeSaveFunc
rdb_save
;
moduleTypeRewriteFunc
aof_rewrite
;
moduleTypeRewriteFunc
aof_rewrite
;
moduleTypeDigestFunc
digest
;
moduleTypeMemUsageFunc
mem_usage
;
moduleTypeMemUsageFunc
mem_usage
;
moduleTypeDigestFunc
digest
;
moduleTypeFreeFunc
free
;
moduleTypeFreeFunc
free
;
}
*
tms
=
(
struct
typemethods
*
)
typemethods_ptr
;
}
*
tms
=
(
struct
typemethods
*
)
typemethods_ptr
;
...
@@ -3264,7 +3264,7 @@ void *RM_GetBlockedClientPrivateData(RedisModuleCtx *ctx) {
...
@@ -3264,7 +3264,7 @@ void *RM_GetBlockedClientPrivateData(RedisModuleCtx *ctx) {
/* server.moduleapi dictionary type. Only uses plain C strings since
/* server.moduleapi dictionary type. Only uses plain C strings since
* this gets queries from modules. */
* this gets queries from modules. */
u
nsigned
in
t
dictCStringKeyHash
(
const
void
*
key
)
{
u
int64_
t
dictCStringKeyHash
(
const
void
*
key
)
{
return
dictGenHashFunction
((
unsigned
char
*
)
key
,
strlen
((
char
*
)
key
));
return
dictGenHashFunction
((
unsigned
char
*
)
key
,
strlen
((
char
*
)
key
));
}
}
...
...
src/modules/hellotype.c
View file @
71a8df6a
...
@@ -226,10 +226,12 @@ void HelloTypeAofRewrite(RedisModuleIO *aof, RedisModuleString *key, void *value
...
@@ -226,10 +226,12 @@ void HelloTypeAofRewrite(RedisModuleIO *aof, RedisModuleString *key, void *value
}
}
}
}
void
HelloTypeDigest
(
RedisModuleDigest
*
digest
,
void
*
value
)
{
/* The goal of this function is to return the amount of memory used by
REDISMODULE_NOT_USED
(
digest
);
* the HelloType value. */
REDISMODULE_NOT_USED
(
value
);
size_t
HelloTypeMemUsage
(
const
void
*
value
)
{
/* TODO: The DIGEST module interface is yet not implemented. */
const
struct
HelloTypeObject
*
hto
=
value
;
struct
HelloTypeNode
*
node
=
hto
->
head
;
return
sizeof
(
*
hto
)
+
sizeof
(
*
node
)
*
hto
->
len
;
}
}
void
HelloTypeFree
(
void
*
value
)
{
void
HelloTypeFree
(
void
*
value
)
{
...
@@ -250,6 +252,7 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
...
@@ -250,6 +252,7 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
.
rdb_load
=
HelloTypeRdbLoad
,
.
rdb_load
=
HelloTypeRdbLoad
,
.
rdb_save
=
HelloTypeRdbSave
,
.
rdb_save
=
HelloTypeRdbSave
,
.
aof_rewrite
=
HelloTypeAofRewrite
,
.
aof_rewrite
=
HelloTypeAofRewrite
,
.
mem_usage
=
HelloTypeMemUsage
,
.
free
=
HelloTypeFree
.
free
=
HelloTypeFree
};
};
...
...
src/object.c
View file @
71a8df6a
...
@@ -246,11 +246,9 @@ void freeStringObject(robj *o) {
...
@@ -246,11 +246,9 @@ void freeStringObject(robj *o) {
}
}
void
freeListObject
(
robj
*
o
)
{
void
freeListObject
(
robj
*
o
)
{
switch
(
o
->
encoding
)
{
if
(
o
->
encoding
==
OBJ_ENCODING_QUICKLIST
)
{
case
OBJ_ENCODING_QUICKLIST
:
quicklistRelease
(
o
->
ptr
);
quicklistRelease
(
o
->
ptr
);
break
;
}
else
{
default:
serverPanic
(
"Unknown list encoding type"
);
serverPanic
(
"Unknown list encoding type"
);
}
}
}
}
...
@@ -786,6 +784,14 @@ size_t objectComputeSize(robj *o, size_t sample_size) {
...
@@ -786,6 +784,14 @@ size_t objectComputeSize(robj *o, size_t sample_size) {
}
else
{
}
else
{
serverPanic
(
"Unknown hash encoding"
);
serverPanic
(
"Unknown hash encoding"
);
}
}
}
else
if
(
o
->
type
==
OBJ_MODULE
)
{
moduleValue
*
mv
=
o
->
ptr
;
moduleType
*
mt
=
mv
->
type
;
if
(
mt
->
mem_usage
!=
NULL
)
{
asize
=
mt
->
mem_usage
(
mv
->
value
);
}
else
{
asize
=
0
;
}
}
else
{
}
else
{
serverPanic
(
"Unknown object type"
);
serverPanic
(
"Unknown object type"
);
}
}
...
@@ -945,7 +951,7 @@ sds getMemoryDoctorReport(void) {
...
@@ -945,7 +951,7 @@ sds getMemoryDoctorReport(void) {
}
}
/* Slaves using more than 10 MB each? */
/* Slaves using more than 10 MB each? */
if
(
mh
->
clients_slaves
/
numslaves
>
(
1024
*
1024
*
10
))
{
if
(
numslaves
>
0
&&
mh
->
clients_slaves
/
numslaves
>
(
1024
*
1024
*
10
))
{
big_slave_buf
=
1
;
big_slave_buf
=
1
;
num_reports
++
;
num_reports
++
;
}
}
...
...
src/redis-cli.c
View file @
71a8df6a
...
@@ -1273,6 +1273,11 @@ static void repl(void) {
...
@@ -1273,6 +1273,11 @@ static void repl(void) {
int
argc
;
int
argc
;
sds
*
argv
;
sds
*
argv
;
/* Initialize the help and, if possible, use the COMMAND command in order
* to retrieve missing entries. */
cliInitHelp
();
cliIntegrateHelp
();
config
.
interactive
=
1
;
config
.
interactive
=
1
;
linenoiseSetMultiLine
(
1
);
linenoiseSetMultiLine
(
1
);
linenoiseSetCompletionCallback
(
completionCallback
);
linenoiseSetCompletionCallback
(
completionCallback
);
...
@@ -2606,11 +2611,6 @@ int main(int argc, char **argv) {
...
@@ -2606,11 +2611,6 @@ int main(int argc, char **argv) {
argc
-=
firstarg
;
argc
-=
firstarg
;
argv
+=
firstarg
;
argv
+=
firstarg
;
/* Initialize the help and, if possible, use the COMMAND command in order
* to retrieve missing entries. */
cliInitHelp
();
cliIntegrateHelp
();
/* Latency mode */
/* Latency mode */
if
(
config
.
latency_mode
)
{
if
(
config
.
latency_mode
)
{
if
(
cliConnect
(
0
)
==
REDIS_ERR
)
exit
(
1
);
if
(
cliConnect
(
0
)
==
REDIS_ERR
)
exit
(
1
);
...
...
src/redisassert.h
View file @
71a8df6a
...
@@ -41,7 +41,9 @@
...
@@ -41,7 +41,9 @@
#include <unistd.h>
/* for _exit() */
#include <unistd.h>
/* for _exit() */
#define assert(_e) ((_e)?(void)0 : (_serverAssert(#_e,__FILE__,__LINE__),_exit(1)))
#define assert(_e) ((_e)?(void)0 : (_serverAssert(#_e,__FILE__,__LINE__),_exit(1)))
#define panic(...) _serverPanic(__FILE__,__LINE__,__VA_ARGS__),_exit(1)
void
_serverAssert
(
char
*
estr
,
char
*
file
,
int
line
);
void
_serverAssert
(
char
*
estr
,
char
*
file
,
int
line
);
void
_serverPanic
(
const
char
*
file
,
int
line
,
const
char
*
msg
,
...);
#endif
#endif
src/redismodule.h
View file @
71a8df6a
...
@@ -91,7 +91,7 @@ typedef int (*RedisModuleCmdFunc) (RedisModuleCtx *ctx, RedisModuleString **argv
...
@@ -91,7 +91,7 @@ typedef int (*RedisModuleCmdFunc) (RedisModuleCtx *ctx, RedisModuleString **argv
typedef
void
*
(
*
RedisModuleTypeLoadFunc
)(
RedisModuleIO
*
rdb
,
int
encver
);
typedef
void
*
(
*
RedisModuleTypeLoadFunc
)(
RedisModuleIO
*
rdb
,
int
encver
);
typedef
void
(
*
RedisModuleTypeSaveFunc
)(
RedisModuleIO
*
rdb
,
void
*
value
);
typedef
void
(
*
RedisModuleTypeSaveFunc
)(
RedisModuleIO
*
rdb
,
void
*
value
);
typedef
void
(
*
RedisModuleTypeRewriteFunc
)(
RedisModuleIO
*
aof
,
RedisModuleString
*
key
,
void
*
value
);
typedef
void
(
*
RedisModuleTypeRewriteFunc
)(
RedisModuleIO
*
aof
,
RedisModuleString
*
key
,
void
*
value
);
typedef
size_t
(
*
RedisModuleTypeMemUsageFunc
)(
void
*
value
);
typedef
size_t
(
*
RedisModuleTypeMemUsageFunc
)(
const
void
*
value
);
typedef
void
(
*
RedisModuleTypeDigestFunc
)(
RedisModuleDigest
*
digest
,
void
*
value
);
typedef
void
(
*
RedisModuleTypeDigestFunc
)(
RedisModuleDigest
*
digest
,
void
*
value
);
typedef
void
(
*
RedisModuleTypeFreeFunc
)(
void
*
value
);
typedef
void
(
*
RedisModuleTypeFreeFunc
)(
void
*
value
);
...
...
src/replication.c
View file @
71a8df6a
...
@@ -1568,7 +1568,7 @@ int slaveTryPartialResynchronization(int fd, int read_reply) {
...
@@ -1568,7 +1568,7 @@ int slaveTryPartialResynchronization(int fd, int read_reply) {
* establish a connection with the master. */
* establish a connection with the master. */
void
syncWithMaster
(
aeEventLoop
*
el
,
int
fd
,
void
*
privdata
,
int
mask
)
{
void
syncWithMaster
(
aeEventLoop
*
el
,
int
fd
,
void
*
privdata
,
int
mask
)
{
char
tmpfile
[
256
],
*
err
=
NULL
;
char
tmpfile
[
256
],
*
err
=
NULL
;
int
dfd
,
maxtries
=
5
;
int
dfd
=
-
1
,
maxtries
=
5
;
int
sockerr
=
0
,
psync_result
;
int
sockerr
=
0
,
psync_result
;
socklen_t
errlen
=
sizeof
(
sockerr
);
socklen_t
errlen
=
sizeof
(
sockerr
);
UNUSED
(
el
);
UNUSED
(
el
);
...
@@ -1832,6 +1832,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) {
...
@@ -1832,6 +1832,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) {
error:
error:
aeDeleteFileEvent
(
server
.
el
,
fd
,
AE_READABLE
|
AE_WRITABLE
);
aeDeleteFileEvent
(
server
.
el
,
fd
,
AE_READABLE
|
AE_WRITABLE
);
if
(
dfd
!=
-
1
)
close
(
dfd
);
close
(
fd
);
close
(
fd
);
server
.
repl_transfer_s
=
-
1
;
server
.
repl_transfer_s
=
-
1
;
server
.
repl_state
=
REPL_STATE_CONNECT
;
server
.
repl_state
=
REPL_STATE_CONNECT
;
...
...
src/sentinel.c
View file @
71a8df6a
...
@@ -379,7 +379,7 @@ void sentinelSimFailureCrash(void);
...
@@ -379,7 +379,7 @@ void sentinelSimFailureCrash(void);
/* ========================= Dictionary types =============================== */
/* ========================= Dictionary types =============================== */
u
nsigned
in
t
dictSdsHash
(
const
void
*
key
);
u
int64_
t
dictSdsHash
(
const
void
*
key
);
int
dictSdsKeyCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
int
dictSdsKeyCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
void
releaseSentinelRedisInstance
(
sentinelRedisInstance
*
ri
);
void
releaseSentinelRedisInstance
(
sentinelRedisInstance
*
ri
);
...
...
src/server.c
View file @
71a8df6a
...
@@ -482,16 +482,16 @@ int dictObjKeyCompare(void *privdata, const void *key1,
...
@@ -482,16 +482,16 @@ int dictObjKeyCompare(void *privdata, const void *key1,
return
dictSdsKeyCompare
(
privdata
,
o1
->
ptr
,
o2
->
ptr
);
return
dictSdsKeyCompare
(
privdata
,
o1
->
ptr
,
o2
->
ptr
);
}
}
u
nsigned
in
t
dictObjHash
(
const
void
*
key
)
{
u
int64_
t
dictObjHash
(
const
void
*
key
)
{
const
robj
*
o
=
key
;
const
robj
*
o
=
key
;
return
dictGenHashFunction
(
o
->
ptr
,
sdslen
((
sds
)
o
->
ptr
));
return
dictGenHashFunction
(
o
->
ptr
,
sdslen
((
sds
)
o
->
ptr
));
}
}
u
nsigned
in
t
dictSdsHash
(
const
void
*
key
)
{
u
int64_
t
dictSdsHash
(
const
void
*
key
)
{
return
dictGenHashFunction
((
unsigned
char
*
)
key
,
sdslen
((
char
*
)
key
));
return
dictGenHashFunction
((
unsigned
char
*
)
key
,
sdslen
((
char
*
)
key
));
}
}
u
nsigned
in
t
dictSdsCaseHash
(
const
void
*
key
)
{
u
int64_
t
dictSdsCaseHash
(
const
void
*
key
)
{
return
dictGenCaseHashFunction
((
unsigned
char
*
)
key
,
sdslen
((
char
*
)
key
));
return
dictGenCaseHashFunction
((
unsigned
char
*
)
key
,
sdslen
((
char
*
)
key
));
}
}
...
@@ -513,7 +513,7 @@ int dictEncObjKeyCompare(void *privdata, const void *key1,
...
@@ -513,7 +513,7 @@ int dictEncObjKeyCompare(void *privdata, const void *key1,
return
cmp
;
return
cmp
;
}
}
u
nsigned
in
t
dictEncObjHash
(
const
void
*
key
)
{
u
int64_
t
dictEncObjHash
(
const
void
*
key
)
{
robj
*
o
=
(
robj
*
)
key
;
robj
*
o
=
(
robj
*
)
key
;
if
(
sdsEncodedObject
(
o
))
{
if
(
sdsEncodedObject
(
o
))
{
...
@@ -526,7 +526,7 @@ unsigned int dictEncObjHash(const void *key) {
...
@@ -526,7 +526,7 @@ unsigned int dictEncObjHash(const void *key) {
len
=
ll2string
(
buf
,
32
,(
long
)
o
->
ptr
);
len
=
ll2string
(
buf
,
32
,(
long
)
o
->
ptr
);
return
dictGenHashFunction
((
unsigned
char
*
)
buf
,
len
);
return
dictGenHashFunction
((
unsigned
char
*
)
buf
,
len
);
}
else
{
}
else
{
u
nsigned
in
t
hash
;
u
int64_
t
hash
;
o
=
getDecodedObject
(
o
);
o
=
getDecodedObject
(
o
);
hash
=
dictGenHashFunction
(
o
->
ptr
,
sdslen
((
sds
)
o
->
ptr
));
hash
=
dictGenHashFunction
(
o
->
ptr
,
sdslen
((
sds
)
o
->
ptr
));
...
@@ -3639,7 +3639,9 @@ int main(int argc, char **argv) {
...
@@ -3639,7 +3639,9 @@ int main(int argc, char **argv) {
zmalloc_set_oom_handler
(
redisOutOfMemoryHandler
);
zmalloc_set_oom_handler
(
redisOutOfMemoryHandler
);
srand
(
time
(
NULL
)
^
getpid
());
srand
(
time
(
NULL
)
^
getpid
());
gettimeofday
(
&
tv
,
NULL
);
gettimeofday
(
&
tv
,
NULL
);
dictSetHashFunctionSeed
(
tv
.
tv_sec
^
tv
.
tv_usec
^
getpid
());
char
hashseed
[
16
];
getRandomHexChars
(
hashseed
,
sizeof
(
hashseed
));
dictSetHashFunctionSeed
((
uint8_t
*
)
hashseed
);
server
.
sentinel_mode
=
checkForSentinelMode
(
argc
,
argv
);
server
.
sentinel_mode
=
checkForSentinelMode
(
argc
,
argv
);
initServerConfig
();
initServerConfig
();
moduleInitModulesSystem
();
moduleInitModulesSystem
();
...
...
src/server.h
View file @
71a8df6a
...
@@ -435,7 +435,7 @@ typedef long long mstime_t; /* millisecond time type. */
...
@@ -435,7 +435,7 @@ typedef long long mstime_t; /* millisecond time type. */
/* We can print the stacktrace, so our assert is defined this way: */
/* We can print the stacktrace, so our assert is defined this way: */
#define serverAssertWithInfo(_c,_o,_e) ((_e)?(void)0 : (_serverAssertWithInfo(_c,_o,#_e,__FILE__,__LINE__),_exit(1)))
#define serverAssertWithInfo(_c,_o,_e) ((_e)?(void)0 : (_serverAssertWithInfo(_c,_o,#_e,__FILE__,__LINE__),_exit(1)))
#define serverAssert(_e) ((_e)?(void)0 : (_serverAssert(#_e,__FILE__,__LINE__),_exit(1)))
#define serverAssert(_e) ((_e)?(void)0 : (_serverAssert(#_e,__FILE__,__LINE__),_exit(1)))
#define serverPanic(
_e
) _serverPanic(
#_e,
__FILE__,__LINE__),_exit(1)
#define serverPanic(
...
) _serverPanic(__FILE__,__LINE__
,__VA_ARGS__
),_exit(1)
/*-----------------------------------------------------------------------------
/*-----------------------------------------------------------------------------
* Data types
* Data types
...
@@ -483,7 +483,7 @@ typedef void *(*moduleTypeLoadFunc)(struct RedisModuleIO *io, int encver);
...
@@ -483,7 +483,7 @@ typedef void *(*moduleTypeLoadFunc)(struct RedisModuleIO *io, int encver);
typedef
void
(
*
moduleTypeSaveFunc
)(
struct
RedisModuleIO
*
io
,
void
*
value
);
typedef
void
(
*
moduleTypeSaveFunc
)(
struct
RedisModuleIO
*
io
,
void
*
value
);
typedef
void
(
*
moduleTypeRewriteFunc
)(
struct
RedisModuleIO
*
io
,
struct
redisObject
*
key
,
void
*
value
);
typedef
void
(
*
moduleTypeRewriteFunc
)(
struct
RedisModuleIO
*
io
,
struct
redisObject
*
key
,
void
*
value
);
typedef
void
(
*
moduleTypeDigestFunc
)(
struct
RedisModuleDigest
*
digest
,
void
*
value
);
typedef
void
(
*
moduleTypeDigestFunc
)(
struct
RedisModuleDigest
*
digest
,
void
*
value
);
typedef
size_t
(
*
moduleTypeMemUsageFunc
)(
void
*
value
);
typedef
size_t
(
*
moduleTypeMemUsageFunc
)(
const
void
*
value
);
typedef
void
(
*
moduleTypeFreeFunc
)(
void
*
value
);
typedef
void
(
*
moduleTypeFreeFunc
)(
void
*
value
);
/* The module type, which is referenced in each value of a given type, defines
/* The module type, which is referenced in each value of a given type, defines
...
@@ -1765,7 +1765,7 @@ unsigned long LFUGetTimeInMinutes(void);
...
@@ -1765,7 +1765,7 @@ unsigned long LFUGetTimeInMinutes(void);
uint8_t
LFULogIncr
(
uint8_t
value
);
uint8_t
LFULogIncr
(
uint8_t
value
);
/* Keys hashing / comparison functions for dict.c hash tables. */
/* Keys hashing / comparison functions for dict.c hash tables. */
u
nsigned
in
t
dictSdsHash
(
const
void
*
key
);
u
int64_
t
dictSdsHash
(
const
void
*
key
);
int
dictSdsKeyCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
int
dictSdsKeyCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
void
dictSdsDestructor
(
void
*
privdata
,
void
*
val
);
void
dictSdsDestructor
(
void
*
privdata
,
void
*
val
);
...
@@ -1960,7 +1960,7 @@ void *realloc(void *ptr, size_t size) __attribute__ ((deprecated));
...
@@ -1960,7 +1960,7 @@ void *realloc(void *ptr, size_t size) __attribute__ ((deprecated));
/* Debugging stuff */
/* Debugging stuff */
void
_serverAssertWithInfo
(
const
client
*
c
,
const
robj
*
o
,
const
char
*
estr
,
const
char
*
file
,
int
line
);
void
_serverAssertWithInfo
(
const
client
*
c
,
const
robj
*
o
,
const
char
*
estr
,
const
char
*
file
,
int
line
);
void
_serverAssert
(
const
char
*
estr
,
const
char
*
file
,
int
line
);
void
_serverAssert
(
const
char
*
estr
,
const
char
*
file
,
int
line
);
void
_serverPanic
(
const
char
*
msg
,
const
char
*
file
,
int
line
);
void
_serverPanic
(
const
char
*
file
,
int
line
,
const
char
*
msg
,
...
);
void
bugReportStart
(
void
);
void
bugReportStart
(
void
);
void
serverLogObjectDebugInfo
(
const
robj
*
o
);
void
serverLogObjectDebugInfo
(
const
robj
*
o
);
void
sigsegvHandler
(
int
sig
,
siginfo_t
*
info
,
void
*
secret
);
void
sigsegvHandler
(
int
sig
,
siginfo_t
*
info
,
void
*
secret
);
...
...
src/siphash.c
0 → 100644
View file @
71a8df6a
/*
SipHash reference C implementation
Copyright (c) 2012-2016 Jean-Philippe Aumasson
<jeanphilippe.aumasson@gmail.com>
Copyright (c) 2012-2014 Daniel J. Bernstein <djb@cr.yp.to>
Copyright (c) 2017 Salvatore Sanfilippo <antirez@gmail.com>
To the extent possible under law, the author(s) have dedicated all copyright
and related and neighboring rights to this software to the public domain
worldwide. This software is distributed without any warranty.
You should have received a copy of the CC0 Public Domain Dedication along
with this software. If not, see
<http://creativecommons.org/publicdomain/zero/1.0/>.
----------------------------------------------------------------------------
This version was modified by Salvatore Sanfilippo <antirez@gmail.com>
in the following ways:
1. We use SipHash 1-2. This is not believed to be as strong as the
suggested 2-4 variant, but AFAIK there are not trivial attacks
against this reduced-rounds version, and it runs at the same speed
as Murmurhash2 that we used previously, why the 2-4 variant slowed
down Redis by a 4% figure more or less.
2. Hard-code rounds in the hope the compiler can optimize it more
in this raw from. Anyway we always want the standard 2-4 variant.
3. Modify the prototype and implementation so that the function directly
returns an uint64_t value, the hash itself, instead of receiving an
output buffer. This also means that the output size is set to 8 bytes
and the 16 bytes output code handling was removed.
4. Provide a case insensitive variant to be used when hashing strings that
must be considered identical by the hash table regardless of the case.
If we don't have directly a case insensitive hash function, we need to
perform a text transformation in some temporary buffer, which is costly.
5. Remove debugging code.
6. Modified the original test.c file to be a stand-alone function testing
the function in the new form (returing an uint64_t) using just the
relevant test vector.
*/
#include <assert.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
/* Fast tolower() alike function that does not care about locale
* but just returns a-z insetad of A-Z. */
int
siptlw
(
int
c
)
{
if
(
c
>=
'A'
&&
c
<=
'Z'
)
{
return
c
+
(
'a'
-
'A'
);
}
else
{
return
c
;
}
}
/* Test of the CPU is Little Endian and supports not aligned accesses.
* Two interesting conditions to speedup the function that happen to be
* in most of x86 servers. */
#if defined(__X86_64__) || defined(__x86_64__) || defined (__i386__)
#define UNALIGNED_LE_CPU
#endif
#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
#define U32TO8_LE(p, v) \
(p)[0] = (uint8_t)((v)); \
(p)[1] = (uint8_t)((v) >> 8); \
(p)[2] = (uint8_t)((v) >> 16); \
(p)[3] = (uint8_t)((v) >> 24);
#define U64TO8_LE(p, v) \
U32TO8_LE((p), (uint32_t)((v))); \
U32TO8_LE((p) + 4, (uint32_t)((v) >> 32));
#ifdef UNALIGNED_LE_CPU
#define U8TO64_LE(p) (*((uint64_t*)(p)))
#else
#define U8TO64_LE(p) \
(((uint64_t)((p)[0])) | ((uint64_t)((p)[1]) << 8) | \
((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24) | \
((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) | \
((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56))
#endif
#define U8TO64_LE_NOCASE(p) \
(((uint64_t)(siptlw((p)[0]))) | \
((uint64_t)(siptlw((p)[1])) << 8) | \
((uint64_t)(siptlw((p)[2])) << 16) | \
((uint64_t)(siptlw((p)[3])) << 24) | \
((uint64_t)(siptlw((p)[4])) << 32) | \
((uint64_t)(siptlw((p)[5])) << 40) | \
((uint64_t)(siptlw((p)[6])) << 48) | \
((uint64_t)(siptlw((p)[7])) << 56))
#define SIPROUND \
do { \
v0 += v1; \
v1 = ROTL(v1, 13); \
v1 ^= v0; \
v0 = ROTL(v0, 32); \
v2 += v3; \
v3 = ROTL(v3, 16); \
v3 ^= v2; \
v0 += v3; \
v3 = ROTL(v3, 21); \
v3 ^= v0; \
v2 += v1; \
v1 = ROTL(v1, 17); \
v1 ^= v2; \
v2 = ROTL(v2, 32); \
} while (0)
uint64_t
siphash
(
const
uint8_t
*
in
,
const
size_t
inlen
,
const
uint8_t
*
k
)
{
#ifndef UNALIGNED_LE_CPU
uint64_t
hash
;
uint8_t
*
out
=
(
uint8_t
*
)
&
hash
;
#endif
uint64_t
v0
=
0x736f6d6570736575ULL
;
uint64_t
v1
=
0x646f72616e646f6dULL
;
uint64_t
v2
=
0x6c7967656e657261ULL
;
uint64_t
v3
=
0x7465646279746573ULL
;
uint64_t
k0
=
U8TO64_LE
(
k
);
uint64_t
k1
=
U8TO64_LE
(
k
+
8
);
uint64_t
m
;
const
uint8_t
*
end
=
in
+
inlen
-
(
inlen
%
sizeof
(
uint64_t
));
const
int
left
=
inlen
&
7
;
uint64_t
b
=
((
uint64_t
)
inlen
)
<<
56
;
v3
^=
k1
;
v2
^=
k0
;
v1
^=
k1
;
v0
^=
k0
;
for
(;
in
!=
end
;
in
+=
8
)
{
m
=
U8TO64_LE
(
in
);
v3
^=
m
;
SIPROUND
;
v0
^=
m
;
}
switch
(
left
)
{
case
7
:
b
|=
((
uint64_t
)
in
[
6
])
<<
48
;
case
6
:
b
|=
((
uint64_t
)
in
[
5
])
<<
40
;
case
5
:
b
|=
((
uint64_t
)
in
[
4
])
<<
32
;
case
4
:
b
|=
((
uint64_t
)
in
[
3
])
<<
24
;
case
3
:
b
|=
((
uint64_t
)
in
[
2
])
<<
16
;
case
2
:
b
|=
((
uint64_t
)
in
[
1
])
<<
8
;
case
1
:
b
|=
((
uint64_t
)
in
[
0
]);
break
;
case
0
:
break
;
}
v3
^=
b
;
SIPROUND
;
v0
^=
b
;
v2
^=
0xff
;
SIPROUND
;
SIPROUND
;
b
=
v0
^
v1
^
v2
^
v3
;
#ifndef UNALIGNED_LE_CPU
U64TO8_LE
(
out
,
b
);
return
hash
;
#else
return
b
;
#endif
}
uint64_t
siphash_nocase
(
const
uint8_t
*
in
,
const
size_t
inlen
,
const
uint8_t
*
k
)
{
#ifndef UNALIGNED_LE_CPU
uint64_t
hash
;
uint8_t
*
out
=
(
uint8_t
*
)
&
hash
;
#endif
uint64_t
v0
=
0x736f6d6570736575ULL
;
uint64_t
v1
=
0x646f72616e646f6dULL
;
uint64_t
v2
=
0x6c7967656e657261ULL
;
uint64_t
v3
=
0x7465646279746573ULL
;
uint64_t
k0
=
U8TO64_LE
(
k
);
uint64_t
k1
=
U8TO64_LE
(
k
+
8
);
uint64_t
m
;
const
uint8_t
*
end
=
in
+
inlen
-
(
inlen
%
sizeof
(
uint64_t
));
const
int
left
=
inlen
&
7
;
uint64_t
b
=
((
uint64_t
)
inlen
)
<<
56
;
v3
^=
k1
;
v2
^=
k0
;
v1
^=
k1
;
v0
^=
k0
;
for
(;
in
!=
end
;
in
+=
8
)
{
m
=
U8TO64_LE_NOCASE
(
in
);
v3
^=
m
;
SIPROUND
;
v0
^=
m
;
}
switch
(
left
)
{
case
7
:
b
|=
((
uint64_t
)
siptlw
(
in
[
6
]))
<<
48
;
case
6
:
b
|=
((
uint64_t
)
siptlw
(
in
[
5
]))
<<
40
;
case
5
:
b
|=
((
uint64_t
)
siptlw
(
in
[
4
]))
<<
32
;
case
4
:
b
|=
((
uint64_t
)
siptlw
(
in
[
3
]))
<<
24
;
case
3
:
b
|=
((
uint64_t
)
siptlw
(
in
[
2
]))
<<
16
;
case
2
:
b
|=
((
uint64_t
)
siptlw
(
in
[
1
]))
<<
8
;
case
1
:
b
|=
((
uint64_t
)
siptlw
(
in
[
0
]));
break
;
case
0
:
break
;
}
v3
^=
b
;
SIPROUND
;
v0
^=
b
;
v2
^=
0xff
;
SIPROUND
;
SIPROUND
;
b
=
v0
^
v1
^
v2
^
v3
;
#ifndef UNALIGNED_LE_CPU
U64TO8_LE
(
out
,
b
);
return
hash
;
#else
return
b
;
#endif
}
/* --------------------------------- TEST ------------------------------------ */
#ifdef SIPHASH_TEST
const
uint8_t
vectors_sip64
[
64
][
8
]
=
{
{
0x31
,
0x0e
,
0x0e
,
0xdd
,
0x47
,
0xdb
,
0x6f
,
0x72
,
},
{
0xfd
,
0x67
,
0xdc
,
0x93
,
0xc5
,
0x39
,
0xf8
,
0x74
,
},
{
0x5a
,
0x4f
,
0xa9
,
0xd9
,
0x09
,
0x80
,
0x6c
,
0x0d
,
},
{
0x2d
,
0x7e
,
0xfb
,
0xd7
,
0x96
,
0x66
,
0x67
,
0x85
,
},
{
0xb7
,
0x87
,
0x71
,
0x27
,
0xe0
,
0x94
,
0x27
,
0xcf
,
},
{
0x8d
,
0xa6
,
0x99
,
0xcd
,
0x64
,
0x55
,
0x76
,
0x18
,
},
{
0xce
,
0xe3
,
0xfe
,
0x58
,
0x6e
,
0x46
,
0xc9
,
0xcb
,
},
{
0x37
,
0xd1
,
0x01
,
0x8b
,
0xf5
,
0x00
,
0x02
,
0xab
,
},
{
0x62
,
0x24
,
0x93
,
0x9a
,
0x79
,
0xf5
,
0xf5
,
0x93
,
},
{
0xb0
,
0xe4
,
0xa9
,
0x0b
,
0xdf
,
0x82
,
0x00
,
0x9e
,
},
{
0xf3
,
0xb9
,
0xdd
,
0x94
,
0xc5
,
0xbb
,
0x5d
,
0x7a
,
},
{
0xa7
,
0xad
,
0x6b
,
0x22
,
0x46
,
0x2f
,
0xb3
,
0xf4
,
},
{
0xfb
,
0xe5
,
0x0e
,
0x86
,
0xbc
,
0x8f
,
0x1e
,
0x75
,
},
{
0x90
,
0x3d
,
0x84
,
0xc0
,
0x27
,
0x56
,
0xea
,
0x14
,
},
{
0xee
,
0xf2
,
0x7a
,
0x8e
,
0x90
,
0xca
,
0x23
,
0xf7
,
},
{
0xe5
,
0x45
,
0xbe
,
0x49
,
0x61
,
0xca
,
0x29
,
0xa1
,
},
{
0xdb
,
0x9b
,
0xc2
,
0x57
,
0x7f
,
0xcc
,
0x2a
,
0x3f
,
},
{
0x94
,
0x47
,
0xbe
,
0x2c
,
0xf5
,
0xe9
,
0x9a
,
0x69
,
},
{
0x9c
,
0xd3
,
0x8d
,
0x96
,
0xf0
,
0xb3
,
0xc1
,
0x4b
,
},
{
0xbd
,
0x61
,
0x79
,
0xa7
,
0x1d
,
0xc9
,
0x6d
,
0xbb
,
},
{
0x98
,
0xee
,
0xa2
,
0x1a
,
0xf2
,
0x5c
,
0xd6
,
0xbe
,
},
{
0xc7
,
0x67
,
0x3b
,
0x2e
,
0xb0
,
0xcb
,
0xf2
,
0xd0
,
},
{
0x88
,
0x3e
,
0xa3
,
0xe3
,
0x95
,
0x67
,
0x53
,
0x93
,
},
{
0xc8
,
0xce
,
0x5c
,
0xcd
,
0x8c
,
0x03
,
0x0c
,
0xa8
,
},
{
0x94
,
0xaf
,
0x49
,
0xf6
,
0xc6
,
0x50
,
0xad
,
0xb8
,
},
{
0xea
,
0xb8
,
0x85
,
0x8a
,
0xde
,
0x92
,
0xe1
,
0xbc
,
},
{
0xf3
,
0x15
,
0xbb
,
0x5b
,
0xb8
,
0x35
,
0xd8
,
0x17
,
},
{
0xad
,
0xcf
,
0x6b
,
0x07
,
0x63
,
0x61
,
0x2e
,
0x2f
,
},
{
0xa5
,
0xc9
,
0x1d
,
0xa7
,
0xac
,
0xaa
,
0x4d
,
0xde
,
},
{
0x71
,
0x65
,
0x95
,
0x87
,
0x66
,
0x50
,
0xa2
,
0xa6
,
},
{
0x28
,
0xef
,
0x49
,
0x5c
,
0x53
,
0xa3
,
0x87
,
0xad
,
},
{
0x42
,
0xc3
,
0x41
,
0xd8
,
0xfa
,
0x92
,
0xd8
,
0x32
,
},
{
0xce
,
0x7c
,
0xf2
,
0x72
,
0x2f
,
0x51
,
0x27
,
0x71
,
},
{
0xe3
,
0x78
,
0x59
,
0xf9
,
0x46
,
0x23
,
0xf3
,
0xa7
,
},
{
0x38
,
0x12
,
0x05
,
0xbb
,
0x1a
,
0xb0
,
0xe0
,
0x12
,
},
{
0xae
,
0x97
,
0xa1
,
0x0f
,
0xd4
,
0x34
,
0xe0
,
0x15
,
},
{
0xb4
,
0xa3
,
0x15
,
0x08
,
0xbe
,
0xff
,
0x4d
,
0x31
,
},
{
0x81
,
0x39
,
0x62
,
0x29
,
0xf0
,
0x90
,
0x79
,
0x02
,
},
{
0x4d
,
0x0c
,
0xf4
,
0x9e
,
0xe5
,
0xd4
,
0xdc
,
0xca
,
},
{
0x5c
,
0x73
,
0x33
,
0x6a
,
0x76
,
0xd8
,
0xbf
,
0x9a
,
},
{
0xd0
,
0xa7
,
0x04
,
0x53
,
0x6b
,
0xa9
,
0x3e
,
0x0e
,
},
{
0x92
,
0x59
,
0x58
,
0xfc
,
0xd6
,
0x42
,
0x0c
,
0xad
,
},
{
0xa9
,
0x15
,
0xc2
,
0x9b
,
0xc8
,
0x06
,
0x73
,
0x18
,
},
{
0x95
,
0x2b
,
0x79
,
0xf3
,
0xbc
,
0x0a
,
0xa6
,
0xd4
,
},
{
0xf2
,
0x1d
,
0xf2
,
0xe4
,
0x1d
,
0x45
,
0x35
,
0xf9
,
},
{
0x87
,
0x57
,
0x75
,
0x19
,
0x04
,
0x8f
,
0x53
,
0xa9
,
},
{
0x10
,
0xa5
,
0x6c
,
0xf5
,
0xdf
,
0xcd
,
0x9a
,
0xdb
,
},
{
0xeb
,
0x75
,
0x09
,
0x5c
,
0xcd
,
0x98
,
0x6c
,
0xd0
,
},
{
0x51
,
0xa9
,
0xcb
,
0x9e
,
0xcb
,
0xa3
,
0x12
,
0xe6
,
},
{
0x96
,
0xaf
,
0xad
,
0xfc
,
0x2c
,
0xe6
,
0x66
,
0xc7
,
},
{
0x72
,
0xfe
,
0x52
,
0x97
,
0x5a
,
0x43
,
0x64
,
0xee
,
},
{
0x5a
,
0x16
,
0x45
,
0xb2
,
0x76
,
0xd5
,
0x92
,
0xa1
,
},
{
0xb2
,
0x74
,
0xcb
,
0x8e
,
0xbf
,
0x87
,
0x87
,
0x0a
,
},
{
0x6f
,
0x9b
,
0xb4
,
0x20
,
0x3d
,
0xe7
,
0xb3
,
0x81
,
},
{
0xea
,
0xec
,
0xb2
,
0xa3
,
0x0b
,
0x22
,
0xa8
,
0x7f
,
},
{
0x99
,
0x24
,
0xa4
,
0x3c
,
0xc1
,
0x31
,
0x57
,
0x24
,
},
{
0xbd
,
0x83
,
0x8d
,
0x3a
,
0xaf
,
0xbf
,
0x8d
,
0xb7
,
},
{
0x0b
,
0x1a
,
0x2a
,
0x32
,
0x65
,
0xd5
,
0x1a
,
0xea
,
},
{
0x13
,
0x50
,
0x79
,
0xa3
,
0x23
,
0x1c
,
0xe6
,
0x60
,
},
{
0x93
,
0x2b
,
0x28
,
0x46
,
0xe4
,
0xd7
,
0x06
,
0x66
,
},
{
0xe1
,
0x91
,
0x5f
,
0x5c
,
0xb1
,
0xec
,
0xa4
,
0x6c
,
},
{
0xf3
,
0x25
,
0x96
,
0x5c
,
0xa1
,
0x6d
,
0x62
,
0x9f
,
},
{
0x57
,
0x5f
,
0xf2
,
0x8e
,
0x60
,
0x38
,
0x1b
,
0xe5
,
},
{
0x72
,
0x45
,
0x06
,
0xeb
,
0x4c
,
0x32
,
0x8a
,
0x95
,
},
};
/* Test siphash using a test vector. Returns 0 if the function passed
* all the tests, otherwise 1 is returned.
*
* IMPORTANT: The test vector is for SipHash 2-4. Before running
* the test revert back the siphash() function to 2-4 rounds since
* now it uses 1-2 rounds. */
int
siphash_test
(
void
)
{
uint8_t
in
[
64
],
k
[
16
];
int
i
;
int
fails
=
0
;
for
(
i
=
0
;
i
<
16
;
++
i
)
k
[
i
]
=
i
;
for
(
i
=
0
;
i
<
64
;
++
i
)
{
in
[
i
]
=
i
;
uint64_t
hash
=
siphash
(
in
,
i
,
k
);
const
uint8_t
*
v
=
NULL
;
v
=
(
uint8_t
*
)
vectors_sip64
;
if
(
memcmp
(
&
hash
,
v
+
(
i
*
8
),
8
))
{
/* printf("fail for %d bytes\n", i); */
fails
++
;
}
}
/* Run a few basic tests with the case insensitive version. */
uint64_t
h1
,
h2
;
h1
=
siphash
((
uint8_t
*
)
"hello world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
h2
=
siphash_nocase
((
uint8_t
*
)
"hello world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
if
(
h1
!=
h2
)
fails
++
;
h1
=
siphash
((
uint8_t
*
)
"hello world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
h2
=
siphash_nocase
((
uint8_t
*
)
"HELLO world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
if
(
h1
!=
h2
)
fails
++
;
h1
=
siphash
((
uint8_t
*
)
"HELLO world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
h2
=
siphash_nocase
((
uint8_t
*
)
"HELLO world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
if
(
h1
==
h2
)
fails
++
;
if
(
!
fails
)
return
0
;
return
1
;
}
int
main
(
void
)
{
if
(
siphash_test
()
==
0
)
{
printf
(
"SipHash test: OK
\n
"
);
return
0
;
}
else
{
printf
(
"SipHash test: FAILED
\n
"
);
return
1
;
}
}
#endif
src/t_zset.c
View file @
71a8df6a
...
@@ -1521,7 +1521,7 @@ void zaddGenericCommand(client *c, int flags) {
...
@@ -1521,7 +1521,7 @@ void zaddGenericCommand(client *c, int flags) {
/* After the options, we expect to have an even number of args, since
/* After the options, we expect to have an even number of args, since
* we expect any number of score-element pairs. */
* we expect any number of score-element pairs. */
elements
=
c
->
argc
-
scoreidx
;
elements
=
c
->
argc
-
scoreidx
;
if
(
elements
%
2
)
{
if
(
elements
%
2
||
!
elements
)
{
addReply
(
c
,
shared
.
syntaxerr
);
addReply
(
c
,
shared
.
syntaxerr
);
return
;
return
;
}
}
...
@@ -2110,7 +2110,7 @@ inline static void zunionInterAggregate(double *target, double val, int aggregat
...
@@ -2110,7 +2110,7 @@ inline static void zunionInterAggregate(double *target, double val, int aggregat
}
}
}
}
u
nsigned
in
t
dictSdsHash
(
const
void
*
key
);
u
int64_
t
dictSdsHash
(
const
void
*
key
);
int
dictSdsKeyCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
int
dictSdsKeyCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
dictType
setAccumulatorDictType
=
{
dictType
setAccumulatorDictType
=
{
...
...
src/ziplist.c
View file @
71a8df6a
...
@@ -9,78 +9,149 @@
...
@@ -9,78 +9,149 @@
* ----------------------------------------------------------------------------
* ----------------------------------------------------------------------------
*
*
* ZIPLIST OVERALL LAYOUT
* ZIPLIST OVERALL LAYOUT
* ======================
*
*
* The general layout of the ziplist is as follows:
* The general layout of the ziplist is as follows:
*
*
* <zlbytes> <zltail> <zllen> <entry> <entry> ... <entry> <zlend>
* <zlbytes> <zltail> <zllen> <entry> <entry> ... <entry> <zlend>
*
*
*
A
ll fields are stored in little endian.
*
NOTE: a
ll fields are stored in little endian
, if not specified otherwise
.
*
*
* <uint32_t zlbytes> is an unsigned integer to hold the number of bytes that
* <uint32_t zlbytes> is an unsigned integer to hold the number of bytes that
* the ziplist occupies. This value needs to be stored to be able to resize the
* the ziplist occupies, including the four bytes of the zlbytes field itself.
* entire structure without the need to traverse it first.
* This value needs to be stored to be able to resize the entire structure
* without the need to traverse it first.
*
*
* <uint32_t zltail> is the offset to the last entry in the list. This allows
* <uint32_t zltail> is the offset to the last entry in the list. This allows
* a pop operation on the far side of the list without the need for full
* a pop operation on the far side of the list without the need for full
* traversal.
* traversal.
*
*
* <uint16_t zllen> is the number of entries. When th
is value is larger
* <uint16_t zllen> is the number of entries. When th
ere are more than
*
than
2^16-2
, we need to traverse the entir
e
l
ist to
know how many items it
* 2^16-2
entires, this valu
e is
se
t to
2^16-1 and we need to traverse the
* holds.
*
entire list to know how many items it
holds.
*
*
* <uint8_t zlend> is a single byte special value, equal to 255, which
* <uint8_t zlend> is a special entry representing the end of the ziplist.
* indicates the end of the list.
* Is encoded as a single byte equal to 255. No other normal entry starts
* with a byte set to the value of 255.
*
*
* ZIPLIST ENTRIES
* ZIPLIST ENTRIES
* ===============
*
*
* Every entry in the ziplist is prefixed by
a header
that contains two pieces
* Every entry in the ziplist is prefixed by
metadata
that contains two pieces
* of information. First, the length of the previous entry is stored to be
* of information. First, the length of the previous entry is stored to be
* able to traverse the list from back to front. Second, the encoding with an
* able to traverse the list from back to front. Second, the entry encoding is
* optional string length of the entry itself is stored.
* provided. It represents the entry type, integer or string, and in the case
* of strings it also represents the length of the string payload.
* So a complete entry is stored like this:
*
*
* The length of the previous entry is encoded in the following way:
* <prevlen> <encoding> <entry-data>
* If this length is smaller than 254 bytes, it will only consume a single
* byte that takes the length as value. When the length is greater than or
* equal to 254, it will consume 5 bytes. The first byte is set to 254 to
* indicate a larger value is following. The remaining 4 bytes take the
* length of the previous entry as value.
*
*
* The other header field of the entry itself depends on the contents of the
* Sometimes the encoding represents the entry itself, like for small integers
* entry. When the entry is a string, the first 2 bits of this header will hold
* as we'll see later. In such a case the <entry-data> part is missing, and we
* the type of encoding used to store the length of the string, followed by the
* could have just:
* actual length of the string. When the entry is an integer the first 2 bits
*
* are both set to 1. The following 2 bits are used to specify what kind of
* <prevlen> <encoding>
* integer will be stored after this header. An overview of the different
*
* types and encodings is as follows:
* The length of the previous entry, <prevlen>, is encoded in the following way:
* If this length is smaller than 255 bytes, it will only consume a single
* byte representing the length as an unsinged 8 bit integer. When the length
* is greater than or equal to 255, it will consume 5 bytes. The first byte is
* set to 255 (FF) to indicate a larger value is following. The remaining 4
* bytes take the length of the previous entry as value.
*
* So practically an entry is encoded in the following way:
*
* <prevlen from 0 to 254> <encoding> <entry>
*
* Or alternatively if the previous entry length is greater than 254 bytes
* the following encoding is used:
*
* 0xFF <4 bytes unsigned little endian prevlen> <encoding> <entry>
*
* The encoding field of the entry depends on the content of the
* entry. When the entry is a string, the first 2 bits of the encoding first
* byte will hold the type of encoding used to store the length of the string,
* followed by the actual length of the string. When the entry is an integer
* the first 2 bits are both set to 1. The following 2 bits are used to specify
* what kind of integer will be stored after this header. An overview of the
* different types and encodings is as follows. The first byte is always enough
* to determine the kind of entry.
*
*
* |00pppppp| - 1 byte
* |00pppppp| - 1 byte
* String value with length less than or equal to 63 bytes (6 bits).
* String value with length less than or equal to 63 bytes (6 bits).
* "pppppp" represents the unsigned 6 bit length.
* |01pppppp|qqqqqqqq| - 2 bytes
* |01pppppp|qqqqqqqq| - 2 bytes
* String value with length less than or equal to 16383 bytes (14 bits).
* String value with length less than or equal to 16383 bytes (14 bits).
* |10______|qqqqqqqq|rrrrrrrr|ssssssss|tttttttt| - 5 bytes
* IMPORTANT: The 14 bit number is stored in big endian.
* |10000000|qqqqqqqq|rrrrrrrr|ssssssss|tttttttt| - 5 bytes
* String value with length greater than or equal to 16384 bytes.
* String value with length greater than or equal to 16384 bytes.
* |11000000| - 1 byte
* Only the 4 bytes following the first byte represents the length
* up to 32^2-1. The 6 lower bits of the first byte are not used and
* are set to zero.
* IMPORTANT: The 32 bit number is stored in big endian.
* |11000000| - 3 bytes
* Integer encoded as int16_t (2 bytes).
* Integer encoded as int16_t (2 bytes).
* |11010000| -
1
byte
* |11010000| -
5
byte
s
* Integer encoded as int32_t (4 bytes).
* Integer encoded as int32_t (4 bytes).
* |11100000| -
1
byte
* |11100000| -
9
byte
s
* Integer encoded as int64_t (8 bytes).
* Integer encoded as int64_t (8 bytes).
* |11110000| -
1
byte
* |11110000| -
4
byte
s
* Integer encoded as 24 bit signed (3 bytes).
* Integer encoded as 24 bit signed (3 bytes).
* |11111110| -
1
byte
* |11111110| -
2
byte
s
* Integer encoded as 8 bit signed (1 byte).
* Integer encoded as 8 bit signed (1 byte).
* |1111xxxx| - (with xxxx between 0000 and 1101) immediate 4 bit integer.
* |1111xxxx| - (with xxxx between 0000 and 1101) immediate 4 bit integer.
* Unsigned integer from 0 to 12. The encoded value is actually from
* Unsigned integer from 0 to 12. The encoded value is actually from
* 1 to 13 because 0000 and 1111 can not be used, so 1 should be
* 1 to 13 because 0000 and 1111 can not be used, so 1 should be
* subtracted from the encoded 4 bit value to obtain the right value.
* subtracted from the encoded 4 bit value to obtain the right value.
* |11111111| - End of ziplist.
* |11111111| - End of ziplist special entry.
*
* Like for the ziplist header, all the integers are represented in little
* endian byte order, even when this code is compiled in big endian systems.
*
* EXAMPLES OF ACTUAL ZIPLISTS
* ===========================
*
* The following is a ziplist containing the two elements representing
* the strings "2" and "5". It is composed of 15 bytes, that we visually
* split into sections:
*
* [0f 00 00 00] [0c 00 00 00] [02 00] [00 f3] [02 f6] [ff]
* | | | | | |
* zlbytes zltail entries "2" "5" end
*
*
* All the integers are represented in little endian byte order.
* The first 4 bytes represent the number 15, that is the number of bytes
* the whole ziplist is composed of. The second 4 bytes are the offset
* at which the last ziplist entry is found, that is 12, in fact the
* last entry, that is "5", is at offset 12 inside the ziplist.
* The next 16 bit integer represents the number of elements inside the
* ziplist, its value is 2 since there are just two elements inside.
* Finally "00 f3" is the first entry representing the number 2. It is
* composed of the previous entry length, which is zero because this is
* our first entry, and the byte F3 which corresponds to the encoding
* |1111xxxx| with xxxx between 0001 and 1101. We need to remove the "F"
* higher order bits 1111, and subtract 1 from the "3", so the entry value
* is "2". The next entry has a prevlen of 02, since the first entry is
* composed of exactly two bytes. The entry itself, F6, is encoded exactly
* like the first entry, and 6-1 = 5, so the value of the entry is 5.
* Finally the special entry FF signals the end of the ziplist.
*
* Adding another element to the above string with the value "Hello World"
* allows us to show how the ziplist encodes small strings. We'll just show
* the hex dump of the entry itself. Imagine the bytes as following the
* entry that stores "5" in the ziplist above:
*
* [02] [0b] [48 65 6c 6c 6f 20 57 6f 72 6c 64]
*
* The first byte, 02, is the length of the previous entry. The next
* byte represents the encoding in the pattern |00pppppp| that means
* that the entry is a string of length <pppppp>, so 0B means that
* an 11 bytes string follows. From the third byte (48) to the last (64)
* there are just the ASCII characters for "Hello World".
*
*
* ----------------------------------------------------------------------------
* ----------------------------------------------------------------------------
*
*
* Copyright (c) 2009-2012, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2009-2012, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2009-201
2
, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2009-201
7
, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* Redistribution and use in source and binary forms, with or without
...
@@ -119,8 +190,13 @@
...
@@ -119,8 +190,13 @@
#include "endianconv.h"
#include "endianconv.h"
#include "redisassert.h"
#include "redisassert.h"
#define ZIP_END 255
#define ZIP_END 255
/* Special "end of ziplist" entry. */
#define ZIP_BIGLEN 254
#define ZIP_BIG_PREVLEN 254
/* Max number of bytes of the previous entry, for
the "prevlen" field prefixing each entry, to be
represented with just a single byte. Otherwise
it is represented as FF AA BB CC DD, where
AA BB CC DD are a 4 bytes unsigned integer
representing the previous entry len. */
/* Different encoding/length possibilities */
/* Different encoding/length possibilities */
#define ZIP_STR_MASK 0xc0
#define ZIP_STR_MASK 0xc0
...
@@ -133,41 +209,83 @@
...
@@ -133,41 +209,83 @@
#define ZIP_INT_64B (0xc0 | 2<<4)
#define ZIP_INT_64B (0xc0 | 2<<4)
#define ZIP_INT_24B (0xc0 | 3<<4)
#define ZIP_INT_24B (0xc0 | 3<<4)
#define ZIP_INT_8B 0xfe
#define ZIP_INT_8B 0xfe
/* 4 bit integer immediate encoding */
#define ZIP_INT_IMM_MASK 0x0f
/* 4 bit integer immediate encoding |1111xxxx| with xxxx between
* 0001 and 1101. */
#define ZIP_INT_IMM_MASK 0x0f
/* Mask to extract the 4 bits value. To add
one is needed to reconstruct the value. */
#define ZIP_INT_IMM_MIN 0xf1
/* 11110001 */
#define ZIP_INT_IMM_MIN 0xf1
/* 11110001 */
#define ZIP_INT_IMM_MAX 0xfd
/* 11111101 */
#define ZIP_INT_IMM_MAX 0xfd
/* 11111101 */
#define ZIP_INT_IMM_VAL(v) (v & ZIP_INT_IMM_MASK)
#define INT24_MAX 0x7fffff
#define INT24_MAX 0x7fffff
#define INT24_MIN (-INT24_MAX - 1)
#define INT24_MIN (-INT24_MAX - 1)
/* Macro to determine type */
/* Macro to determine if the entry is a string. String entries never start
* with "11" as most significant bits of the first byte. */
#define ZIP_IS_STR(enc) (((enc) & ZIP_STR_MASK) < ZIP_STR_MASK)
#define ZIP_IS_STR(enc) (((enc) & ZIP_STR_MASK) < ZIP_STR_MASK)
/* Utility macros */
/* Utility macros.*/
/* Return total bytes a ziplist is composed of. */
#define ZIPLIST_BYTES(zl) (*((uint32_t*)(zl)))
#define ZIPLIST_BYTES(zl) (*((uint32_t*)(zl)))
/* Return the offset of the last item inside the ziplist. */
#define ZIPLIST_TAIL_OFFSET(zl) (*((uint32_t*)((zl)+sizeof(uint32_t))))
#define ZIPLIST_TAIL_OFFSET(zl) (*((uint32_t*)((zl)+sizeof(uint32_t))))
/* Return the length of a ziplist, or UINT16_MAX if the length cannot be
* determined without scanning the whole ziplist. */
#define ZIPLIST_LENGTH(zl) (*((uint16_t*)((zl)+sizeof(uint32_t)*2)))
#define ZIPLIST_LENGTH(zl) (*((uint16_t*)((zl)+sizeof(uint32_t)*2)))
/* The size of a ziplist header: two 32 bit integers for the total
* bytes count and last item offset. One 16 bit integer for the number
* of items field. */
#define ZIPLIST_HEADER_SIZE (sizeof(uint32_t)*2+sizeof(uint16_t))
#define ZIPLIST_HEADER_SIZE (sizeof(uint32_t)*2+sizeof(uint16_t))
/* Size of the "end of ziplist" entry. Just one byte. */
#define ZIPLIST_END_SIZE (sizeof(uint8_t))
#define ZIPLIST_END_SIZE (sizeof(uint8_t))
/* Return the pointer to the first entry of a ziplist. */
#define ZIPLIST_ENTRY_HEAD(zl) ((zl)+ZIPLIST_HEADER_SIZE)
#define ZIPLIST_ENTRY_HEAD(zl) ((zl)+ZIPLIST_HEADER_SIZE)
/* Return the pointer to the last entry of a ziplist, using the
* last entry offset inside the ziplist header. */
#define ZIPLIST_ENTRY_TAIL(zl) ((zl)+intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl)))
#define ZIPLIST_ENTRY_TAIL(zl) ((zl)+intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl)))
/* Return the pointer to the last byte of a ziplist, which is, the
* end of ziplist FF entry. */
#define ZIPLIST_ENTRY_END(zl) ((zl)+intrev32ifbe(ZIPLIST_BYTES(zl))-1)
#define ZIPLIST_ENTRY_END(zl) ((zl)+intrev32ifbe(ZIPLIST_BYTES(zl))-1)
/* We know a positive increment can only be 1 because entries can only be
/* Increment the number of items field in the ziplist header. Note that this
* pushed one at a time. */
* macro should never overflow the unsigned 16 bit integer, since entires are
* always pushed one at a time. When UINT16_MAX is reached we want the count
* to stay there to signal that a full scan is needed to get the number of
* items inside the ziplist. */
#define ZIPLIST_INCR_LENGTH(zl,incr) { \
#define ZIPLIST_INCR_LENGTH(zl,incr) { \
if (ZIPLIST_LENGTH(zl) < UINT16_MAX) \
if (ZIPLIST_LENGTH(zl) < UINT16_MAX) \
ZIPLIST_LENGTH(zl) = intrev16ifbe(intrev16ifbe(ZIPLIST_LENGTH(zl))+incr); \
ZIPLIST_LENGTH(zl) = intrev16ifbe(intrev16ifbe(ZIPLIST_LENGTH(zl))+incr); \
}
}
/* We use this function to receive information about a ziplist entry.
* Note that this is not how the data is actually encoded, is just what we
* get filled by a function in order to operate more easily. */
typedef
struct
zlentry
{
typedef
struct
zlentry
{
unsigned
int
prevrawlensize
,
prevrawlen
;
unsigned
int
prevrawlensize
;
/* Bytes used to encode the previos entry len*/
unsigned
int
lensize
,
len
;
unsigned
int
prevrawlen
;
/* Previous entry len. */
unsigned
int
headersize
;
unsigned
int
lensize
;
/* Bytes used to encode this entry type/len.
unsigned
char
encoding
;
For example strings have a 1, 2 or 5 bytes
unsigned
char
*
p
;
header. Integers always use a single byte.*/
unsigned
int
len
;
/* Bytes used to represent the actual entry.
For strings this is just the string length
while for integers it is 1, 2, 3, 4, 8 or
0 (for 4 bit immediate) depending on the
number range. */
unsigned
int
headersize
;
/* prevrawlensize + lensize. */
unsigned
char
encoding
;
/* Set to ZIP_STR_* or ZIP_INT_* depending on
the entry encoding. However for 4 bits
immediate integers this can assume a range
of values and must be range-checked. */
unsigned
char
*
p
;
/* Pointer to the very start of the entry, that
is, this points to prev-entry-len field. */
}
zlentry
;
}
zlentry
;
#define ZIPLIST_ENTRY_ZERO(zle) { \
#define ZIPLIST_ENTRY_ZERO(zle) { \
...
@@ -178,31 +296,40 @@ typedef struct zlentry {
...
@@ -178,31 +296,40 @@ typedef struct zlentry {
}
}
/* Extract the encoding from the byte pointed by 'ptr' and set it into
/* Extract the encoding from the byte pointed by 'ptr' and set it into
* 'encoding'. */
* 'encoding'
field of the zlentry structure
. */
#define ZIP_ENTRY_ENCODING(ptr, encoding) do { \
#define ZIP_ENTRY_ENCODING(ptr, encoding) do { \
(encoding) = (ptr[0]); \
(encoding) = (ptr[0]); \
if ((encoding) < ZIP_STR_MASK) (encoding) &= ZIP_STR_MASK; \
if ((encoding) < ZIP_STR_MASK) (encoding) &= ZIP_STR_MASK; \
} while(0)
} while(0)
void
ziplistRepr
(
unsigned
char
*
zl
);
/* Return bytes needed to store integer encoded by 'encoding'. */
unsigned
int
zipIntSize
(
unsigned
char
encoding
)
{
/* Return bytes needed to store integer encoded by 'encoding' */
static
unsigned
int
zipIntSize
(
unsigned
char
encoding
)
{
switch
(
encoding
)
{
switch
(
encoding
)
{
case
ZIP_INT_8B
:
return
1
;
case
ZIP_INT_8B
:
return
1
;
case
ZIP_INT_16B
:
return
2
;
case
ZIP_INT_16B
:
return
2
;
case
ZIP_INT_24B
:
return
3
;
case
ZIP_INT_24B
:
return
3
;
case
ZIP_INT_32B
:
return
4
;
case
ZIP_INT_32B
:
return
4
;
case
ZIP_INT_64B
:
return
8
;
case
ZIP_INT_64B
:
return
8
;
default:
return
0
;
/* 4 bit immediate */
}
}
assert
(
NULL
);
if
(
encoding
>=
ZIP_INT_IMM_MIN
&&
encoding
<=
ZIP_INT_IMM_MAX
)
return
0
;
/* 4 bit immediate */
panic
(
"Invalid integer encoding 0x%02X"
,
encoding
);
return
0
;
return
0
;
}
}
/* Encode the length 'rawlen' writing it in 'p'. If p is NULL it just returns
/* Write the encoidng header of the entry in 'p'. If p is NULL it just returns
* the amount of bytes required to encode such a length. */
* the amount of bytes required to encode such a length. Arguments:
static
unsigned
int
zipEncodeLength
(
unsigned
char
*
p
,
unsigned
char
encoding
,
unsigned
int
rawlen
)
{
*
* 'encoding' is the encoding we are using for the entry. It could be
* ZIP_INT_* or ZIP_STR_* or between ZIP_INT_IMM_MIN and ZIP_INT_IMM_MAX
* for single-byte small immediate integers.
*
* 'rawlen' is only used for ZIP_STR_* encodings and is the length of the
* srting that this entry represents.
*
* The function returns the number of bytes used by the encoding/length
* header stored in 'p'. */
unsigned
int
zipStoreEntryEncoding
(
unsigned
char
*
p
,
unsigned
char
encoding
,
unsigned
int
rawlen
)
{
unsigned
char
len
=
1
,
buf
[
5
];
unsigned
char
len
=
1
,
buf
[
5
];
if
(
ZIP_IS_STR
(
encoding
))
{
if
(
ZIP_IS_STR
(
encoding
))
{
...
@@ -231,15 +358,16 @@ static unsigned int zipEncodeLength(unsigned char *p, unsigned char encoding, un
...
@@ -231,15 +358,16 @@ static unsigned int zipEncodeLength(unsigned char *p, unsigned char encoding, un
buf
[
0
]
=
encoding
;
buf
[
0
]
=
encoding
;
}
}
/* Store this length at p */
/* Store this length at p
.
*/
memcpy
(
p
,
buf
,
len
);
memcpy
(
p
,
buf
,
len
);
return
len
;
return
len
;
}
}
/* Decode the length encoded in 'ptr'. The 'encoding' variable will hold the
/* Decode the entry encoding type and data length (string length for strings,
* entries encoding, the 'lensize' variable will hold the number of bytes
* number of bytes used for the integer for integer entries) encoded in 'ptr'.
* required to encode the entries length, and the 'len' variable will hold the
* The 'encoding' variable will hold the entry encoding, the 'lensize'
* entries length. */
* variable will hold the number of bytes required to encode the entry
* length, and the 'len' variable will hold the entry length. */
#define ZIP_DECODE_LENGTH(ptr, encoding, lensize, len) do { \
#define ZIP_DECODE_LENGTH(ptr, encoding, lensize, len) do { \
ZIP_ENTRY_ENCODING((ptr), (encoding)); \
ZIP_ENTRY_ENCODING((ptr), (encoding)); \
if ((encoding) < ZIP_STR_MASK) { \
if ((encoding) < ZIP_STR_MASK) { \
...
@@ -256,7 +384,7 @@ static unsigned int zipEncodeLength(unsigned char *p, unsigned char encoding, un
...
@@ -256,7 +384,7 @@ static unsigned int zipEncodeLength(unsigned char *p, unsigned char encoding, un
((ptr)[3] << 8) | \
((ptr)[3] << 8) | \
((ptr)[4]); \
((ptr)[4]); \
} else { \
} else { \
assert(NULL);
\
panic("Invalid string encoding 0x%02X", (encoding));
\
} \
} \
} else { \
} else { \
(lensize) = 1; \
(lensize) = 1; \
...
@@ -264,45 +392,49 @@ static unsigned int zipEncodeLength(unsigned char *p, unsigned char encoding, un
...
@@ -264,45 +392,49 @@ static unsigned int zipEncodeLength(unsigned char *p, unsigned char encoding, un
} \
} \
} while(0);
} while(0);
/* Encode the length of the previous entry and write it to "p". This only
* uses the larger encoding (required in __ziplistCascadeUpdate). */
int
zipStorePrevEntryLengthLarge
(
unsigned
char
*
p
,
unsigned
int
len
)
{
if
(
p
!=
NULL
)
{
p
[
0
]
=
ZIP_BIG_PREVLEN
;
memcpy
(
p
+
1
,
&
len
,
sizeof
(
len
));
memrev32ifbe
(
p
+
1
);
}
return
1
+
sizeof
(
len
);
}
/* Encode the length of the previous entry and write it to "p". Return the
/* Encode the length of the previous entry and write it to "p". Return the
* number of bytes needed to encode this length if "p" is NULL. */
* number of bytes needed to encode this length if "p" is NULL. */
static
unsigned
int
zipPrevEn
code
Length
(
unsigned
char
*
p
,
unsigned
int
len
)
{
unsigned
int
zip
Store
PrevEn
try
Length
(
unsigned
char
*
p
,
unsigned
int
len
)
{
if
(
p
==
NULL
)
{
if
(
p
==
NULL
)
{
return
(
len
<
ZIP_BIGLEN
)
?
1
:
sizeof
(
len
)
+
1
;
return
(
len
<
ZIP_BIG
_PREV
LEN
)
?
1
:
sizeof
(
len
)
+
1
;
}
else
{
}
else
{
if
(
len
<
ZIP_BIGLEN
)
{
if
(
len
<
ZIP_BIG
_PREV
LEN
)
{
p
[
0
]
=
len
;
p
[
0
]
=
len
;
return
1
;
return
1
;
}
else
{
}
else
{
p
[
0
]
=
ZIP_BIGLEN
;
return
zipStorePrevEntryLengthLarge
(
p
,
len
);
memcpy
(
p
+
1
,
&
len
,
sizeof
(
len
));
memrev32ifbe
(
p
+
1
);
return
1
+
sizeof
(
len
);
}
}
}
}
}
}
/* Encode the length of the previous entry and write it to "p". This only
/* Return the number of bytes used to encode the length of the previous
* uses the larger encoding (required in __ziplistCascadeUpdate). */
* entry. The length is returned by setting the var 'prevlensize'. */
static
void
zipPrevEncodeLengthForceLarge
(
unsigned
char
*
p
,
unsigned
int
len
)
{
if
(
p
==
NULL
)
return
;
p
[
0
]
=
ZIP_BIGLEN
;
memcpy
(
p
+
1
,
&
len
,
sizeof
(
len
));
memrev32ifbe
(
p
+
1
);
}
/* Decode the number of bytes required to store the length of the previous
* element, from the perspective of the entry pointed to by 'ptr'. */
#define ZIP_DECODE_PREVLENSIZE(ptr, prevlensize) do { \
#define ZIP_DECODE_PREVLENSIZE(ptr, prevlensize) do { \
if ((ptr)[0] < ZIP_BIGLEN) {
\
if ((ptr)[0] < ZIP_BIG
_PREV
LEN) { \
(prevlensize) = 1; \
(prevlensize) = 1; \
} else { \
} else { \
(prevlensize) = 5; \
(prevlensize) = 5; \
} \
} \
} while(0);
} while(0);
/* Decode the length of the previous element, from the perspective of the entry
/* Return the length of the previous element, and the number of bytes that
* pointed to by 'ptr'. */
* are used in order to encode the previous element length.
* 'ptr' must point to the prevlen prefix of an entry (that encodes the
* length of the previos entry in order to navigate the elements backward).
* The length of the previous entry is stored in 'prevlen', the number of
* bytes needed to encode the previous entry length are stored in
* 'prevlensize'. */
#define ZIP_DECODE_PREVLEN(ptr, prevlensize, prevlen) do { \
#define ZIP_DECODE_PREVLEN(ptr, prevlensize, prevlen) do { \
ZIP_DECODE_PREVLENSIZE(ptr, prevlensize); \
ZIP_DECODE_PREVLENSIZE(ptr, prevlensize); \
if ((prevlensize) == 1) { \
if ((prevlensize) == 1) { \
...
@@ -314,16 +446,29 @@ static void zipPrevEncodeLengthForceLarge(unsigned char *p, unsigned int len) {
...
@@ -314,16 +446,29 @@ static void zipPrevEncodeLengthForceLarge(unsigned char *p, unsigned int len) {
} \
} \
} while(0);
} while(0);
/* Return the difference in number of bytes needed to store the length of the
/* Given a pointer 'p' to the prevlen info that prefixes an entry, this
* previous element 'len', in the entry pointed to by 'p'. */
* function returns the difference in number of bytes needed to encode
static
int
zipPrevLenByteDiff
(
unsigned
char
*
p
,
unsigned
int
len
)
{
* the prevlen if the previous entry changes of size.
*
* So if A is the number of bytes used right now to encode the 'prevlen'
* field.
*
* And B is the number of bytes that are needed in order to encode the
* 'prevlen' if the previous element will be updated to one of size 'len'.
*
* Then the function returns B - A
*
* So the function returns a positive number if more space is needed,
* a negative number if less space is needed, or zero if the same space
* is needed. */
int
zipPrevLenByteDiff
(
unsigned
char
*
p
,
unsigned
int
len
)
{
unsigned
int
prevlensize
;
unsigned
int
prevlensize
;
ZIP_DECODE_PREVLENSIZE
(
p
,
prevlensize
);
ZIP_DECODE_PREVLENSIZE
(
p
,
prevlensize
);
return
zipPrevEn
code
Length
(
NULL
,
len
)
-
prevlensize
;
return
zip
Store
PrevEn
try
Length
(
NULL
,
len
)
-
prevlensize
;
}
}
/* Return the total number of bytes used by the entry pointed to by 'p'. */
/* Return the total number of bytes used by the entry pointed to by 'p'. */
static
unsigned
int
zipRawEntryLength
(
unsigned
char
*
p
)
{
unsigned
int
zipRawEntryLength
(
unsigned
char
*
p
)
{
unsigned
int
prevlensize
,
encoding
,
lensize
,
len
;
unsigned
int
prevlensize
,
encoding
,
lensize
,
len
;
ZIP_DECODE_PREVLENSIZE
(
p
,
prevlensize
);
ZIP_DECODE_PREVLENSIZE
(
p
,
prevlensize
);
ZIP_DECODE_LENGTH
(
p
+
prevlensize
,
encoding
,
lensize
,
len
);
ZIP_DECODE_LENGTH
(
p
+
prevlensize
,
encoding
,
lensize
,
len
);
...
@@ -332,7 +477,7 @@ static unsigned int zipRawEntryLength(unsigned char *p) {
...
@@ -332,7 +477,7 @@ static unsigned int zipRawEntryLength(unsigned char *p) {
/* Check if string pointed to by 'entry' can be encoded as an integer.
/* Check if string pointed to by 'entry' can be encoded as an integer.
* Stores the integer value in 'v' and its encoding in 'encoding'. */
* Stores the integer value in 'v' and its encoding in 'encoding'. */
static
int
zipTryEncoding
(
unsigned
char
*
entry
,
unsigned
int
entrylen
,
long
long
*
v
,
unsigned
char
*
encoding
)
{
int
zipTryEncoding
(
unsigned
char
*
entry
,
unsigned
int
entrylen
,
long
long
*
v
,
unsigned
char
*
encoding
)
{
long
long
value
;
long
long
value
;
if
(
entrylen
>=
32
||
entrylen
==
0
)
return
0
;
if
(
entrylen
>=
32
||
entrylen
==
0
)
return
0
;
...
@@ -359,7 +504,7 @@ static int zipTryEncoding(unsigned char *entry, unsigned int entrylen, long long
...
@@ -359,7 +504,7 @@ static int zipTryEncoding(unsigned char *entry, unsigned int entrylen, long long
}
}
/* Store integer 'value' at 'p', encoded as 'encoding' */
/* Store integer 'value' at 'p', encoded as 'encoding' */
static
void
zipSaveInteger
(
unsigned
char
*
p
,
int64_t
value
,
unsigned
char
encoding
)
{
void
zipSaveInteger
(
unsigned
char
*
p
,
int64_t
value
,
unsigned
char
encoding
)
{
int16_t
i16
;
int16_t
i16
;
int32_t
i32
;
int32_t
i32
;
int64_t
i64
;
int64_t
i64
;
...
@@ -389,7 +534,7 @@ static void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encodi
...
@@ -389,7 +534,7 @@ static void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encodi
}
}
/* Read integer encoded as 'encoding' from 'p' */
/* Read integer encoded as 'encoding' from 'p' */
static
int64_t
zipLoadInteger
(
unsigned
char
*
p
,
unsigned
char
encoding
)
{
int64_t
zipLoadInteger
(
unsigned
char
*
p
,
unsigned
char
encoding
)
{
int16_t
i16
;
int16_t
i16
;
int32_t
i32
;
int32_t
i32
;
int64_t
i64
,
ret
=
0
;
int64_t
i64
,
ret
=
0
;
...
@@ -421,7 +566,7 @@ static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) {
...
@@ -421,7 +566,7 @@ static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) {
}
}
/* Return a struct with all information about an entry. */
/* Return a struct with all information about an entry. */
static
void
zipEntry
(
unsigned
char
*
p
,
zlentry
*
e
)
{
void
zipEntry
(
unsigned
char
*
p
,
zlentry
*
e
)
{
ZIP_DECODE_PREVLEN
(
p
,
e
->
prevrawlensize
,
e
->
prevrawlen
);
ZIP_DECODE_PREVLEN
(
p
,
e
->
prevrawlensize
,
e
->
prevrawlen
);
ZIP_DECODE_LENGTH
(
p
+
e
->
prevrawlensize
,
e
->
encoding
,
e
->
lensize
,
e
->
len
);
ZIP_DECODE_LENGTH
(
p
+
e
->
prevrawlensize
,
e
->
encoding
,
e
->
lensize
,
e
->
len
);
...
@@ -441,7 +586,7 @@ unsigned char *ziplistNew(void) {
...
@@ -441,7 +586,7 @@ unsigned char *ziplistNew(void) {
}
}
/* Resize the ziplist. */
/* Resize the ziplist. */
static
unsigned
char
*
ziplistResize
(
unsigned
char
*
zl
,
unsigned
int
len
)
{
unsigned
char
*
ziplistResize
(
unsigned
char
*
zl
,
unsigned
int
len
)
{
zl
=
zrealloc
(
zl
,
len
);
zl
=
zrealloc
(
zl
,
len
);
ZIPLIST_BYTES
(
zl
)
=
intrev32ifbe
(
len
);
ZIPLIST_BYTES
(
zl
)
=
intrev32ifbe
(
len
);
zl
[
len
-
1
]
=
ZIP_END
;
zl
[
len
-
1
]
=
ZIP_END
;
...
@@ -456,8 +601,8 @@ static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
...
@@ -456,8 +601,8 @@ static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
* causes a realloc and memmove). However, encoding the prevlen may require
* causes a realloc and memmove). However, encoding the prevlen may require
* that this entry is grown as well. This effect may cascade throughout
* that this entry is grown as well. This effect may cascade throughout
* the ziplist when there are consecutive entries with a size close to
* the ziplist when there are consecutive entries with a size close to
* ZIP_BIGLEN, so we need to check that the prevlen can be encoded in
every
* ZIP_BIG
_PREV
LEN, so we need to check that the prevlen can be encoded in
* consecutive entry.
*
every
consecutive entry.
*
*
* Note that this effect can also happen in reverse, where the bytes required
* Note that this effect can also happen in reverse, where the bytes required
* to encode the prevlen field can shrink. This effect is deliberately ignored,
* to encode the prevlen field can shrink. This effect is deliberately ignored,
...
@@ -468,7 +613,7 @@ static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
...
@@ -468,7 +613,7 @@ static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
*
*
* The pointer "p" points to the first entry that does NOT need to be
* The pointer "p" points to the first entry that does NOT need to be
* updated, i.e. consecutive fields MAY need an update. */
* updated, i.e. consecutive fields MAY need an update. */
static
unsigned
char
*
__ziplistCascadeUpdate
(
unsigned
char
*
zl
,
unsigned
char
*
p
)
{
unsigned
char
*
__ziplistCascadeUpdate
(
unsigned
char
*
zl
,
unsigned
char
*
p
)
{
size_t
curlen
=
intrev32ifbe
(
ZIPLIST_BYTES
(
zl
)),
rawlen
,
rawlensize
;
size_t
curlen
=
intrev32ifbe
(
ZIPLIST_BYTES
(
zl
)),
rawlen
,
rawlensize
;
size_t
offset
,
noffset
,
extra
;
size_t
offset
,
noffset
,
extra
;
unsigned
char
*
np
;
unsigned
char
*
np
;
...
@@ -477,7 +622,7 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p
...
@@ -477,7 +622,7 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p
while
(
p
[
0
]
!=
ZIP_END
)
{
while
(
p
[
0
]
!=
ZIP_END
)
{
zipEntry
(
p
,
&
cur
);
zipEntry
(
p
,
&
cur
);
rawlen
=
cur
.
headersize
+
cur
.
len
;
rawlen
=
cur
.
headersize
+
cur
.
len
;
rawlensize
=
zipPrevEn
code
Length
(
NULL
,
rawlen
);
rawlensize
=
zip
Store
PrevEn
try
Length
(
NULL
,
rawlen
);
/* Abort if there is no next entry. */
/* Abort if there is no next entry. */
if
(
p
[
rawlen
]
==
ZIP_END
)
break
;
if
(
p
[
rawlen
]
==
ZIP_END
)
break
;
...
@@ -508,7 +653,7 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p
...
@@ -508,7 +653,7 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p
memmove
(
np
+
rawlensize
,
memmove
(
np
+
rawlensize
,
np
+
next
.
prevrawlensize
,
np
+
next
.
prevrawlensize
,
curlen
-
noffset
-
next
.
prevrawlensize
-
1
);
curlen
-
noffset
-
next
.
prevrawlensize
-
1
);
zipPrevEn
code
Length
(
np
,
rawlen
);
zip
Store
PrevEn
try
Length
(
np
,
rawlen
);
/* Advance the cursor */
/* Advance the cursor */
p
+=
rawlen
;
p
+=
rawlen
;
...
@@ -517,9 +662,9 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p
...
@@ -517,9 +662,9 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p
if
(
next
.
prevrawlensize
>
rawlensize
)
{
if
(
next
.
prevrawlensize
>
rawlensize
)
{
/* This would result in shrinking, which we want to avoid.
/* This would result in shrinking, which we want to avoid.
* So, set "rawlen" in the available bytes. */
* So, set "rawlen" in the available bytes. */
zipPrevEn
code
Length
Force
Large
(
p
+
rawlen
,
rawlen
);
zip
Store
PrevEn
try
LengthLarge
(
p
+
rawlen
,
rawlen
);
}
else
{
}
else
{
zipPrevEn
code
Length
(
p
+
rawlen
,
rawlen
);
zip
Store
PrevEn
try
Length
(
p
+
rawlen
,
rawlen
);
}
}
/* Stop here, as the raw length of "next" has not changed. */
/* Stop here, as the raw length of "next" has not changed. */
...
@@ -530,7 +675,7 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p
...
@@ -530,7 +675,7 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p
}
}
/* Delete "num" entries, starting at "p". Returns pointer to the ziplist. */
/* Delete "num" entries, starting at "p". Returns pointer to the ziplist. */
static
unsigned
char
*
__ziplistDelete
(
unsigned
char
*
zl
,
unsigned
char
*
p
,
unsigned
int
num
)
{
unsigned
char
*
__ziplistDelete
(
unsigned
char
*
zl
,
unsigned
char
*
p
,
unsigned
int
num
)
{
unsigned
int
i
,
totlen
,
deleted
=
0
;
unsigned
int
i
,
totlen
,
deleted
=
0
;
size_t
offset
;
size_t
offset
;
int
nextdiff
=
0
;
int
nextdiff
=
0
;
...
@@ -542,7 +687,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
...
@@ -542,7 +687,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
deleted
++
;
deleted
++
;
}
}
totlen
=
p
-
first
.
p
;
totlen
=
p
-
first
.
p
;
/* Bytes taken by the element(s) to delete. */
if
(
totlen
>
0
)
{
if
(
totlen
>
0
)
{
if
(
p
[
0
]
!=
ZIP_END
)
{
if
(
p
[
0
]
!=
ZIP_END
)
{
/* Storing `prevrawlen` in this entry may increase or decrease the
/* Storing `prevrawlen` in this entry may increase or decrease the
...
@@ -550,8 +695,13 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
...
@@ -550,8 +695,13 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
* There always is room to store this, because it was previously
* There always is room to store this, because it was previously
* stored by an entry that is now being deleted. */
* stored by an entry that is now being deleted. */
nextdiff
=
zipPrevLenByteDiff
(
p
,
first
.
prevrawlen
);
nextdiff
=
zipPrevLenByteDiff
(
p
,
first
.
prevrawlen
);
/* Note that there is always space when p jumps backward: if
* the new previous entry is large, one of the deleted elements
* had a 5 bytes prevlen header, so there is for sure at least
* 5 bytes free and we need just 4. */
p
-=
nextdiff
;
p
-=
nextdiff
;
zipPrevEn
code
Length
(
p
,
first
.
prevrawlen
);
zip
Store
PrevEn
try
Length
(
p
,
first
.
prevrawlen
);
/* Update offset for tail */
/* Update offset for tail */
ZIPLIST_TAIL_OFFSET
(
zl
)
=
ZIPLIST_TAIL_OFFSET
(
zl
)
=
...
@@ -590,7 +740,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
...
@@ -590,7 +740,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
}
}
/* Insert item at "p". */
/* Insert item at "p". */
static
unsigned
char
*
__ziplistInsert
(
unsigned
char
*
zl
,
unsigned
char
*
p
,
unsigned
char
*
s
,
unsigned
int
slen
)
{
unsigned
char
*
__ziplistInsert
(
unsigned
char
*
zl
,
unsigned
char
*
p
,
unsigned
char
*
s
,
unsigned
int
slen
)
{
size_t
curlen
=
intrev32ifbe
(
ZIPLIST_BYTES
(
zl
)),
reqlen
;
size_t
curlen
=
intrev32ifbe
(
ZIPLIST_BYTES
(
zl
)),
reqlen
;
unsigned
int
prevlensize
,
prevlen
=
0
;
unsigned
int
prevlensize
,
prevlen
=
0
;
size_t
offset
;
size_t
offset
;
...
@@ -616,19 +766,24 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
...
@@ -616,19 +766,24 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
/* 'encoding' is set to the appropriate integer encoding */
/* 'encoding' is set to the appropriate integer encoding */
reqlen
=
zipIntSize
(
encoding
);
reqlen
=
zipIntSize
(
encoding
);
}
else
{
}
else
{
/* 'encoding' is untouched, however zip
EncodeLength
will use the
/* 'encoding' is untouched, however zip
StoreEntryEncoding
will use the
* string length to figure out how to encode it. */
* string length to figure out how to encode it. */
reqlen
=
slen
;
reqlen
=
slen
;
}
}
/* We need space for both the length of the previous entry and
/* We need space for both the length of the previous entry and
* the length of the payload. */
* the length of the payload. */
reqlen
+=
zipPrevEn
code
Length
(
NULL
,
prevlen
);
reqlen
+=
zip
Store
PrevEn
try
Length
(
NULL
,
prevlen
);
reqlen
+=
zip
EncodeLength
(
NULL
,
encoding
,
slen
);
reqlen
+=
zip
StoreEntryEncoding
(
NULL
,
encoding
,
slen
);
/* When the insert position is not equal to the tail, we need to
/* When the insert position is not equal to the tail, we need to
* make sure that the next entry can hold this entry's length in
* make sure that the next entry can hold this entry's length in
* its prevlen field. */
* its prevlen field. */
int
forcelarge
=
0
;
nextdiff
=
(
p
[
0
]
!=
ZIP_END
)
?
zipPrevLenByteDiff
(
p
,
reqlen
)
:
0
;
nextdiff
=
(
p
[
0
]
!=
ZIP_END
)
?
zipPrevLenByteDiff
(
p
,
reqlen
)
:
0
;
if
(
nextdiff
==
-
4
&&
reqlen
<
4
)
{
nextdiff
=
0
;
forcelarge
=
1
;
}
/* Store offset because a realloc may change the address of zl. */
/* Store offset because a realloc may change the address of zl. */
offset
=
p
-
zl
;
offset
=
p
-
zl
;
...
@@ -641,7 +796,10 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
...
@@ -641,7 +796,10 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
memmove
(
p
+
reqlen
,
p
-
nextdiff
,
curlen
-
offset
-
1
+
nextdiff
);
memmove
(
p
+
reqlen
,
p
-
nextdiff
,
curlen
-
offset
-
1
+
nextdiff
);
/* Encode this entry's raw length in the next entry. */
/* Encode this entry's raw length in the next entry. */
zipPrevEncodeLength
(
p
+
reqlen
,
reqlen
);
if
(
forcelarge
)
zipStorePrevEntryLengthLarge
(
p
+
reqlen
,
reqlen
);
else
zipStorePrevEntryLength
(
p
+
reqlen
,
reqlen
);
/* Update offset for tail */
/* Update offset for tail */
ZIPLIST_TAIL_OFFSET
(
zl
)
=
ZIPLIST_TAIL_OFFSET
(
zl
)
=
...
@@ -669,8 +827,8 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
...
@@ -669,8 +827,8 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
}
}
/* Write the entry */
/* Write the entry */
p
+=
zipPrevEn
code
Length
(
p
,
prevlen
);
p
+=
zip
Store
PrevEn
try
Length
(
p
,
prevlen
);
p
+=
zip
EncodeLength
(
p
,
encoding
,
slen
);
p
+=
zip
StoreEntryEncoding
(
p
,
encoding
,
slen
);
if
(
ZIP_IS_STR
(
encoding
))
{
if
(
ZIP_IS_STR
(
encoding
))
{
memcpy
(
p
,
s
,
slen
);
memcpy
(
p
,
s
,
slen
);
}
else
{
}
else
{
...
...
tests/integration/aof.tcl
View file @
71a8df6a
...
@@ -88,7 +88,7 @@ tags {"aof"} {
...
@@ -88,7 +88,7 @@ tags {"aof"} {
set pattern
"*Bad file format reading the append only file*"
set pattern
"*Bad file format reading the append only file*"
set retry 10
set retry 10
while
{
$retry
}
{
while
{
$retry
}
{
set result
[
exec tail -
n
1 <
[
dict get $srv stdout
]]
set result
[
exec tail -1 <
[
dict get $srv stdout
]]
if
{[
string match $pattern $result
]}
{
if
{[
string match $pattern $result
]}
{
break
break
}
}
...
@@ -113,7 +113,7 @@ tags {"aof"} {
...
@@ -113,7 +113,7 @@ tags {"aof"} {
set pattern
"*Unexpected end of file reading the append only file*"
set pattern
"*Unexpected end of file reading the append only file*"
set retry 10
set retry 10
while
{
$retry
}
{
while
{
$retry
}
{
set result
[
exec tail -
n
1 <
[
dict get $srv stdout
]]
set result
[
exec tail -1 <
[
dict get $srv stdout
]]
if
{[
string match $pattern $result
]}
{
if
{[
string match $pattern $result
]}
{
break
break
}
}
...
@@ -137,7 +137,7 @@ tags {"aof"} {
...
@@ -137,7 +137,7 @@ tags {"aof"} {
set pattern
"*Unexpected end of file reading the append only file*"
set pattern
"*Unexpected end of file reading the append only file*"
set retry 10
set retry 10
while
{
$retry
}
{
while
{
$retry
}
{
set result
[
exec tail -
n
1 <
[
dict get $srv stdout
]]
set result
[
exec tail -1 <
[
dict get $srv stdout
]]
if
{[
string match $pattern $result
]}
{
if
{[
string match $pattern $result
]}
{
break
break
}
}
...
...
tests/integration/rdb.tcl
View file @
71a8df6a
...
@@ -66,7 +66,7 @@ if {!$isroot} {
...
@@ -66,7 +66,7 @@ if {!$isroot} {
test
{
Server should not start if RDB file can't be open
}
{
test
{
Server should not start if RDB file can't be open
}
{
wait_for_condition 50 100
{
wait_for_condition 50 100
{
[
string match
{
*Fatal error loading*
}
\
[
string match
{
*Fatal error loading*
}
\
[
exec tail -
n
1 <
[
dict get $srv stdout
]]]
[
exec tail -1 <
[
dict get $srv stdout
]]]
}
else
{
}
else
{
fail
"Server started even if RDB was unreadable!"
fail
"Server started even if RDB was unreadable!"
}
}
...
@@ -90,7 +90,7 @@ start_server_and_kill_it [list "dir" $server_path] {
...
@@ -90,7 +90,7 @@ start_server_and_kill_it [list "dir" $server_path] {
test
{
Server should not start if RDB is corrupted
}
{
test
{
Server should not start if RDB is corrupted
}
{
wait_for_condition 50 100
{
wait_for_condition 50 100
{
[
string match
{
*CRC error*
}
\
[
string match
{
*CRC error*
}
\
[
exec tail -
n
10 <
[
dict get $srv stdout
]]]
[
exec tail -10 <
[
dict get $srv stdout
]]]
}
else
{
}
else
{
fail
"Server started even if RDB was corrupted!"
fail
"Server started even if RDB was corrupted!"
}
}
...
...
tests/integration/replication-psync.tcl
View file @
71a8df6a
...
@@ -47,7 +47,7 @@ proc test_psync {descr duration backlog_size backlog_ttl delay cond diskless rec
...
@@ -47,7 +47,7 @@ proc test_psync {descr duration backlog_size backlog_ttl delay cond diskless rec
# Check that the background clients are actually writing.
# Check that the background clients are actually writing.
test
{
Detect write load to master
}
{
test
{
Detect write load to master
}
{
wait_for_condition 50 100
{
wait_for_condition 50 100
0
{
[
$master
dbsize
]
> 100
[
$master
dbsize
]
> 100
}
else
{
}
else
{
fail
"Can't detect write load from background clients."
fail
"Can't detect write load from background clients."
...
...
Prev
1
…
5
6
7
8
9
10
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment