Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
06263485
Commit
06263485
authored
Feb 21, 2017
by
antirez
Browse files
Merge branch 'siphash' into unstable
parents
e084b5a3
0285c271
Changes
12
Hide whitespace changes
Inline
Side-by-side
src/Makefile
View file @
06263485
...
@@ -132,7 +132,7 @@ endif
...
@@ -132,7 +132,7 @@ endif
REDIS_SERVER_NAME
=
redis-server
REDIS_SERVER_NAME
=
redis-server
REDIS_SENTINEL_NAME
=
redis-sentinel
REDIS_SENTINEL_NAME
=
redis-sentinel
REDIS_SERVER_OBJ
=
adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o
REDIS_SERVER_OBJ
=
adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o
siphash.o
REDIS_CLI_NAME
=
redis-cli
REDIS_CLI_NAME
=
redis-cli
REDIS_CLI_OBJ
=
anet.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o
REDIS_CLI_OBJ
=
anet.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o
REDIS_BENCHMARK_NAME
=
redis-benchmark
REDIS_BENCHMARK_NAME
=
redis-benchmark
...
@@ -208,7 +208,7 @@ $(REDIS_CHECK_AOF_NAME): $(REDIS_CHECK_AOF_OBJ)
...
@@ -208,7 +208,7 @@ $(REDIS_CHECK_AOF_NAME): $(REDIS_CHECK_AOF_OBJ)
$(REDIS_LD)
-o
$@
$^
$(FINAL_LIBS)
$(REDIS_LD)
-o
$@
$^
$(FINAL_LIBS)
dict-benchmark
:
dict.c zmalloc.c sds.c
dict-benchmark
:
dict.c zmalloc.c sds.c
$(REDIS_CC)
$(FINAL_CFLAGS)
dict.c zmalloc.c sds.c
-D
DICT_BENCHMARK_MAIN
-o
dict-benchmark
$(REDIS_CC)
$(FINAL_CFLAGS)
dict.c zmalloc.c sds.c
siphash.c
-D
DICT_BENCHMARK_MAIN
-o
dict-benchmark
# Because the jemalloc.h header is generated as a part of the jemalloc build,
# Because the jemalloc.h header is generated as a part of the jemalloc build,
# building it should complete before building any other object. Instead of
# building it should complete before building any other object. Instead of
...
...
src/config.c
View file @
06263485
...
@@ -1423,7 +1423,7 @@ void configGetCommand(client *c) {
...
@@ -1423,7 +1423,7 @@ void configGetCommand(client *c) {
/* We use the following dictionary type to store where a configuration
/* We use the following dictionary type to store where a configuration
* option is mentioned in the old configuration file, so it's
* option is mentioned in the old configuration file, so it's
* like "maxmemory" -> list of line numbers (first line is zero). */
* like "maxmemory" -> list of line numbers (first line is zero). */
u
nsigned
in
t
dictSdsCaseHash
(
const
void
*
key
);
u
int64_
t
dictSdsCaseHash
(
const
void
*
key
);
int
dictSdsKeyCaseCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
int
dictSdsKeyCaseCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
void
dictSdsDestructor
(
void
*
privdata
,
void
*
val
);
void
dictSdsDestructor
(
void
*
privdata
,
void
*
val
);
void
dictListDestructor
(
void
*
privdata
,
void
*
val
);
void
dictListDestructor
(
void
*
privdata
,
void
*
val
);
...
...
src/debug.c
View file @
06263485
...
@@ -1029,8 +1029,6 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
...
@@ -1029,8 +1029,6 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
/* Log INFO and CLIENT LIST */
/* Log INFO and CLIENT LIST */
serverLogRaw
(
LL_WARNING
|
LL_RAW
,
"
\n
------ INFO OUTPUT ------
\n
"
);
serverLogRaw
(
LL_WARNING
|
LL_RAW
,
"
\n
------ INFO OUTPUT ------
\n
"
);
infostring
=
genRedisInfoString
(
"all"
);
infostring
=
genRedisInfoString
(
"all"
);
infostring
=
sdscatprintf
(
infostring
,
"hash_init_value: %u
\n
"
,
dictGetHashFunctionSeed
());
serverLogRaw
(
LL_WARNING
|
LL_RAW
,
infostring
);
serverLogRaw
(
LL_WARNING
|
LL_RAW
,
infostring
);
serverLogRaw
(
LL_WARNING
|
LL_RAW
,
"
\n
------ CLIENT LIST OUTPUT ------
\n
"
);
serverLogRaw
(
LL_WARNING
|
LL_RAW
,
"
\n
------ CLIENT LIST OUTPUT ------
\n
"
);
clients
=
getAllClientsInfoString
();
clients
=
getAllClientsInfoString
();
...
...
src/dict.c
View file @
06263485
...
@@ -37,11 +37,11 @@
...
@@ -37,11 +37,11 @@
#include <stdio.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <string.h>
#include <stdarg.h>
#include <stdarg.h>
#include <limits.h>
#include <limits.h>
#include <sys/time.h>
#include <sys/time.h>
#include <ctype.h>
#include "dict.h"
#include "dict.h"
#include "zmalloc.h"
#include "zmalloc.h"
...
@@ -71,77 +71,28 @@ static int _dictInit(dict *ht, dictType *type, void *privDataPtr);
...
@@ -71,77 +71,28 @@ static int _dictInit(dict *ht, dictType *type, void *privDataPtr);
/* -------------------------- hash functions -------------------------------- */
/* -------------------------- hash functions -------------------------------- */
static
uint
32
_t
dict_hash_function_seed
=
5381
;
static
uint
8
_t
dict_hash_function_seed
[
16
]
;
void
dictSetHashFunctionSeed
(
uint
32
_t
seed
)
{
void
dictSetHashFunctionSeed
(
uint
8
_t
*
seed
)
{
dict_hash_function_seed
=
seed
;
memcpy
(
dict_hash_function_seed
,
seed
,
sizeof
(
dict_hash_function_seed
))
;
}
}
uint
32
_t
dictGetHashFunctionSeed
(
void
)
{
uint
8
_t
*
dictGetHashFunctionSeed
(
void
)
{
return
dict_hash_function_seed
;
return
dict_hash_function_seed
;
}
}
/* MurmurHash2, by Austin Appleby
/* The default hashing function uses SipHash implementation
* Note - This code makes a few assumptions about how your machine behaves -
* in siphash.c. */
* 1. We can read a 4-byte value from any address without crashing
* 2. sizeof(int) == 4
*
* And it has a few limitations -
*
* 1. It will not work incrementally.
* 2. It will not produce the same results on little-endian and big-endian
* machines.
*/
unsigned
int
dictGenHashFunction
(
const
void
*
key
,
int
len
)
{
/* 'm' and 'r' are mixing constants generated offline.
They're not really 'magic', they just happen to work well. */
uint32_t
seed
=
dict_hash_function_seed
;
const
uint32_t
m
=
0x5bd1e995
;
const
int
r
=
24
;
/* Initialize the hash to a 'random' value */
uint32_t
h
=
seed
^
len
;
/* Mix 4 bytes at a time into the hash */
const
unsigned
char
*
data
=
(
const
unsigned
char
*
)
key
;
while
(
len
>=
4
)
{
uint32_t
k
=
*
(
uint32_t
*
)
data
;
k
*=
m
;
k
^=
k
>>
r
;
k
*=
m
;
h
*=
m
;
uint64_t
siphash
(
const
uint8_t
*
in
,
const
size_t
inlen
,
const
uint8_t
*
k
)
;
h
^=
k
;
uint64_t
siphash_nocase
(
const
uint8_t
*
in
,
const
size_t
inlen
,
const
uint8_t
*
k
)
;
data
+=
4
;
uint64_t
dictGenHashFunction
(
const
void
*
key
,
int
len
)
{
len
-=
4
;
return
siphash
(
key
,
len
,
dict_hash_function_seed
);
}
/* Handle the last few bytes of the input array */
switch
(
len
)
{
case
3
:
h
^=
data
[
2
]
<<
16
;
case
2
:
h
^=
data
[
1
]
<<
8
;
case
1
:
h
^=
data
[
0
];
h
*=
m
;
};
/* Do a few final mixes of the hash to ensure the last few
* bytes are well-incorporated. */
h
^=
h
>>
13
;
h
*=
m
;
h
^=
h
>>
15
;
return
(
unsigned
int
)
h
;
}
}
/* And a case insensitive hash function (based on djb hash) */
uint64_t
dictGenCaseHashFunction
(
const
unsigned
char
*
buf
,
int
len
)
{
unsigned
int
dictGenCaseHashFunction
(
const
unsigned
char
*
buf
,
int
len
)
{
return
siphash_nocase
(
buf
,
len
,
dict_hash_function_seed
);
unsigned
int
hash
=
(
unsigned
int
)
dict_hash_function_seed
;
while
(
len
--
)
hash
=
((
hash
<<
5
)
+
hash
)
+
(
tolower
(
*
buf
++
));
/* hash * 33 + c */
return
hash
;
}
}
/* ----------------------------- API implementation ------------------------- */
/* ----------------------------- API implementation ------------------------- */
...
@@ -1158,7 +1109,7 @@ void dictGetStats(char *buf, size_t bufsize, dict *d) {
...
@@ -1158,7 +1109,7 @@ void dictGetStats(char *buf, size_t bufsize, dict *d) {
#include "sds.h"
#include "sds.h"
u
nsigned
in
t
hashCallback
(
const
void
*
key
)
{
u
int64_
t
hashCallback
(
const
void
*
key
)
{
return
dictGenHashFunction
((
unsigned
char
*
)
key
,
sdslen
((
char
*
)
key
));
return
dictGenHashFunction
((
unsigned
char
*
)
key
,
sdslen
((
char
*
)
key
));
}
}
...
...
src/dict.h
View file @
06263485
...
@@ -56,7 +56,7 @@ typedef struct dictEntry {
...
@@ -56,7 +56,7 @@ typedef struct dictEntry {
}
dictEntry
;
}
dictEntry
;
typedef
struct
dictType
{
typedef
struct
dictType
{
u
nsigned
in
t
(
*
hashFunction
)(
const
void
*
key
);
u
int64_
t
(
*
hashFunction
)(
const
void
*
key
);
void
*
(
*
keyDup
)(
void
*
privdata
,
const
void
*
key
);
void
*
(
*
keyDup
)(
void
*
privdata
,
const
void
*
key
);
void
*
(
*
valDup
)(
void
*
privdata
,
const
void
*
obj
);
void
*
(
*
valDup
)(
void
*
privdata
,
const
void
*
obj
);
int
(
*
keyCompare
)(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
int
(
*
keyCompare
)(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
...
@@ -168,15 +168,15 @@ void dictReleaseIterator(dictIterator *iter);
...
@@ -168,15 +168,15 @@ void dictReleaseIterator(dictIterator *iter);
dictEntry
*
dictGetRandomKey
(
dict
*
d
);
dictEntry
*
dictGetRandomKey
(
dict
*
d
);
unsigned
int
dictGetSomeKeys
(
dict
*
d
,
dictEntry
**
des
,
unsigned
int
count
);
unsigned
int
dictGetSomeKeys
(
dict
*
d
,
dictEntry
**
des
,
unsigned
int
count
);
void
dictGetStats
(
char
*
buf
,
size_t
bufsize
,
dict
*
d
);
void
dictGetStats
(
char
*
buf
,
size_t
bufsize
,
dict
*
d
);
u
nsigned
in
t
dictGenHashFunction
(
const
void
*
key
,
int
len
);
u
int64_
t
dictGenHashFunction
(
const
void
*
key
,
int
len
);
u
nsigned
in
t
dictGenCaseHashFunction
(
const
unsigned
char
*
buf
,
int
len
);
u
int64_
t
dictGenCaseHashFunction
(
const
unsigned
char
*
buf
,
int
len
);
void
dictEmpty
(
dict
*
d
,
void
(
callback
)(
void
*
));
void
dictEmpty
(
dict
*
d
,
void
(
callback
)(
void
*
));
void
dictEnableResize
(
void
);
void
dictEnableResize
(
void
);
void
dictDisableResize
(
void
);
void
dictDisableResize
(
void
);
int
dictRehash
(
dict
*
d
,
int
n
);
int
dictRehash
(
dict
*
d
,
int
n
);
int
dictRehashMilliseconds
(
dict
*
d
,
int
ms
);
int
dictRehashMilliseconds
(
dict
*
d
,
int
ms
);
void
dictSetHashFunctionSeed
(
u
nsigned
int
initval
);
void
dictSetHashFunctionSeed
(
u
int8_t
*
seed
);
u
nsigned
in
t
dictGetHashFunctionSeed
(
void
);
u
int8_
t
*
dictGetHashFunctionSeed
(
void
);
unsigned
long
dictScan
(
dict
*
d
,
unsigned
long
v
,
dictScanFunction
*
fn
,
dictScanBucketFunction
*
bucketfn
,
void
*
privdata
);
unsigned
long
dictScan
(
dict
*
d
,
unsigned
long
v
,
dictScanFunction
*
fn
,
dictScanBucketFunction
*
bucketfn
,
void
*
privdata
);
unsigned
int
dictGetHash
(
dict
*
d
,
const
void
*
key
);
unsigned
int
dictGetHash
(
dict
*
d
,
const
void
*
key
);
dictEntry
**
dictFindEntryRefByPtrAndHash
(
dict
*
d
,
const
void
*
oldptr
,
unsigned
int
hash
);
dictEntry
**
dictFindEntryRefByPtrAndHash
(
dict
*
d
,
const
void
*
oldptr
,
unsigned
int
hash
);
...
...
src/latency.c
View file @
06263485
...
@@ -41,7 +41,7 @@ int dictStringKeyCompare(void *privdata, const void *key1, const void *key2) {
...
@@ -41,7 +41,7 @@ int dictStringKeyCompare(void *privdata, const void *key1, const void *key2) {
return
strcmp
(
key1
,
key2
)
==
0
;
return
strcmp
(
key1
,
key2
)
==
0
;
}
}
u
nsigned
in
t
dictStringHash
(
const
void
*
key
)
{
u
int64_
t
dictStringHash
(
const
void
*
key
)
{
return
dictGenHashFunction
(
key
,
strlen
(
key
));
return
dictGenHashFunction
(
key
,
strlen
(
key
));
}
}
...
...
src/module.c
View file @
06263485
...
@@ -3264,7 +3264,7 @@ void *RM_GetBlockedClientPrivateData(RedisModuleCtx *ctx) {
...
@@ -3264,7 +3264,7 @@ void *RM_GetBlockedClientPrivateData(RedisModuleCtx *ctx) {
/* server.moduleapi dictionary type. Only uses plain C strings since
/* server.moduleapi dictionary type. Only uses plain C strings since
* this gets queries from modules. */
* this gets queries from modules. */
u
nsigned
in
t
dictCStringKeyHash
(
const
void
*
key
)
{
u
int64_
t
dictCStringKeyHash
(
const
void
*
key
)
{
return
dictGenHashFunction
((
unsigned
char
*
)
key
,
strlen
((
char
*
)
key
));
return
dictGenHashFunction
((
unsigned
char
*
)
key
,
strlen
((
char
*
)
key
));
}
}
...
...
src/sentinel.c
View file @
06263485
...
@@ -379,7 +379,7 @@ void sentinelSimFailureCrash(void);
...
@@ -379,7 +379,7 @@ void sentinelSimFailureCrash(void);
/* ========================= Dictionary types =============================== */
/* ========================= Dictionary types =============================== */
u
nsigned
in
t
dictSdsHash
(
const
void
*
key
);
u
int64_
t
dictSdsHash
(
const
void
*
key
);
int
dictSdsKeyCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
int
dictSdsKeyCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
void
releaseSentinelRedisInstance
(
sentinelRedisInstance
*
ri
);
void
releaseSentinelRedisInstance
(
sentinelRedisInstance
*
ri
);
...
...
src/server.c
View file @
06263485
...
@@ -482,16 +482,16 @@ int dictObjKeyCompare(void *privdata, const void *key1,
...
@@ -482,16 +482,16 @@ int dictObjKeyCompare(void *privdata, const void *key1,
return
dictSdsKeyCompare
(
privdata
,
o1
->
ptr
,
o2
->
ptr
);
return
dictSdsKeyCompare
(
privdata
,
o1
->
ptr
,
o2
->
ptr
);
}
}
u
nsigned
in
t
dictObjHash
(
const
void
*
key
)
{
u
int64_
t
dictObjHash
(
const
void
*
key
)
{
const
robj
*
o
=
key
;
const
robj
*
o
=
key
;
return
dictGenHashFunction
(
o
->
ptr
,
sdslen
((
sds
)
o
->
ptr
));
return
dictGenHashFunction
(
o
->
ptr
,
sdslen
((
sds
)
o
->
ptr
));
}
}
u
nsigned
in
t
dictSdsHash
(
const
void
*
key
)
{
u
int64_
t
dictSdsHash
(
const
void
*
key
)
{
return
dictGenHashFunction
((
unsigned
char
*
)
key
,
sdslen
((
char
*
)
key
));
return
dictGenHashFunction
((
unsigned
char
*
)
key
,
sdslen
((
char
*
)
key
));
}
}
u
nsigned
in
t
dictSdsCaseHash
(
const
void
*
key
)
{
u
int64_
t
dictSdsCaseHash
(
const
void
*
key
)
{
return
dictGenCaseHashFunction
((
unsigned
char
*
)
key
,
sdslen
((
char
*
)
key
));
return
dictGenCaseHashFunction
((
unsigned
char
*
)
key
,
sdslen
((
char
*
)
key
));
}
}
...
@@ -513,7 +513,7 @@ int dictEncObjKeyCompare(void *privdata, const void *key1,
...
@@ -513,7 +513,7 @@ int dictEncObjKeyCompare(void *privdata, const void *key1,
return
cmp
;
return
cmp
;
}
}
u
nsigned
in
t
dictEncObjHash
(
const
void
*
key
)
{
u
int64_
t
dictEncObjHash
(
const
void
*
key
)
{
robj
*
o
=
(
robj
*
)
key
;
robj
*
o
=
(
robj
*
)
key
;
if
(
sdsEncodedObject
(
o
))
{
if
(
sdsEncodedObject
(
o
))
{
...
@@ -526,7 +526,7 @@ unsigned int dictEncObjHash(const void *key) {
...
@@ -526,7 +526,7 @@ unsigned int dictEncObjHash(const void *key) {
len
=
ll2string
(
buf
,
32
,(
long
)
o
->
ptr
);
len
=
ll2string
(
buf
,
32
,(
long
)
o
->
ptr
);
return
dictGenHashFunction
((
unsigned
char
*
)
buf
,
len
);
return
dictGenHashFunction
((
unsigned
char
*
)
buf
,
len
);
}
else
{
}
else
{
u
nsigned
in
t
hash
;
u
int64_
t
hash
;
o
=
getDecodedObject
(
o
);
o
=
getDecodedObject
(
o
);
hash
=
dictGenHashFunction
(
o
->
ptr
,
sdslen
((
sds
)
o
->
ptr
));
hash
=
dictGenHashFunction
(
o
->
ptr
,
sdslen
((
sds
)
o
->
ptr
));
...
@@ -3639,7 +3639,9 @@ int main(int argc, char **argv) {
...
@@ -3639,7 +3639,9 @@ int main(int argc, char **argv) {
zmalloc_set_oom_handler
(
redisOutOfMemoryHandler
);
zmalloc_set_oom_handler
(
redisOutOfMemoryHandler
);
srand
(
time
(
NULL
)
^
getpid
());
srand
(
time
(
NULL
)
^
getpid
());
gettimeofday
(
&
tv
,
NULL
);
gettimeofday
(
&
tv
,
NULL
);
dictSetHashFunctionSeed
(
tv
.
tv_sec
^
tv
.
tv_usec
^
getpid
());
char
hashseed
[
16
];
getRandomHexChars
(
hashseed
,
sizeof
(
hashseed
));
dictSetHashFunctionSeed
((
uint8_t
*
)
hashseed
);
server
.
sentinel_mode
=
checkForSentinelMode
(
argc
,
argv
);
server
.
sentinel_mode
=
checkForSentinelMode
(
argc
,
argv
);
initServerConfig
();
initServerConfig
();
moduleInitModulesSystem
();
moduleInitModulesSystem
();
...
...
src/server.h
View file @
06263485
...
@@ -1765,7 +1765,7 @@ unsigned long LFUGetTimeInMinutes(void);
...
@@ -1765,7 +1765,7 @@ unsigned long LFUGetTimeInMinutes(void);
uint8_t
LFULogIncr
(
uint8_t
value
);
uint8_t
LFULogIncr
(
uint8_t
value
);
/* Keys hashing / comparison functions for dict.c hash tables. */
/* Keys hashing / comparison functions for dict.c hash tables. */
u
nsigned
in
t
dictSdsHash
(
const
void
*
key
);
u
int64_
t
dictSdsHash
(
const
void
*
key
);
int
dictSdsKeyCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
int
dictSdsKeyCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
void
dictSdsDestructor
(
void
*
privdata
,
void
*
val
);
void
dictSdsDestructor
(
void
*
privdata
,
void
*
val
);
...
...
src/siphash.c
0 → 100644
View file @
06263485
/*
SipHash reference C implementation
Copyright (c) 2012-2016 Jean-Philippe Aumasson
<jeanphilippe.aumasson@gmail.com>
Copyright (c) 2012-2014 Daniel J. Bernstein <djb@cr.yp.to>
Copyright (c) 2017 Salvatore Sanfilippo <antirez@gmail.com>
To the extent possible under law, the author(s) have dedicated all copyright
and related and neighboring rights to this software to the public domain
worldwide. This software is distributed without any warranty.
You should have received a copy of the CC0 Public Domain Dedication along
with this software. If not, see
<http://creativecommons.org/publicdomain/zero/1.0/>.
----------------------------------------------------------------------------
This version was modified by Salvatore Sanfilippo <antirez@gmail.com>
in the following ways:
1. We use SipHash 1-2. This is not believed to be as strong as the
suggested 2-4 variant, but AFAIK there are not trivial attacks
against this reduced-rounds version, and it runs at the same speed
as Murmurhash2 that we used previously, why the 2-4 variant slowed
down Redis by a 4% figure more or less.
2. Hard-code rounds in the hope the compiler can optimize it more
in this raw from. Anyway we always want the standard 2-4 variant.
3. Modify the prototype and implementation so that the function directly
returns an uint64_t value, the hash itself, instead of receiving an
output buffer. This also means that the output size is set to 8 bytes
and the 16 bytes output code handling was removed.
4. Provide a case insensitive variant to be used when hashing strings that
must be considered identical by the hash table regardless of the case.
If we don't have directly a case insensitive hash function, we need to
perform a text transformation in some temporary buffer, which is costly.
5. Remove debugging code.
6. Modified the original test.c file to be a stand-alone function testing
the function in the new form (returing an uint64_t) using just the
relevant test vector.
*/
#include <assert.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
/* Fast tolower() alike function that does not care about locale
* but just returns a-z insetad of A-Z. */
int
siptlw
(
int
c
)
{
if
(
c
>=
'A'
&&
c
<=
'Z'
)
{
return
c
+
(
'a'
-
'A'
);
}
else
{
return
c
;
}
}
/* Test of the CPU is Little Endian and supports not aligned accesses.
* Two interesting conditions to speedup the function that happen to be
* in most of x86 servers. */
#if defined(__X86_64__) || defined(__x86_64__) || defined (__i386__)
#define UNALIGNED_LE_CPU
#endif
#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
#define U32TO8_LE(p, v) \
(p)[0] = (uint8_t)((v)); \
(p)[1] = (uint8_t)((v) >> 8); \
(p)[2] = (uint8_t)((v) >> 16); \
(p)[3] = (uint8_t)((v) >> 24);
#define U64TO8_LE(p, v) \
U32TO8_LE((p), (uint32_t)((v))); \
U32TO8_LE((p) + 4, (uint32_t)((v) >> 32));
#ifdef UNALIGNED_LE_CPU
#define U8TO64_LE(p) (*((uint64_t*)(p)))
#else
#define U8TO64_LE(p) \
(((uint64_t)((p)[0])) | ((uint64_t)((p)[1]) << 8) | \
((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24) | \
((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) | \
((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56))
#endif
#define U8TO64_LE_NOCASE(p) \
(((uint64_t)(siptlw((p)[0]))) | \
((uint64_t)(siptlw((p)[1])) << 8) | \
((uint64_t)(siptlw((p)[2])) << 16) | \
((uint64_t)(siptlw((p)[3])) << 24) | \
((uint64_t)(siptlw((p)[4])) << 32) | \
((uint64_t)(siptlw((p)[5])) << 40) | \
((uint64_t)(siptlw((p)[6])) << 48) | \
((uint64_t)(siptlw((p)[7])) << 56))
#define SIPROUND \
do { \
v0 += v1; \
v1 = ROTL(v1, 13); \
v1 ^= v0; \
v0 = ROTL(v0, 32); \
v2 += v3; \
v3 = ROTL(v3, 16); \
v3 ^= v2; \
v0 += v3; \
v3 = ROTL(v3, 21); \
v3 ^= v0; \
v2 += v1; \
v1 = ROTL(v1, 17); \
v1 ^= v2; \
v2 = ROTL(v2, 32); \
} while (0)
uint64_t
siphash
(
const
uint8_t
*
in
,
const
size_t
inlen
,
const
uint8_t
*
k
)
{
#ifndef UNALIGNED_LE_CPU
uint64_t
hash
;
uint8_t
*
out
=
(
uint8_t
*
)
&
hash
;
#endif
uint64_t
v0
=
0x736f6d6570736575ULL
;
uint64_t
v1
=
0x646f72616e646f6dULL
;
uint64_t
v2
=
0x6c7967656e657261ULL
;
uint64_t
v3
=
0x7465646279746573ULL
;
uint64_t
k0
=
U8TO64_LE
(
k
);
uint64_t
k1
=
U8TO64_LE
(
k
+
8
);
uint64_t
m
;
const
uint8_t
*
end
=
in
+
inlen
-
(
inlen
%
sizeof
(
uint64_t
));
const
int
left
=
inlen
&
7
;
uint64_t
b
=
((
uint64_t
)
inlen
)
<<
56
;
v3
^=
k1
;
v2
^=
k0
;
v1
^=
k1
;
v0
^=
k0
;
for
(;
in
!=
end
;
in
+=
8
)
{
m
=
U8TO64_LE
(
in
);
v3
^=
m
;
SIPROUND
;
v0
^=
m
;
}
switch
(
left
)
{
case
7
:
b
|=
((
uint64_t
)
in
[
6
])
<<
48
;
case
6
:
b
|=
((
uint64_t
)
in
[
5
])
<<
40
;
case
5
:
b
|=
((
uint64_t
)
in
[
4
])
<<
32
;
case
4
:
b
|=
((
uint64_t
)
in
[
3
])
<<
24
;
case
3
:
b
|=
((
uint64_t
)
in
[
2
])
<<
16
;
case
2
:
b
|=
((
uint64_t
)
in
[
1
])
<<
8
;
case
1
:
b
|=
((
uint64_t
)
in
[
0
]);
break
;
case
0
:
break
;
}
v3
^=
b
;
SIPROUND
;
v0
^=
b
;
v2
^=
0xff
;
SIPROUND
;
SIPROUND
;
b
=
v0
^
v1
^
v2
^
v3
;
#ifndef UNALIGNED_LE_CPU
U64TO8_LE
(
out
,
b
);
return
hash
;
#else
return
b
;
#endif
}
uint64_t
siphash_nocase
(
const
uint8_t
*
in
,
const
size_t
inlen
,
const
uint8_t
*
k
)
{
#ifndef UNALIGNED_LE_CPU
uint64_t
hash
;
uint8_t
*
out
=
(
uint8_t
*
)
&
hash
;
#endif
uint64_t
v0
=
0x736f6d6570736575ULL
;
uint64_t
v1
=
0x646f72616e646f6dULL
;
uint64_t
v2
=
0x6c7967656e657261ULL
;
uint64_t
v3
=
0x7465646279746573ULL
;
uint64_t
k0
=
U8TO64_LE
(
k
);
uint64_t
k1
=
U8TO64_LE
(
k
+
8
);
uint64_t
m
;
const
uint8_t
*
end
=
in
+
inlen
-
(
inlen
%
sizeof
(
uint64_t
));
const
int
left
=
inlen
&
7
;
uint64_t
b
=
((
uint64_t
)
inlen
)
<<
56
;
v3
^=
k1
;
v2
^=
k0
;
v1
^=
k1
;
v0
^=
k0
;
for
(;
in
!=
end
;
in
+=
8
)
{
m
=
U8TO64_LE_NOCASE
(
in
);
v3
^=
m
;
SIPROUND
;
v0
^=
m
;
}
switch
(
left
)
{
case
7
:
b
|=
((
uint64_t
)
siptlw
(
in
[
6
]))
<<
48
;
case
6
:
b
|=
((
uint64_t
)
siptlw
(
in
[
5
]))
<<
40
;
case
5
:
b
|=
((
uint64_t
)
siptlw
(
in
[
4
]))
<<
32
;
case
4
:
b
|=
((
uint64_t
)
siptlw
(
in
[
3
]))
<<
24
;
case
3
:
b
|=
((
uint64_t
)
siptlw
(
in
[
2
]))
<<
16
;
case
2
:
b
|=
((
uint64_t
)
siptlw
(
in
[
1
]))
<<
8
;
case
1
:
b
|=
((
uint64_t
)
siptlw
(
in
[
0
]));
break
;
case
0
:
break
;
}
v3
^=
b
;
SIPROUND
;
v0
^=
b
;
v2
^=
0xff
;
SIPROUND
;
SIPROUND
;
b
=
v0
^
v1
^
v2
^
v3
;
#ifndef UNALIGNED_LE_CPU
U64TO8_LE
(
out
,
b
);
return
hash
;
#else
return
b
;
#endif
}
/* --------------------------------- TEST ------------------------------------ */
#ifdef SIPHASH_TEST
const
uint8_t
vectors_sip64
[
64
][
8
]
=
{
{
0x31
,
0x0e
,
0x0e
,
0xdd
,
0x47
,
0xdb
,
0x6f
,
0x72
,
},
{
0xfd
,
0x67
,
0xdc
,
0x93
,
0xc5
,
0x39
,
0xf8
,
0x74
,
},
{
0x5a
,
0x4f
,
0xa9
,
0xd9
,
0x09
,
0x80
,
0x6c
,
0x0d
,
},
{
0x2d
,
0x7e
,
0xfb
,
0xd7
,
0x96
,
0x66
,
0x67
,
0x85
,
},
{
0xb7
,
0x87
,
0x71
,
0x27
,
0xe0
,
0x94
,
0x27
,
0xcf
,
},
{
0x8d
,
0xa6
,
0x99
,
0xcd
,
0x64
,
0x55
,
0x76
,
0x18
,
},
{
0xce
,
0xe3
,
0xfe
,
0x58
,
0x6e
,
0x46
,
0xc9
,
0xcb
,
},
{
0x37
,
0xd1
,
0x01
,
0x8b
,
0xf5
,
0x00
,
0x02
,
0xab
,
},
{
0x62
,
0x24
,
0x93
,
0x9a
,
0x79
,
0xf5
,
0xf5
,
0x93
,
},
{
0xb0
,
0xe4
,
0xa9
,
0x0b
,
0xdf
,
0x82
,
0x00
,
0x9e
,
},
{
0xf3
,
0xb9
,
0xdd
,
0x94
,
0xc5
,
0xbb
,
0x5d
,
0x7a
,
},
{
0xa7
,
0xad
,
0x6b
,
0x22
,
0x46
,
0x2f
,
0xb3
,
0xf4
,
},
{
0xfb
,
0xe5
,
0x0e
,
0x86
,
0xbc
,
0x8f
,
0x1e
,
0x75
,
},
{
0x90
,
0x3d
,
0x84
,
0xc0
,
0x27
,
0x56
,
0xea
,
0x14
,
},
{
0xee
,
0xf2
,
0x7a
,
0x8e
,
0x90
,
0xca
,
0x23
,
0xf7
,
},
{
0xe5
,
0x45
,
0xbe
,
0x49
,
0x61
,
0xca
,
0x29
,
0xa1
,
},
{
0xdb
,
0x9b
,
0xc2
,
0x57
,
0x7f
,
0xcc
,
0x2a
,
0x3f
,
},
{
0x94
,
0x47
,
0xbe
,
0x2c
,
0xf5
,
0xe9
,
0x9a
,
0x69
,
},
{
0x9c
,
0xd3
,
0x8d
,
0x96
,
0xf0
,
0xb3
,
0xc1
,
0x4b
,
},
{
0xbd
,
0x61
,
0x79
,
0xa7
,
0x1d
,
0xc9
,
0x6d
,
0xbb
,
},
{
0x98
,
0xee
,
0xa2
,
0x1a
,
0xf2
,
0x5c
,
0xd6
,
0xbe
,
},
{
0xc7
,
0x67
,
0x3b
,
0x2e
,
0xb0
,
0xcb
,
0xf2
,
0xd0
,
},
{
0x88
,
0x3e
,
0xa3
,
0xe3
,
0x95
,
0x67
,
0x53
,
0x93
,
},
{
0xc8
,
0xce
,
0x5c
,
0xcd
,
0x8c
,
0x03
,
0x0c
,
0xa8
,
},
{
0x94
,
0xaf
,
0x49
,
0xf6
,
0xc6
,
0x50
,
0xad
,
0xb8
,
},
{
0xea
,
0xb8
,
0x85
,
0x8a
,
0xde
,
0x92
,
0xe1
,
0xbc
,
},
{
0xf3
,
0x15
,
0xbb
,
0x5b
,
0xb8
,
0x35
,
0xd8
,
0x17
,
},
{
0xad
,
0xcf
,
0x6b
,
0x07
,
0x63
,
0x61
,
0x2e
,
0x2f
,
},
{
0xa5
,
0xc9
,
0x1d
,
0xa7
,
0xac
,
0xaa
,
0x4d
,
0xde
,
},
{
0x71
,
0x65
,
0x95
,
0x87
,
0x66
,
0x50
,
0xa2
,
0xa6
,
},
{
0x28
,
0xef
,
0x49
,
0x5c
,
0x53
,
0xa3
,
0x87
,
0xad
,
},
{
0x42
,
0xc3
,
0x41
,
0xd8
,
0xfa
,
0x92
,
0xd8
,
0x32
,
},
{
0xce
,
0x7c
,
0xf2
,
0x72
,
0x2f
,
0x51
,
0x27
,
0x71
,
},
{
0xe3
,
0x78
,
0x59
,
0xf9
,
0x46
,
0x23
,
0xf3
,
0xa7
,
},
{
0x38
,
0x12
,
0x05
,
0xbb
,
0x1a
,
0xb0
,
0xe0
,
0x12
,
},
{
0xae
,
0x97
,
0xa1
,
0x0f
,
0xd4
,
0x34
,
0xe0
,
0x15
,
},
{
0xb4
,
0xa3
,
0x15
,
0x08
,
0xbe
,
0xff
,
0x4d
,
0x31
,
},
{
0x81
,
0x39
,
0x62
,
0x29
,
0xf0
,
0x90
,
0x79
,
0x02
,
},
{
0x4d
,
0x0c
,
0xf4
,
0x9e
,
0xe5
,
0xd4
,
0xdc
,
0xca
,
},
{
0x5c
,
0x73
,
0x33
,
0x6a
,
0x76
,
0xd8
,
0xbf
,
0x9a
,
},
{
0xd0
,
0xa7
,
0x04
,
0x53
,
0x6b
,
0xa9
,
0x3e
,
0x0e
,
},
{
0x92
,
0x59
,
0x58
,
0xfc
,
0xd6
,
0x42
,
0x0c
,
0xad
,
},
{
0xa9
,
0x15
,
0xc2
,
0x9b
,
0xc8
,
0x06
,
0x73
,
0x18
,
},
{
0x95
,
0x2b
,
0x79
,
0xf3
,
0xbc
,
0x0a
,
0xa6
,
0xd4
,
},
{
0xf2
,
0x1d
,
0xf2
,
0xe4
,
0x1d
,
0x45
,
0x35
,
0xf9
,
},
{
0x87
,
0x57
,
0x75
,
0x19
,
0x04
,
0x8f
,
0x53
,
0xa9
,
},
{
0x10
,
0xa5
,
0x6c
,
0xf5
,
0xdf
,
0xcd
,
0x9a
,
0xdb
,
},
{
0xeb
,
0x75
,
0x09
,
0x5c
,
0xcd
,
0x98
,
0x6c
,
0xd0
,
},
{
0x51
,
0xa9
,
0xcb
,
0x9e
,
0xcb
,
0xa3
,
0x12
,
0xe6
,
},
{
0x96
,
0xaf
,
0xad
,
0xfc
,
0x2c
,
0xe6
,
0x66
,
0xc7
,
},
{
0x72
,
0xfe
,
0x52
,
0x97
,
0x5a
,
0x43
,
0x64
,
0xee
,
},
{
0x5a
,
0x16
,
0x45
,
0xb2
,
0x76
,
0xd5
,
0x92
,
0xa1
,
},
{
0xb2
,
0x74
,
0xcb
,
0x8e
,
0xbf
,
0x87
,
0x87
,
0x0a
,
},
{
0x6f
,
0x9b
,
0xb4
,
0x20
,
0x3d
,
0xe7
,
0xb3
,
0x81
,
},
{
0xea
,
0xec
,
0xb2
,
0xa3
,
0x0b
,
0x22
,
0xa8
,
0x7f
,
},
{
0x99
,
0x24
,
0xa4
,
0x3c
,
0xc1
,
0x31
,
0x57
,
0x24
,
},
{
0xbd
,
0x83
,
0x8d
,
0x3a
,
0xaf
,
0xbf
,
0x8d
,
0xb7
,
},
{
0x0b
,
0x1a
,
0x2a
,
0x32
,
0x65
,
0xd5
,
0x1a
,
0xea
,
},
{
0x13
,
0x50
,
0x79
,
0xa3
,
0x23
,
0x1c
,
0xe6
,
0x60
,
},
{
0x93
,
0x2b
,
0x28
,
0x46
,
0xe4
,
0xd7
,
0x06
,
0x66
,
},
{
0xe1
,
0x91
,
0x5f
,
0x5c
,
0xb1
,
0xec
,
0xa4
,
0x6c
,
},
{
0xf3
,
0x25
,
0x96
,
0x5c
,
0xa1
,
0x6d
,
0x62
,
0x9f
,
},
{
0x57
,
0x5f
,
0xf2
,
0x8e
,
0x60
,
0x38
,
0x1b
,
0xe5
,
},
{
0x72
,
0x45
,
0x06
,
0xeb
,
0x4c
,
0x32
,
0x8a
,
0x95
,
},
};
/* Test siphash using a test vector. Returns 0 if the function passed
* all the tests, otherwise 1 is returned.
*
* IMPORTANT: The test vector is for SipHash 2-4. Before running
* the test revert back the siphash() function to 2-4 rounds since
* now it uses 1-2 rounds. */
int
siphash_test
(
void
)
{
uint8_t
in
[
64
],
k
[
16
];
int
i
;
int
fails
=
0
;
for
(
i
=
0
;
i
<
16
;
++
i
)
k
[
i
]
=
i
;
for
(
i
=
0
;
i
<
64
;
++
i
)
{
in
[
i
]
=
i
;
uint64_t
hash
=
siphash
(
in
,
i
,
k
);
const
uint8_t
*
v
=
NULL
;
v
=
(
uint8_t
*
)
vectors_sip64
;
if
(
memcmp
(
&
hash
,
v
+
(
i
*
8
),
8
))
{
/* printf("fail for %d bytes\n", i); */
fails
++
;
}
}
/* Run a few basic tests with the case insensitive version. */
uint64_t
h1
,
h2
;
h1
=
siphash
((
uint8_t
*
)
"hello world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
h2
=
siphash_nocase
((
uint8_t
*
)
"hello world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
if
(
h1
!=
h2
)
fails
++
;
h1
=
siphash
((
uint8_t
*
)
"hello world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
h2
=
siphash_nocase
((
uint8_t
*
)
"HELLO world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
if
(
h1
!=
h2
)
fails
++
;
h1
=
siphash
((
uint8_t
*
)
"HELLO world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
h2
=
siphash_nocase
((
uint8_t
*
)
"HELLO world"
,
11
,(
uint8_t
*
)
"1234567812345678"
);
if
(
h1
==
h2
)
fails
++
;
if
(
!
fails
)
return
0
;
return
1
;
}
int
main
(
void
)
{
if
(
siphash_test
()
==
0
)
{
printf
(
"SipHash test: OK
\n
"
);
return
0
;
}
else
{
printf
(
"SipHash test: FAILED
\n
"
);
return
1
;
}
}
#endif
src/t_zset.c
View file @
06263485
...
@@ -2110,7 +2110,7 @@ inline static void zunionInterAggregate(double *target, double val, int aggregat
...
@@ -2110,7 +2110,7 @@ inline static void zunionInterAggregate(double *target, double val, int aggregat
}
}
}
}
u
nsigned
in
t
dictSdsHash
(
const
void
*
key
);
u
int64_
t
dictSdsHash
(
const
void
*
key
);
int
dictSdsKeyCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
int
dictSdsKeyCompare
(
void
*
privdata
,
const
void
*
key1
,
const
void
*
key2
);
dictType
setAccumulatorDictType
=
{
dictType
setAccumulatorDictType
=
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment