Unverified Commit f3f6f7c0 authored by Viktor Söderqvist's avatar Viktor Söderqvist Committed by GitHub
Browse files

Key as dict entry - memory optimization for sets (#11595)

If a dict has only keys, and no use of values, then a key can be stored directly in a
dict's hashtable. The key replaces the dictEntry. To distinguish between a key and
a dictEntry, we only use this optimization if the key is odd, i.e. if the key has the least
significant bit set. This is true for sds strings, since the sds header is always an odd
number of bytes.

Dict entries are used as a fallback when there is a hash collision. A special dict entry
without a value (only key and next) is used so we save one word in this case too.

This saves 24 bytes per set element for larges sets, and also gains some speed improvement
as a side effect (less allocations and cache misses).

A quick test adding 1M elements to a set using the command below resulted in memory
usage of 28.83M, compared to 46.29M on unstable.
That's 18 bytes per set element on average.

    eval 'for i=1,1000000,1 do redis.call("sadd", "myset", "x"..i) end' 0

Other changes:

Allocations are ensured to have at least 8 bits alignment on all systems. This affects 32-bit
builds compiled without HAVE_MALLOC_SIZE (not jemalloc or glibc) in which Redis
stores the size of each allocation, after this change in 8 bytes instead of previously 4 bytes
per allocation. This is done so we can reliably use the 3 least significant bits in a pointer to
encode stuff.
parent b4123663
...@@ -238,46 +238,26 @@ void activeDefragZsetEntry(zset *zs, dictEntry *de) { ...@@ -238,46 +238,26 @@ void activeDefragZsetEntry(zset *zs, dictEntry *de) {
#define DEFRAG_SDS_DICT_VAL_VOID_PTR 3 #define DEFRAG_SDS_DICT_VAL_VOID_PTR 3
#define DEFRAG_SDS_DICT_VAL_LUA_SCRIPT 4 #define DEFRAG_SDS_DICT_VAL_LUA_SCRIPT 4
typedef struct { void activeDefragSdsDictCallback(void *privdata, const dictEntry *de) {
dict *dict; UNUSED(privdata);
int val_type; UNUSED(de);
} activeDefragSdsDictData;
void activeDefragSdsDictCallback(void *privdata, const dictEntry *_de) {
dictEntry *de = (dictEntry*)_de;
activeDefragSdsDictData *data = privdata;
dict *d = data->dict;
int val_type = data->val_type;
sds sdsele = dictGetKey(de), newsds;
if ((newsds = activeDefragSds(sdsele)))
dictSetKey(d, de, newsds);
/* defrag the value */
if (val_type == DEFRAG_SDS_DICT_VAL_IS_SDS) {
sdsele = dictGetVal(de);
if ((newsds = activeDefragSds(sdsele)))
dictSetVal(d, de, newsds);
} else if (val_type == DEFRAG_SDS_DICT_VAL_IS_STROB) {
robj *newele, *ele = dictGetVal(de);
if ((newele = activeDefragStringOb(ele)))
dictSetVal(d, de, newele);
} else if (val_type == DEFRAG_SDS_DICT_VAL_VOID_PTR) {
void *newptr, *ptr = dictGetVal(de);
if ((newptr = activeDefragAlloc(ptr)))
dictSetVal(d, de, newptr);
} else if (val_type == DEFRAG_SDS_DICT_VAL_LUA_SCRIPT) {
void *newptr, *ptr = dictGetVal(de);
if ((newptr = activeDefragLuaScript(ptr)))
dictSetVal(d, de, newptr);
}
} }
/* Defrag a dict with sds key and optional value (either ptr, sds or robj string) */ /* Defrag a dict with sds key and optional value (either ptr, sds or robj string) */
void activeDefragSdsDict(dict* d, int val_type) { void activeDefragSdsDict(dict* d, int val_type) {
activeDefragSdsDictData data = {d, val_type};
unsigned long cursor = 0; unsigned long cursor = 0;
dictDefragFunctions defragfns = {
.defragAlloc = activeDefragAlloc,
.defragKey = (dictDefragAllocFunction *)activeDefragSds,
.defragVal = (val_type == DEFRAG_SDS_DICT_VAL_IS_SDS ? (dictDefragAllocFunction *)activeDefragSds :
val_type == DEFRAG_SDS_DICT_VAL_IS_STROB ? (dictDefragAllocFunction *)activeDefragStringOb :
val_type == DEFRAG_SDS_DICT_VAL_VOID_PTR ? (dictDefragAllocFunction *)activeDefragAlloc :
val_type == DEFRAG_SDS_DICT_VAL_LUA_SCRIPT ? (dictDefragAllocFunction *)activeDefragLuaScript :
NULL)
};
do { do {
cursor = dictScanDefrag(d, cursor, activeDefragSdsDictCallback, cursor = dictScanDefrag(d, cursor, activeDefragSdsDictCallback,
activeDefragAlloc, &data); &defragfns, NULL);
} while (cursor != 0); } while (cursor != 0);
} }
...@@ -407,19 +387,14 @@ void scanLaterZset(robj *ob, unsigned long *cursor) { ...@@ -407,19 +387,14 @@ void scanLaterZset(robj *ob, unsigned long *cursor) {
zset *zs = (zset*)ob->ptr; zset *zs = (zset*)ob->ptr;
dict *d = zs->dict; dict *d = zs->dict;
scanLaterZsetData data = {zs}; scanLaterZsetData data = {zs};
*cursor = dictScanDefrag(d, *cursor, scanLaterZsetCallback, activeDefragAlloc, &data); dictDefragFunctions defragfns = {.defragAlloc = activeDefragAlloc};
*cursor = dictScanDefrag(d, *cursor, scanLaterZsetCallback, &defragfns, &data);
} }
typedef struct { /* Used as scan callback when all the work is done in the dictDefragFunctions. */
dict *dict; void scanCallbackCountScanned(void *privdata, const dictEntry *de) {
} scanLaterDictData; UNUSED(privdata);
UNUSED(de);
void scanLaterSetCallback(void *privdata, const dictEntry *_de) {
dictEntry *de = (dictEntry*)_de;
scanLaterDictData *data = privdata;
sds sdsele = dictGetKey(de), newsds;
if ((newsds = activeDefragSds(sdsele)))
dictSetKey(data->dict, de, newsds);
server.stat_active_defrag_scanned++; server.stat_active_defrag_scanned++;
} }
...@@ -427,28 +402,23 @@ void scanLaterSet(robj *ob, unsigned long *cursor) { ...@@ -427,28 +402,23 @@ void scanLaterSet(robj *ob, unsigned long *cursor) {
if (ob->type != OBJ_SET || ob->encoding != OBJ_ENCODING_HT) if (ob->type != OBJ_SET || ob->encoding != OBJ_ENCODING_HT)
return; return;
dict *d = ob->ptr; dict *d = ob->ptr;
scanLaterDictData data = {d}; dictDefragFunctions defragfns = {
*cursor = dictScanDefrag(d, *cursor, scanLaterSetCallback, activeDefragAlloc, &data); .defragAlloc = activeDefragAlloc,
} .defragKey = (dictDefragAllocFunction *)activeDefragSds
};
void scanLaterHashCallback(void *privdata, const dictEntry *_de) { *cursor = dictScanDefrag(d, *cursor, scanCallbackCountScanned, &defragfns, NULL);
dictEntry *de = (dictEntry*)_de;
scanLaterDictData *data = privdata;
sds sdsele = dictGetKey(de), newsds;
if ((newsds = activeDefragSds(sdsele)))
dictSetKey(data->dict, de, newsds);
sdsele = dictGetVal(de);
if ((newsds = activeDefragSds(sdsele)))
dictSetVal(data->dict, de, newsds);
server.stat_active_defrag_scanned++;
} }
void scanLaterHash(robj *ob, unsigned long *cursor) { void scanLaterHash(robj *ob, unsigned long *cursor) {
if (ob->type != OBJ_HASH || ob->encoding != OBJ_ENCODING_HT) if (ob->type != OBJ_HASH || ob->encoding != OBJ_ENCODING_HT)
return; return;
dict *d = ob->ptr; dict *d = ob->ptr;
scanLaterDictData data = {d}; dictDefragFunctions defragfns = {
*cursor = dictScanDefrag(d, *cursor, scanLaterHashCallback, activeDefragAlloc, &data); .defragAlloc = activeDefragAlloc,
.defragKey = (dictDefragAllocFunction *)activeDefragSds,
.defragVal = (dictDefragAllocFunction *)activeDefragSds
};
*cursor = dictScanDefrag(d, *cursor, scanCallbackCountScanned, &defragfns, NULL);
} }
void defragQuicklist(redisDb *db, dictEntry *kde) { void defragQuicklist(redisDb *db, dictEntry *kde) {
...@@ -788,14 +758,6 @@ void defragScanCallback(void *privdata, const dictEntry *de) { ...@@ -788,14 +758,6 @@ void defragScanCallback(void *privdata, const dictEntry *de) {
server.stat_active_defrag_scanned++; server.stat_active_defrag_scanned++;
} }
/* Dummy scan callback used when defragging the expire dictionary. We only
* defrag the entries, which is done per bucket. */
void defragExpireScanCallback(void *privdata, const dictEntry *de) {
UNUSED(privdata);
UNUSED(de);
server.stat_active_defrag_scanned++;
}
/* Utility function to get the fragmentation ratio from jemalloc. /* Utility function to get the fragmentation ratio from jemalloc.
* It is critical to do that by comparing only heap maps that belong to * It is critical to do that by comparing only heap maps that belong to
* jemalloc, and skip ones the jemalloc keeps as spare. Since we use this * jemalloc, and skip ones the jemalloc keeps as spare. Since we use this
...@@ -1003,6 +965,7 @@ void activeDefragCycle(void) { ...@@ -1003,6 +965,7 @@ void activeDefragCycle(void) {
endtime = start + timelimit; endtime = start + timelimit;
latencyStartMonitor(latency); latencyStartMonitor(latency);
dictDefragFunctions defragfns = {.defragAlloc = activeDefragAlloc};
do { do {
/* if we're not continuing a scan from the last call or loop, start a new one */ /* if we're not continuing a scan from the last call or loop, start a new one */
if (!cursor && !expires_cursor) { if (!cursor && !expires_cursor) {
...@@ -1055,13 +1018,13 @@ void activeDefragCycle(void) { ...@@ -1055,13 +1018,13 @@ void activeDefragCycle(void) {
/* Scan the keyspace dict unless we're scanning the expire dict. */ /* Scan the keyspace dict unless we're scanning the expire dict. */
if (!expires_cursor) if (!expires_cursor)
cursor = dictScanDefrag(db->dict, cursor, defragScanCallback, cursor = dictScanDefrag(db->dict, cursor, defragScanCallback,
activeDefragAlloc, db); &defragfns, db);
/* When done scanning the keyspace dict, we scan the expire dict. */ /* When done scanning the keyspace dict, we scan the expire dict. */
if (!cursor) if (!cursor)
expires_cursor = dictScanDefrag(db->expires, expires_cursor, expires_cursor = dictScanDefrag(db->expires, expires_cursor,
defragExpireScanCallback, scanCallbackCountScanned,
activeDefragAlloc, NULL); &defragfns, NULL);
/* Once in 16 scan iterations, 512 pointer reallocations. or 64 keys /* Once in 16 scan iterations, 512 pointer reallocations. or 64 keys
* (if we have a lot of pointers in one hash bucket or rehashing), * (if we have a lot of pointers in one hash bucket or rehashing),
......
This diff is collapsed.
...@@ -56,6 +56,19 @@ typedef struct dictType { ...@@ -56,6 +56,19 @@ typedef struct dictType {
void (*keyDestructor)(dict *d, void *key); void (*keyDestructor)(dict *d, void *key);
void (*valDestructor)(dict *d, void *obj); void (*valDestructor)(dict *d, void *obj);
int (*expandAllowed)(size_t moreMem, double usedRatio); int (*expandAllowed)(size_t moreMem, double usedRatio);
/* Flags */
/* The 'no_value' flag, if set, indicates that values are not used, i.e. the
* dict is a set. When this flag is set, it's not possible to access the
* value of a dictEntry and it's also impossible to use dictSetKey(). Entry
* metadata can also not be used. */
unsigned int no_value:1;
/* If no_value = 1 and all keys are odd (LSB=1), setting keys_are_odd = 1
* enables one more optimization: to store a key without an allocated
* dictEntry. */
unsigned int keys_are_odd:1;
/* TODO: Add a 'keys_are_even' flag and use a similar optimization if that
* flag is set. */
/* Allow each dict and dictEntry to carry extra caller-defined metadata. The /* Allow each dict and dictEntry to carry extra caller-defined metadata. The
* extra memory is initialized to 0 when allocated. */ * extra memory is initialized to 0 when allocated. */
size_t (*dictEntryMetadataBytes)(dict *d); size_t (*dictEntryMetadataBytes)(dict *d);
...@@ -100,6 +113,11 @@ typedef struct dictIterator { ...@@ -100,6 +113,11 @@ typedef struct dictIterator {
typedef void (dictScanFunction)(void *privdata, const dictEntry *de); typedef void (dictScanFunction)(void *privdata, const dictEntry *de);
typedef void *(dictDefragAllocFunction)(void *ptr); typedef void *(dictDefragAllocFunction)(void *ptr);
typedef struct {
dictDefragAllocFunction *defragAlloc; /* Used for entries etc. */
dictDefragAllocFunction *defragKey; /* Defrag-realloc keys (optional) */
dictDefragAllocFunction *defragVal; /* Defrag-realloc values (optional) */
} dictDefragFunctions;
/* This is the initial size of every hash table */ /* This is the initial size of every hash table */
#define DICT_HT_INITIAL_EXP 2 #define DICT_HT_INITIAL_EXP 2
...@@ -152,6 +170,8 @@ int dictTryExpand(dict *d, unsigned long size); ...@@ -152,6 +170,8 @@ int dictTryExpand(dict *d, unsigned long size);
void *dictMetadata(dict *d); void *dictMetadata(dict *d);
int dictAdd(dict *d, void *key, void *val); int dictAdd(dict *d, void *key, void *val);
dictEntry *dictAddRaw(dict *d, void *key, dictEntry **existing); dictEntry *dictAddRaw(dict *d, void *key, dictEntry **existing);
void *dictFindPositionForInsert(dict *d, const void *key, dictEntry **existing);
dictEntry *dictInsertAtPosition(dict *d, void *key, void *position);
dictEntry *dictAddOrFind(dict *d, void *key); dictEntry *dictAddOrFind(dict *d, void *key);
int dictReplace(dict *d, void *key, void *val); int dictReplace(dict *d, void *key, void *val);
int dictDelete(dict *d, const void *key); int dictDelete(dict *d, const void *key);
...@@ -200,7 +220,7 @@ int dictRehashMilliseconds(dict *d, int ms); ...@@ -200,7 +220,7 @@ int dictRehashMilliseconds(dict *d, int ms);
void dictSetHashFunctionSeed(uint8_t *seed); void dictSetHashFunctionSeed(uint8_t *seed);
uint8_t *dictGetHashFunctionSeed(void); uint8_t *dictGetHashFunctionSeed(void);
unsigned long dictScan(dict *d, unsigned long v, dictScanFunction *fn, void *privdata); unsigned long dictScan(dict *d, unsigned long v, dictScanFunction *fn, void *privdata);
unsigned long dictScanDefrag(dict *d, unsigned long v, dictScanFunction *fn, dictDefragAllocFunction *allocfn, void *privdata); unsigned long dictScanDefrag(dict *d, unsigned long v, dictScanFunction *fn, dictDefragFunctions *defragfns, void *privdata);
uint64_t dictGetHash(dict *d, const void *key); uint64_t dictGetHash(dict *d, const void *key);
dictEntry *dictFindEntryByPtrAndHash(dict *d, const void *oldptr, uint64_t hash); dictEntry *dictFindEntryByPtrAndHash(dict *d, const void *oldptr, uint64_t hash);
......
...@@ -448,7 +448,8 @@ dictType setDictType = { ...@@ -448,7 +448,8 @@ dictType setDictType = {
NULL, /* val dup */ NULL, /* val dup */
dictSdsKeyCompare, /* key compare */ dictSdsKeyCompare, /* key compare */
dictSdsDestructor, /* key destructor */ dictSdsDestructor, /* key destructor */
NULL /* val destructor */ .no_value = 1, /* no values in this dict */
.keys_are_odd = 1 /* an SDS string is always an odd pointer */
}; };
/* Sorted sets hash (note: a skiplist is used in addition to the hash table) */ /* Sorted sets hash (note: a skiplist is used in addition to the hash table) */
...@@ -471,9 +472,9 @@ dictType dbDictType = { ...@@ -471,9 +472,9 @@ dictType dbDictType = {
dictSdsDestructor, /* key destructor */ dictSdsDestructor, /* key destructor */
dictObjectDestructor, /* val destructor */ dictObjectDestructor, /* val destructor */
dictExpandAllowed, /* allow to expand */ dictExpandAllowed, /* allow to expand */
dbDictEntryMetadataSize, /* size of entry metadata in bytes */ .dictEntryMetadataBytes = dbDictEntryMetadataSize,
dbDictMetadataSize, /* size of dict metadata in bytes */ .dictMetadataBytes = dbDictMetadataSize,
dbDictAfterReplaceEntry /* notify entry moved/reallocated */ .afterReplaceEntry = dbDictAfterReplaceEntry
}; };
/* Db->expires */ /* Db->expires */
......
...@@ -122,17 +122,16 @@ int setTypeAddAux(robj *set, char *str, size_t len, int64_t llval, int str_is_sd ...@@ -122,17 +122,16 @@ int setTypeAddAux(robj *set, char *str, size_t len, int64_t llval, int str_is_sd
/* Avoid duping the string if it is an sds string. */ /* Avoid duping the string if it is an sds string. */
sds sdsval = str_is_sds ? (sds)str : sdsnewlen(str, len); sds sdsval = str_is_sds ? (sds)str : sdsnewlen(str, len);
dict *ht = set->ptr; dict *ht = set->ptr;
dictEntry *de = dictAddRaw(ht,sdsval,NULL); void *position = dictFindPositionForInsert(ht, sdsval, NULL);
if (de && sdsval == str) { if (position) {
/* String was added but we don't own this sds string. Dup it and /* Key doesn't already exist in the set. Add it but dup the key. */
* replace it in the dict entry. */ if (sdsval == str) sdsval = sdsdup(sdsval);
dictSetKey(ht,de,sdsdup((sds)str)); dictInsertAtPosition(ht, sdsval, position);
dictSetVal(ht,de,NULL); } else if (sdsval != str) {
} else if (!de && sdsval != str) { /* String is already a member. Free our temporary sds copy. */
/* String was already a member. Free our temporary sds copy. */
sdsfree(sdsval); sdsfree(sdsval);
} }
return (de != NULL); return (position != NULL);
} else if (set->encoding == OBJ_ENCODING_LISTPACK) { } else if (set->encoding == OBJ_ENCODING_LISTPACK) {
unsigned char *lp = set->ptr; unsigned char *lp = set->ptr;
unsigned char *p = lpFirst(lp); unsigned char *p = lpFirst(lp);
......
...@@ -60,8 +60,9 @@ void zlibc_free(void *ptr) { ...@@ -60,8 +60,9 @@ void zlibc_free(void *ptr) {
#ifdef HAVE_MALLOC_SIZE #ifdef HAVE_MALLOC_SIZE
#define PREFIX_SIZE (0) #define PREFIX_SIZE (0)
#else #else
#if defined(__sun) || defined(__sparc) || defined(__sparc__) /* Use at least 8 bits alignment on all systems. */
#define PREFIX_SIZE (sizeof(long long)) #if SIZE_MAX < 0xffffffffffffffffull
#define PREFIX_SIZE 8
#else #else
#define PREFIX_SIZE (sizeof(size_t)) #define PREFIX_SIZE (sizeof(size_t))
#endif #endif
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment