Unverified Commit 2ec78d26 authored by Moti Cohen's avatar Moti Cohen Committed by GitHub
Browse files

Add KEYSIZES section to INFO (#13592)

This PR adds a new section to the `INFO` command output, called
`keysizes`. This section provides detailed statistics on the
distribution of key sizes for each data type (strings, lists, sets,
hashes and zsets) within the dataset. The distribution is tracked using
a base-2 logarithmic histogram.

# Motivation
Currently, Redis lacks a built-in feature to track key sizes and item
sizes per data type at a granular level. Understanding the distribution
of key sizes is critical for monitoring memory usage and optimizing
performance, particularly in large datasets. This enhancement will allow
users to inspect the size distribution of keys directly from the `INFO`
command, assisting with performance analysis and capacity planning.

# Changes
New Section in `INFO` Command: A new section called `keysizes` has been
added to the `INFO` command output. This section reports a per-database,
per-type histogram of key sizes. It provides insights into how many keys
fall into specific size ranges (represented in powers of 2).

**Example output:**
```
127.0.0.1:6379> INFO keysizes
# Keysizes
db0_distrib_strings_sizes:1=19,2=655,512=100899,1K=31,2K=29,4K=23,8K=16,16K=3,32K=2
db0_distrib_lists_items:1=5784492,32=3558,64=1047,128=676,256=533,512=218,4K=1,8K=42
db0_distrib_sets_items:1=735564=50612,8=21462,64=1365,128=974,2K=292,4K=154,8K=89,
db0_distrib_hashes_items:2=1,4=544,32=141169,64=207329,128=4349,256=136226,1K=1
```
## Future Use Cases:
The key size distribution is collected per slot as well, laying the
groundwork for future enhancements related to Redis Cluster.
parent 611c9502
...@@ -489,22 +489,27 @@ int getBitfieldTypeFromArgument(client *c, robj *o, int *sign, int *bits) { ...@@ -489,22 +489,27 @@ int getBitfieldTypeFromArgument(client *c, robj *o, int *sign, int *bits) {
* bits to a string object. The command creates or pad with zeroes the string * bits to a string object. The command creates or pad with zeroes the string
* so that the 'maxbit' bit can be addressed. The object is finally * so that the 'maxbit' bit can be addressed. The object is finally
* returned. Otherwise if the key holds a wrong type NULL is returned and * returned. Otherwise if the key holds a wrong type NULL is returned and
* an error is sent to the client. */ * an error is sent to the client.
robj *lookupStringForBitCommand(client *c, uint64_t maxbit, int *dirty) { *
* (Must provide all the arguments to the function)
*/
static robj *lookupStringForBitCommand(client *c, uint64_t maxbit,
size_t *strOldSize, size_t *strGrowSize)
{
size_t byte = maxbit >> 3; size_t byte = maxbit >> 3;
robj *o = lookupKeyWrite(c->db,c->argv[1]); robj *o = lookupKeyWrite(c->db,c->argv[1]);
if (checkType(c,o,OBJ_STRING)) return NULL; if (checkType(c,o,OBJ_STRING)) return NULL;
if (dirty) *dirty = 0;
if (o == NULL) { if (o == NULL) {
o = createObject(OBJ_STRING,sdsnewlen(NULL, byte+1)); o = createObject(OBJ_STRING,sdsnewlen(NULL, byte+1));
dbAdd(c->db,c->argv[1],o); dbAdd(c->db,c->argv[1],o);
if (dirty) *dirty = 1; *strGrowSize = byte + 1;
*strOldSize = 0;
} else { } else {
o = dbUnshareStringValue(c->db,c->argv[1],o); o = dbUnshareStringValue(c->db,c->argv[1],o);
size_t oldlen = sdslen(o->ptr); *strOldSize = sdslen(o->ptr);
o->ptr = sdsgrowzero(o->ptr,byte+1); o->ptr = sdsgrowzero(o->ptr,byte+1);
if (dirty && oldlen != sdslen(o->ptr)) *dirty = 1; *strGrowSize = sdslen(o->ptr) - *strOldSize;
} }
return o; return o;
} }
...@@ -561,8 +566,9 @@ void setbitCommand(client *c) { ...@@ -561,8 +566,9 @@ void setbitCommand(client *c) {
return; return;
} }
int dirty; size_t strOldSize, strGrowSize;
if ((o = lookupStringForBitCommand(c,bitoffset,&dirty)) == NULL) return; if ((o = lookupStringForBitCommand(c,bitoffset,&strOldSize,&strGrowSize)) == NULL)
return;
/* Get current values */ /* Get current values */
byte = bitoffset >> 3; byte = bitoffset >> 3;
...@@ -573,7 +579,7 @@ void setbitCommand(client *c) { ...@@ -573,7 +579,7 @@ void setbitCommand(client *c) {
/* Either it is newly created, changed length, or the bit changes before and after. /* Either it is newly created, changed length, or the bit changes before and after.
* Note that the bitval here is actually a decimal number. * Note that the bitval here is actually a decimal number.
* So we need to use `!!` to convert it to 0 or 1 for comparison. */ * So we need to use `!!` to convert it to 0 or 1 for comparison. */
if (dirty || (!!bitval != on)) { if (strGrowSize || (!!bitval != on)) {
/* Update byte with new bit value. */ /* Update byte with new bit value. */
byteval &= ~(1 << bit); byteval &= ~(1 << bit);
byteval |= ((on & 0x1) << bit); byteval |= ((on & 0x1) << bit);
...@@ -581,6 +587,13 @@ void setbitCommand(client *c) { ...@@ -581,6 +587,13 @@ void setbitCommand(client *c) {
signalModifiedKey(c,c->db,c->argv[1]); signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id);
server.dirty++; server.dirty++;
/* If this is not a new key (old size not 0) and size changed, then
* update the keysizes histogram. Otherwise, the histogram already
* updated in lookupStringForBitCommand() by calling dbAdd(). */
if ((strOldSize > 0) && (strGrowSize != 0))
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_STRING,
strOldSize, strOldSize + strGrowSize);
} }
/* Return original value. */ /* Return original value. */
...@@ -1065,7 +1078,8 @@ struct bitfieldOp { ...@@ -1065,7 +1078,8 @@ struct bitfieldOp {
void bitfieldGeneric(client *c, int flags) { void bitfieldGeneric(client *c, int flags) {
robj *o; robj *o;
uint64_t bitoffset; uint64_t bitoffset;
int j, numops = 0, changes = 0, dirty = 0; int j, numops = 0, changes = 0;
size_t strOldSize, strGrowSize = 0;
struct bitfieldOp *ops = NULL; /* Array of ops to execute at end. */ struct bitfieldOp *ops = NULL; /* Array of ops to execute at end. */
int owtype = BFOVERFLOW_WRAP; /* Overflow type. */ int owtype = BFOVERFLOW_WRAP; /* Overflow type. */
int readonly = 1; int readonly = 1;
...@@ -1159,7 +1173,7 @@ void bitfieldGeneric(client *c, int flags) { ...@@ -1159,7 +1173,7 @@ void bitfieldGeneric(client *c, int flags) {
/* Lookup by making room up to the farthest bit reached by /* Lookup by making room up to the farthest bit reached by
* this operation. */ * this operation. */
if ((o = lookupStringForBitCommand(c, if ((o = lookupStringForBitCommand(c,
highest_write_offset,&dirty)) == NULL) { highest_write_offset,&strOldSize,&strGrowSize)) == NULL) {
zfree(ops); zfree(ops);
return; return;
} }
...@@ -1209,7 +1223,7 @@ void bitfieldGeneric(client *c, int flags) { ...@@ -1209,7 +1223,7 @@ void bitfieldGeneric(client *c, int flags) {
setSignedBitfield(o->ptr,thisop->offset, setSignedBitfield(o->ptr,thisop->offset,
thisop->bits,newval); thisop->bits,newval);
if (dirty || (oldval != newval)) if (strGrowSize || (oldval != newval))
changes++; changes++;
} else { } else {
addReplyNull(c); addReplyNull(c);
...@@ -1243,7 +1257,7 @@ void bitfieldGeneric(client *c, int flags) { ...@@ -1243,7 +1257,7 @@ void bitfieldGeneric(client *c, int flags) {
setUnsignedBitfield(o->ptr,thisop->offset, setUnsignedBitfield(o->ptr,thisop->offset,
thisop->bits,newval); thisop->bits,newval);
if (dirty || (oldval != newval)) if (strGrowSize || (oldval != newval))
changes++; changes++;
} else { } else {
addReplyNull(c); addReplyNull(c);
...@@ -1286,6 +1300,14 @@ void bitfieldGeneric(client *c, int flags) { ...@@ -1286,6 +1300,14 @@ void bitfieldGeneric(client *c, int flags) {
} }
if (changes) { if (changes) {
/* If this is not a new key (old size not 0) and size changed, then
* update the keysizes histogram. Otherwise, the histogram already
* updated in lookupStringForBitCommand() by calling dbAdd(). */
if ((strOldSize > 0) && (strGrowSize != 0))
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_STRING,
strOldSize, strOldSize + strGrowSize);
signalModifiedKey(c,c->db,c->argv[1]); signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id);
server.dirty += changes; server.dirty += changes;
......
...@@ -21,6 +21,8 @@ ...@@ -21,6 +21,8 @@
* C-level DB API * C-level DB API
*----------------------------------------------------------------------------*/ *----------------------------------------------------------------------------*/
static_assert(MAX_KEYSIZES_TYPES == OBJ_TYPE_BASIC_MAX, "Must be equal");
/* Flags for expireIfNeeded */ /* Flags for expireIfNeeded */
#define EXPIRE_FORCE_DELETE_EXPIRED 1 #define EXPIRE_FORCE_DELETE_EXPIRED 1
#define EXPIRE_AVOID_DELETE_EXPIRED 2 #define EXPIRE_AVOID_DELETE_EXPIRED 2
...@@ -46,6 +48,48 @@ void updateLFU(robj *val) { ...@@ -46,6 +48,48 @@ void updateLFU(robj *val) {
val->lru = (LFUGetTimeInMinutes()<<8) | counter; val->lru = (LFUGetTimeInMinutes()<<8) | counter;
} }
/*
* Update histogram of keys-sizes
*
* It is used to track the distribution of key sizes in the dataset. It is updated
* every time key's length is modified. Available to user via INFO command.
*
* The histogram is a base-2 logarithmic histogram, with 64 bins. The i'th bin
* represents the number of keys with a size in the range 2^i and 2^(i+1)
* exclusive. oldLen/newLen must be smaller than 2^48, and if their value
* equals 0, it means that the key is being created/deleted, respectively. Each
* data type has its own histogram and it is per database (In addition, there is
* histogram per slot for future cluster use).
*
* Examples to LEN values and corresponding bins in histogram:
* [1,2)->0 [2,4)->1 [4,8)->2 [8,16)->3
*/
void updateKeysizesHist(redisDb *db, int didx, uint32_t type, uint64_t oldLen, uint64_t newLen) {
if(unlikely(type >= OBJ_TYPE_BASIC_MAX))
return;
kvstoreDictMetadata *dictMeta = kvstoreGetDictMetadata(db->keys, didx);
kvstoreMetadata *kvstoreMeta = kvstoreGetMetadata(db->keys);
if (oldLen != 0) {
int old_bin = log2ceil(oldLen);
debugServerAssertWithInfo(server.current_client, NULL, old_bin < MAX_KEYSIZES_BINS);
/* If following a key deletion it is last one in slot's dict, then
* slot's dict might get released as well. Verify if metadata is not NULL. */
if(dictMeta) dictMeta->keysizes_hist[type][old_bin]--;
kvstoreMeta->keysizes_hist[type][old_bin]--;
}
if (newLen != 0) {
int new_bin = log2ceil(newLen);
debugServerAssertWithInfo(server.current_client, NULL, new_bin < MAX_KEYSIZES_BINS);
/* If following a key deletion it is last one in slot's dict, then
* slot's dict might get released as well. Verify if metadata is not NULL. */
if(dictMeta) dictMeta->keysizes_hist[type][new_bin]++;
kvstoreMeta->keysizes_hist[type][new_bin]++;
}
}
/* Lookup a key for read or write operations, or return NULL if the key is not /* Lookup a key for read or write operations, or return NULL if the key is not
* found in the specified DB. This function implements the functionality of * found in the specified DB. This function implements the functionality of
* lookupKeyRead(), lookupKeyWrite() and their ...WithFlags() variants. * lookupKeyRead(), lookupKeyWrite() and their ...WithFlags() variants.
...@@ -205,6 +249,7 @@ static dictEntry *dbAddInternal(redisDb *db, robj *key, robj *val, int update_if ...@@ -205,6 +249,7 @@ static dictEntry *dbAddInternal(redisDb *db, robj *key, robj *val, int update_if
kvstoreDictSetVal(db->keys, slot, de, val); kvstoreDictSetVal(db->keys, slot, de, val);
signalKeyAsReady(db, key, val->type); signalKeyAsReady(db, key, val->type);
notifyKeyspaceEvent(NOTIFY_NEW,"new",key,db->id); notifyKeyspaceEvent(NOTIFY_NEW,"new",key,db->id);
updateKeysizesHist(db, slot, val->type, 0, getObjectLength(val)); /* add hist */
return de; return de;
} }
...@@ -250,6 +295,7 @@ int dbAddRDBLoad(redisDb *db, sds key, robj *val) { ...@@ -250,6 +295,7 @@ int dbAddRDBLoad(redisDb *db, sds key, robj *val) {
int slot = getKeySlot(key); int slot = getKeySlot(key);
dictEntry *de = kvstoreDictAddRaw(db->keys, slot, key, NULL); dictEntry *de = kvstoreDictAddRaw(db->keys, slot, key, NULL);
if (de == NULL) return 0; if (de == NULL) return 0;
updateKeysizesHist(db, slot, val->type, 0, getObjectLength(val)); /* add hist */
initObjectLRUOrLFU(val); initObjectLRUOrLFU(val);
kvstoreDictSetVal(db->keys, slot, de, val); kvstoreDictSetVal(db->keys, slot, de, val);
return 1; return 1;
...@@ -273,6 +319,9 @@ static void dbSetValue(redisDb *db, robj *key, robj *val, int overwrite, dictEnt ...@@ -273,6 +319,9 @@ static void dbSetValue(redisDb *db, robj *key, robj *val, int overwrite, dictEnt
serverAssertWithInfo(NULL,key,de != NULL); serverAssertWithInfo(NULL,key,de != NULL);
robj *old = dictGetVal(de); robj *old = dictGetVal(de);
/* Remove old key from keysizes histogram */
updateKeysizesHist(db, slot, old->type, getObjectLength(old), 0); /* remove hist */
val->lru = old->lru; val->lru = old->lru;
if (overwrite) { if (overwrite) {
...@@ -291,6 +340,9 @@ static void dbSetValue(redisDb *db, robj *key, robj *val, int overwrite, dictEnt ...@@ -291,6 +340,9 @@ static void dbSetValue(redisDb *db, robj *key, robj *val, int overwrite, dictEnt
} }
kvstoreDictSetVal(db->keys, slot, de, val); kvstoreDictSetVal(db->keys, slot, de, val);
/* Add new key to keysizes histogram */
updateKeysizesHist(db, slot, val->type, 0, getObjectLength(val));
/* if hash with HFEs, take care to remove from global HFE DS */ /* if hash with HFEs, take care to remove from global HFE DS */
if (old->type == OBJ_HASH) if (old->type == OBJ_HASH)
hashTypeRemoveFromExpires(&db->hexpires, old); hashTypeRemoveFromExpires(&db->hexpires, old);
...@@ -404,6 +456,9 @@ int dbGenericDelete(redisDb *db, robj *key, int async, int flags) { ...@@ -404,6 +456,9 @@ int dbGenericDelete(redisDb *db, robj *key, int async, int flags) {
if (de) { if (de) {
robj *val = dictGetVal(de); robj *val = dictGetVal(de);
/* remove key from histogram */
updateKeysizesHist(db, slot, val->type, getObjectLength(val), 0);
/* If hash object with expiry on fields, remove it from HFE DS of DB */ /* If hash object with expiry on fields, remove it from HFE DS of DB */
if (val->type == OBJ_HASH) if (val->type == OBJ_HASH)
hashTypeRemoveFromExpires(&db->hexpires, val); hashTypeRemoveFromExpires(&db->hexpires, val);
...@@ -599,7 +654,8 @@ redisDb *initTempDb(void) { ...@@ -599,7 +654,8 @@ redisDb *initTempDb(void) {
redisDb *tempDb = zcalloc(sizeof(redisDb)*server.dbnum); redisDb *tempDb = zcalloc(sizeof(redisDb)*server.dbnum);
for (int i=0; i<server.dbnum; i++) { for (int i=0; i<server.dbnum; i++) {
tempDb[i].id = i; tempDb[i].id = i;
tempDb[i].keys = kvstoreCreate(&dbDictType, slot_count_bits, flags); tempDb[i].keys = kvstoreCreate(&dbDictType, slot_count_bits,
flags | KVSTORE_ALLOC_META_KEYS_HIST);
tempDb[i].expires = kvstoreCreate(&dbExpiresDictType, slot_count_bits, flags); tempDb[i].expires = kvstoreCreate(&dbExpiresDictType, slot_count_bits, flags);
tempDb[i].hexpires = ebCreate(); tempDb[i].hexpires = ebCreate();
} }
......
...@@ -42,6 +42,7 @@ struct _kvstore { ...@@ -42,6 +42,7 @@ struct _kvstore {
unsigned long long *dict_size_index; /* Binary indexed tree (BIT) that describes cumulative key frequencies up until given dict-index. */ unsigned long long *dict_size_index; /* Binary indexed tree (BIT) that describes cumulative key frequencies up until given dict-index. */
size_t overhead_hashtable_lut; /* The overhead of all dictionaries. */ size_t overhead_hashtable_lut; /* The overhead of all dictionaries. */
size_t overhead_hashtable_rehashing; /* The overhead of dictionaries rehashing. */ size_t overhead_hashtable_rehashing; /* The overhead of dictionaries rehashing. */
void *metadata[]; /* conditionally allocated based on "flags" */
}; };
/* Structure for kvstore iterator that allows iterating across multiple dicts. */ /* Structure for kvstore iterator that allows iterating across multiple dicts. */
...@@ -59,10 +60,17 @@ struct _kvstoreDictIterator { ...@@ -59,10 +60,17 @@ struct _kvstoreDictIterator {
dictIterator di; dictIterator di;
}; };
/* Dict metadata for database, used for record the position in rehashing list. */ /* Basic metadata allocated per dict */
typedef struct { typedef struct {
listNode *rehashing_node; /* list node in rehashing list */ listNode *rehashing_node; /* list node in rehashing list */
} kvstoreDictMetadata; } kvstoreDictMetaBase;
/* Conditionally metadata allocated per dict (specifically for keysizes histogram) */
typedef struct {
kvstoreDictMetaBase base; /* must be first in struct ! */
/* External metadata */
kvstoreDictMetadata meta;
} kvstoreDictMetaEx;
/**********************************/ /**********************************/
/*** Helpers **********************/ /*** Helpers **********************/
...@@ -184,7 +192,7 @@ static void freeDictIfNeeded(kvstore *kvs, int didx) { ...@@ -184,7 +192,7 @@ static void freeDictIfNeeded(kvstore *kvs, int didx) {
* If there's one dict, bucket count can be retrieved directly from single dict bucket. */ * If there's one dict, bucket count can be retrieved directly from single dict bucket. */
static void kvstoreDictRehashingStarted(dict *d) { static void kvstoreDictRehashingStarted(dict *d) {
kvstore *kvs = d->type->userdata; kvstore *kvs = d->type->userdata;
kvstoreDictMetadata *metadata = (kvstoreDictMetadata *)dictMetadata(d); kvstoreDictMetaBase *metadata = (kvstoreDictMetaBase *)dictMetadata(d);
listAddNodeTail(kvs->rehashing, d); listAddNodeTail(kvs->rehashing, d);
metadata->rehashing_node = listLast(kvs->rehashing); metadata->rehashing_node = listLast(kvs->rehashing);
...@@ -201,7 +209,7 @@ static void kvstoreDictRehashingStarted(dict *d) { ...@@ -201,7 +209,7 @@ static void kvstoreDictRehashingStarted(dict *d) {
* the old ht size of the dictionary from the total sum of buckets for a DB. */ * the old ht size of the dictionary from the total sum of buckets for a DB. */
static void kvstoreDictRehashingCompleted(dict *d) { static void kvstoreDictRehashingCompleted(dict *d) {
kvstore *kvs = d->type->userdata; kvstore *kvs = d->type->userdata;
kvstoreDictMetadata *metadata = (kvstoreDictMetadata *)dictMetadata(d); kvstoreDictMetaBase *metadata = (kvstoreDictMetaBase *)dictMetadata(d);
if (metadata->rehashing_node) { if (metadata->rehashing_node) {
listDelNode(kvs->rehashing, metadata->rehashing_node); listDelNode(kvs->rehashing, metadata->rehashing_node);
metadata->rehashing_node = NULL; metadata->rehashing_node = NULL;
...@@ -214,10 +222,15 @@ static void kvstoreDictRehashingCompleted(dict *d) { ...@@ -214,10 +222,15 @@ static void kvstoreDictRehashingCompleted(dict *d) {
kvs->overhead_hashtable_rehashing -= from; kvs->overhead_hashtable_rehashing -= from;
} }
/* Returns the size of the DB dict metadata in bytes. */ /* Returns the size of the DB dict base metadata in bytes. */
static size_t kvstoreDictMetadataSize(dict *d) { static size_t kvstoreDictMetaBaseSize(dict *d) {
UNUSED(d);
return sizeof(kvstoreDictMetaBase);
}
/* Returns the size of the DB dict extended metadata in bytes. */
static size_t kvstoreDictMetadataExtendSize(dict *d) {
UNUSED(d); UNUSED(d);
return sizeof(kvstoreDictMetadata); return sizeof(kvstoreDictMetaEx);
} }
/**********************************/ /**********************************/
...@@ -232,7 +245,13 @@ kvstore *kvstoreCreate(dictType *type, int num_dicts_bits, int flags) { ...@@ -232,7 +245,13 @@ kvstore *kvstoreCreate(dictType *type, int num_dicts_bits, int flags) {
* for the dict cursor, see kvstoreScan */ * for the dict cursor, see kvstoreScan */
assert(num_dicts_bits <= 16); assert(num_dicts_bits <= 16);
kvstore *kvs = zcalloc(sizeof(*kvs)); /* Calc kvstore size */
size_t kvsize = sizeof(kvstore);
/* Conditionally calc also histogram size */
if (flags & KVSTORE_ALLOC_META_KEYS_HIST)
kvsize += sizeof(kvstoreMetadata);
kvstore *kvs = zcalloc(kvsize);
memcpy(&kvs->dtype, type, sizeof(kvs->dtype)); memcpy(&kvs->dtype, type, sizeof(kvs->dtype));
kvs->flags = flags; kvs->flags = flags;
...@@ -243,7 +262,10 @@ kvstore *kvstoreCreate(dictType *type, int num_dicts_bits, int flags) { ...@@ -243,7 +262,10 @@ kvstore *kvstoreCreate(dictType *type, int num_dicts_bits, int flags) {
assert(!type->rehashingStarted); assert(!type->rehashingStarted);
assert(!type->rehashingCompleted); assert(!type->rehashingCompleted);
kvs->dtype.userdata = kvs; kvs->dtype.userdata = kvs;
kvs->dtype.dictMetadataBytes = kvstoreDictMetadataSize; if (flags & KVSTORE_ALLOC_META_KEYS_HIST)
kvs->dtype.dictMetadataBytes = kvstoreDictMetadataExtendSize;
else
kvs->dtype.dictMetadataBytes = kvstoreDictMetaBaseSize;
kvs->dtype.rehashingStarted = kvstoreDictRehashingStarted; kvs->dtype.rehashingStarted = kvstoreDictRehashingStarted;
kvs->dtype.rehashingCompleted = kvstoreDictRehashingCompleted; kvs->dtype.rehashingCompleted = kvstoreDictRehashingCompleted;
...@@ -263,7 +285,6 @@ kvstore *kvstoreCreate(dictType *type, int num_dicts_bits, int flags) { ...@@ -263,7 +285,6 @@ kvstore *kvstoreCreate(dictType *type, int num_dicts_bits, int flags) {
kvs->bucket_count = 0; kvs->bucket_count = 0;
kvs->overhead_hashtable_lut = 0; kvs->overhead_hashtable_lut = 0;
kvs->overhead_hashtable_rehashing = 0; kvs->overhead_hashtable_rehashing = 0;
return kvs; return kvs;
} }
...@@ -272,9 +293,13 @@ void kvstoreEmpty(kvstore *kvs, void(callback)(dict*)) { ...@@ -272,9 +293,13 @@ void kvstoreEmpty(kvstore *kvs, void(callback)(dict*)) {
dict *d = kvstoreGetDict(kvs, didx); dict *d = kvstoreGetDict(kvs, didx);
if (!d) if (!d)
continue; continue;
kvstoreDictMetadata *metadata = (kvstoreDictMetadata *)dictMetadata(d); kvstoreDictMetaBase *metadata = (kvstoreDictMetaBase *)dictMetadata(d);
if (metadata->rehashing_node) if (metadata->rehashing_node)
metadata->rehashing_node = NULL; metadata->rehashing_node = NULL;
if (kvs->flags & KVSTORE_ALLOC_META_KEYS_HIST) {
kvstoreDictMetaEx *metaExt = (kvstoreDictMetaEx *) metadata;
memset(&metaExt->meta.keysizes_hist, 0, sizeof(metaExt->meta.keysizes_hist));
}
dictEmpty(d, callback); dictEmpty(d, callback);
freeDictIfNeeded(kvs, didx); freeDictIfNeeded(kvs, didx);
} }
...@@ -296,7 +321,7 @@ void kvstoreRelease(kvstore *kvs) { ...@@ -296,7 +321,7 @@ void kvstoreRelease(kvstore *kvs) {
dict *d = kvstoreGetDict(kvs, didx); dict *d = kvstoreGetDict(kvs, didx);
if (!d) if (!d)
continue; continue;
kvstoreDictMetadata *metadata = (kvstoreDictMetadata *)dictMetadata(d); kvstoreDictMetaBase *metadata = (kvstoreDictMetaBase *)dictMetadata(d);
if (metadata->rehashing_node) if (metadata->rehashing_node)
metadata->rehashing_node = NULL; metadata->rehashing_node = NULL;
dictRelease(d); dictRelease(d);
...@@ -330,11 +355,15 @@ unsigned long kvstoreBuckets(kvstore *kvs) { ...@@ -330,11 +355,15 @@ unsigned long kvstoreBuckets(kvstore *kvs) {
size_t kvstoreMemUsage(kvstore *kvs) { size_t kvstoreMemUsage(kvstore *kvs) {
size_t mem = sizeof(*kvs); size_t mem = sizeof(*kvs);
size_t metaSize = sizeof(kvstoreDictMetaBase);
if (kvs->flags & KVSTORE_ALLOC_META_KEYS_HIST)
metaSize = sizeof(kvstoreDictMetaEx);
unsigned long long keys_count = kvstoreSize(kvs); unsigned long long keys_count = kvstoreSize(kvs);
mem += keys_count * dictEntryMemUsage() + mem += keys_count * dictEntryMemUsage() +
kvstoreBuckets(kvs) * sizeof(dictEntry*) + kvstoreBuckets(kvs) * sizeof(dictEntry*) +
kvs->allocated_dicts * (sizeof(dict) + kvstoreDictMetadataSize(NULL)); kvs->allocated_dicts * (sizeof(dict) + metaSize);
/* Values are dict* shared with kvs->dicts */ /* Values are dict* shared with kvs->dicts */
mem += listLength(kvs->rehashing) * sizeof(listNode); mem += listLength(kvs->rehashing) * sizeof(listNode);
...@@ -785,7 +814,7 @@ void kvstoreDictLUTDefrag(kvstore *kvs, kvstoreDictLUTDefragFunction *defragfn) ...@@ -785,7 +814,7 @@ void kvstoreDictLUTDefrag(kvstore *kvs, kvstoreDictLUTDefragFunction *defragfn)
/* After defragmenting the dict, update its corresponding /* After defragmenting the dict, update its corresponding
* rehashing node in the kvstore's rehashing list. */ * rehashing node in the kvstore's rehashing list. */
kvstoreDictMetadata *metadata = (kvstoreDictMetadata *)dictMetadata(*d); kvstoreDictMetaBase *metadata = (kvstoreDictMetaBase *)dictMetadata(*d);
if (metadata->rehashing_node) if (metadata->rehashing_node)
metadata->rehashing_node->value = *d; metadata->rehashing_node->value = *d;
} }
...@@ -856,6 +885,19 @@ int kvstoreDictDelete(kvstore *kvs, int didx, const void *key) { ...@@ -856,6 +885,19 @@ int kvstoreDictDelete(kvstore *kvs, int didx, const void *key) {
return ret; return ret;
} }
kvstoreDictMetadata *kvstoreGetDictMetadata(kvstore *kvs, int didx) {
dict *d = kvstoreGetDict(kvs, didx);
if ((!d) || (!(kvs->flags & KVSTORE_ALLOC_META_KEYS_HIST)))
return NULL;
kvstoreDictMetaEx *metadata = (kvstoreDictMetaEx *)dictMetadata(d);
return &(metadata->meta);
}
kvstoreMetadata *kvstoreGetMetadata(kvstore *kvs) {
return (kvstoreMetadata *) &kvs->metadata;
}
#ifdef REDIS_TEST #ifdef REDIS_TEST
#include <stdio.h> #include <stdio.h>
#include "testhelp.h" #include "testhelp.h"
...@@ -1029,7 +1071,8 @@ int kvstoreTest(int argc, char **argv, int flags) { ...@@ -1029,7 +1071,8 @@ int kvstoreTest(int argc, char **argv, int flags) {
} }
TEST("Verify non-empty dict count is correctly updated") { TEST("Verify non-empty dict count is correctly updated") {
kvstore *kvs = kvstoreCreate(&KvstoreDictTestType, 2, KVSTORE_ALLOCATE_DICTS_ON_DEMAND); kvstore *kvs = kvstoreCreate(&KvstoreDictTestType, 2,
KVSTORE_ALLOCATE_DICTS_ON_DEMAND | KVSTORE_ALLOC_META_KEYS_HIST);
for (int idx = 0; idx < 4; idx++) { for (int idx = 0; idx < 4; idx++) {
for (i = 0; i < 16; i++) { for (i = 0; i < 16; i++) {
de = kvstoreDictAddRaw(kvs, idx, stringFromInt(i), NULL); de = kvstoreDictAddRaw(kvs, idx, stringFromInt(i), NULL);
......
...@@ -4,6 +4,21 @@ ...@@ -4,6 +4,21 @@
#include "dict.h" #include "dict.h"
#include "adlist.h" #include "adlist.h"
/* maximum number of bins of keysizes histogram */
#define MAX_KEYSIZES_BINS 48
#define MAX_KEYSIZES_TYPES 5 /* static_assert at db.c verifies == OBJ_TYPE_BASIC_MAX */
/* When creating kvstore with flag `KVSTORE_ALLOC_META_KEYS_HIST`, then kvstore
* alloc and memset struct kvstoreMetadata on init, yet, managed outside kvstore */
typedef struct {
uint64_t keysizes_hist[MAX_KEYSIZES_TYPES][MAX_KEYSIZES_BINS];
} kvstoreMetadata;
/* Like kvstoreMetadata, this one per dict */
typedef struct {
uint64_t keysizes_hist[MAX_KEYSIZES_TYPES][MAX_KEYSIZES_BINS];
} kvstoreDictMetadata;
typedef struct _kvstore kvstore; typedef struct _kvstore kvstore;
typedef struct _kvstoreIterator kvstoreIterator; typedef struct _kvstoreIterator kvstoreIterator;
typedef struct _kvstoreDictIterator kvstoreDictIterator; typedef struct _kvstoreDictIterator kvstoreDictIterator;
...@@ -13,6 +28,7 @@ typedef int (kvstoreExpandShouldSkipDictIndex)(int didx); ...@@ -13,6 +28,7 @@ typedef int (kvstoreExpandShouldSkipDictIndex)(int didx);
#define KVSTORE_ALLOCATE_DICTS_ON_DEMAND (1<<0) #define KVSTORE_ALLOCATE_DICTS_ON_DEMAND (1<<0)
#define KVSTORE_FREE_EMPTY_DICTS (1<<1) #define KVSTORE_FREE_EMPTY_DICTS (1<<1)
#define KVSTORE_ALLOC_META_KEYS_HIST (1<<2) /* Alloc keysizes histogram */
kvstore *kvstoreCreate(dictType *type, int num_dicts_bits, int flags); kvstore *kvstoreCreate(dictType *type, int num_dicts_bits, int flags);
void kvstoreEmpty(kvstore *kvs, void(callback)(dict*)); void kvstoreEmpty(kvstore *kvs, void(callback)(dict*));
void kvstoreRelease(kvstore *kvs); void kvstoreRelease(kvstore *kvs);
...@@ -71,6 +87,8 @@ void kvstoreDictSetVal(kvstore *kvs, int didx, dictEntry *de, void *val); ...@@ -71,6 +87,8 @@ void kvstoreDictSetVal(kvstore *kvs, int didx, dictEntry *de, void *val);
dictEntry *kvstoreDictTwoPhaseUnlinkFind(kvstore *kvs, int didx, const void *key, dictEntry ***plink, int *table_index); dictEntry *kvstoreDictTwoPhaseUnlinkFind(kvstore *kvs, int didx, const void *key, dictEntry ***plink, int *table_index);
void kvstoreDictTwoPhaseUnlinkFree(kvstore *kvs, int didx, dictEntry *he, dictEntry **plink, int table_index); void kvstoreDictTwoPhaseUnlinkFree(kvstore *kvs, int didx, dictEntry *he, dictEntry **plink, int table_index);
int kvstoreDictDelete(kvstore *kvs, int didx, const void *key); int kvstoreDictDelete(kvstore *kvs, int didx, const void *key);
kvstoreDictMetadata *kvstoreGetDictMetadata(kvstore *kvs, int didx);
kvstoreMetadata *kvstoreGetMetadata(kvstore *kvs);
#ifdef REDIS_TEST #ifdef REDIS_TEST
int kvstoreTest(int argc, char *argv[], int flags); int kvstoreTest(int argc, char *argv[], int flags);
......
...@@ -207,7 +207,7 @@ void emptyDbAsync(redisDb *db) { ...@@ -207,7 +207,7 @@ void emptyDbAsync(redisDb *db) {
} }
kvstore *oldkeys = db->keys, *oldexpires = db->expires; kvstore *oldkeys = db->keys, *oldexpires = db->expires;
ebuckets oldHfe = db->hexpires; ebuckets oldHfe = db->hexpires;
db->keys = kvstoreCreate(&dbDictType, slot_count_bits, flags); db->keys = kvstoreCreate(&dbDictType, slot_count_bits, flags | KVSTORE_ALLOC_META_KEYS_HIST);
db->expires = kvstoreCreate(&dbExpiresDictType, slot_count_bits, flags); db->expires = kvstoreCreate(&dbExpiresDictType, slot_count_bits, flags);
db->hexpires = ebCreate(); db->hexpires = ebCreate();
atomicIncr(lazyfree_objects, kvstoreSize(oldkeys)); atomicIncr(lazyfree_objects, kvstoreSize(oldkeys));
......
...@@ -4171,15 +4171,7 @@ int RM_KeyType(RedisModuleKey *key) { ...@@ -4171,15 +4171,7 @@ int RM_KeyType(RedisModuleKey *key) {
* If the key pointer is NULL or the key is empty, zero is returned. */ * If the key pointer is NULL or the key is empty, zero is returned. */
size_t RM_ValueLength(RedisModuleKey *key) { size_t RM_ValueLength(RedisModuleKey *key) {
if (key == NULL || key->value == NULL) return 0; if (key == NULL || key->value == NULL) return 0;
switch(key->value->type) { return getObjectLength(key->value);
case OBJ_STRING: return stringObjectLen(key->value);
case OBJ_LIST: return listTypeLength(key->value);
case OBJ_SET: return setTypeSize(key->value);
case OBJ_ZSET: return zsetLength(key->value);
case OBJ_HASH: return hashTypeLength(key->value, 0); /* OPEN: To subtract expired fields? */
case OBJ_STREAM: return streamLength(key->value);
default: return 0;
}
} }
   
/* If the key is open for writing, remove it, and setup the key to /* If the key is open for writing, remove it, and setup the key to
......
...@@ -680,6 +680,18 @@ robj *tryObjectEncoding(robj *o) { ...@@ -680,6 +680,18 @@ robj *tryObjectEncoding(robj *o) {
return tryObjectEncodingEx(o, 1); return tryObjectEncodingEx(o, 1);
} }
size_t getObjectLength(robj *o) {
switch(o->type) {
case OBJ_STRING: return stringObjectLen(o);
case OBJ_LIST: return listTypeLength(o);
case OBJ_SET: return setTypeSize(o);
case OBJ_ZSET: return zsetLength(o);
case OBJ_HASH: return hashTypeLength(o, 0);
case OBJ_STREAM: return streamLength(o);
default: return 0;
}
}
/* Get a decoded version of an encoded object (returned as a new object). /* Get a decoded version of an encoded object (returned as a new object).
* If the object is already raw-encoded just increment the ref count. */ * If the object is already raw-encoded just increment the ref count. */
robj *getDecodedObject(robj *o) { robj *getDecodedObject(robj *o) {
......
...@@ -2690,7 +2690,7 @@ void initServer(void) { ...@@ -2690,7 +2690,7 @@ void initServer(void) {
flags |= KVSTORE_FREE_EMPTY_DICTS; flags |= KVSTORE_FREE_EMPTY_DICTS;
} }
for (j = 0; j < server.dbnum; j++) { for (j = 0; j < server.dbnum; j++) {
server.db[j].keys = kvstoreCreate(&dbDictType, slot_count_bits, flags); server.db[j].keys = kvstoreCreate(&dbDictType, slot_count_bits, flags | KVSTORE_ALLOC_META_KEYS_HIST);
server.db[j].expires = kvstoreCreate(&dbExpiresDictType, slot_count_bits, flags); server.db[j].expires = kvstoreCreate(&dbExpiresDictType, slot_count_bits, flags);
server.db[j].hexpires = ebCreate(); server.db[j].hexpires = ebCreate();
server.db[j].expires_cursor = 0; server.db[j].expires_cursor = 0;
...@@ -5521,7 +5521,7 @@ void releaseInfoSectionDict(dict *sec) { ...@@ -5521,7 +5521,7 @@ void releaseInfoSectionDict(dict *sec) {
dict *genInfoSectionDict(robj **argv, int argc, char **defaults, int *out_all, int *out_everything) { dict *genInfoSectionDict(robj **argv, int argc, char **defaults, int *out_all, int *out_everything) {
char *default_sections[] = { char *default_sections[] = {
"server", "clients", "memory", "persistence", "stats", "replication", "server", "clients", "memory", "persistence", "stats", "replication",
"cpu", "module_list", "errorstats", "cluster", "keyspace", NULL}; "cpu", "module_list", "errorstats", "cluster", "keyspace", "keysizes", NULL};
if (!defaults) if (!defaults)
defaults = default_sections; defaults = default_sections;
...@@ -6149,6 +6149,60 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) { ...@@ -6149,6 +6149,60 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) {
} }
} }
/* keysizes */
if (all_sections || (dictFind(section_dict,"keysizes") != NULL)) {
if (sections++) info = sdscat(info,"\r\n");
info = sdscatprintf(info, "# Keysizes\r\n");
char *typestr[] = {
[OBJ_STRING] = "distrib_strings_sizes",
[OBJ_LIST] = "distrib_lists_items",
[OBJ_SET] = "distrib_sets_items",
[OBJ_ZSET] = "distrib_zsets_items",
[OBJ_HASH] = "distrib_hashes_items"
};
serverAssert(sizeof(typestr)/sizeof(typestr[0]) == OBJ_TYPE_BASIC_MAX);
for (int dbnum = 0; dbnum < server.dbnum; dbnum++) {
char *expSizeLabels[] = {
"1", "2", "4", "8", "16", "32", "64", "128", "256", "512", /* Byte */
"1K", "2K", "4K", "8K", "16K", "32K", "64K", "128K", "256K", "512K", /* Kilo */
"1M", "2M", "4M", "8M", "16M", "32M", "64M", "128M", "256M", "512M", /* Mega */
"1G", "2G", "4G", "8G", "16G", "32G", "64G", "128G", "256G", "512G", /* Giga */
"1T", "2T", "4T", "8T", "16T", "32T", "64T", "128T", "256T", "512T", /* Tera */
"1P", "2P", "4P", "8P", "16P", "32P", "64P", "128P", "256P", "512P", /* Peta */
"1E", "2E", "4E", "8E" /* Exa */
};
if (kvstoreSize(server.db[dbnum].keys) == 0)
continue;
for (int type = 0; type < OBJ_TYPE_BASIC_MAX; type++) {
uint64_t *kvstoreHist = kvstoreGetMetadata(server.db[dbnum].keys)->keysizes_hist[type];
char buf[10000];
int cnt = 0, buflen = 0;
/* Print histogram to temp buf[]. First bin is garbage */
buflen += snprintf(buf + buflen, sizeof(buf) - buflen, "db%d_%s:", dbnum, typestr[type]);
for (int i = 0; i < MAX_KEYSIZES_BINS; i++) {
if (kvstoreHist[i] == 0)
continue;
int res = snprintf(buf + buflen, sizeof(buf) - buflen,
(cnt == 0) ? "%s=%llu" : ",%s=%llu",
expSizeLabels[i], (unsigned long long) kvstoreHist[i]);
if (res < 0) break;
buflen += res;
cnt += kvstoreHist[i];
}
/* Print the temp buf[] to the info string */
if (cnt) info = sdscatprintf(info, "%s\r\n", buf);
}
}
}
/* Get info from modules. /* Get info from modules.
* Returned when the user asked for "everything", "modules", or a specific module section. * Returned when the user asked for "everything", "modules", or a specific module section.
* We're not aware of the module section names here, and we rather avoid the search when we can. * We're not aware of the module section names here, and we rather avoid the search when we can.
......
...@@ -41,10 +41,6 @@ ...@@ -41,10 +41,6 @@
#include <systemd/sd-daemon.h> #include <systemd/sd-daemon.h>
#endif #endif
#ifndef static_assert
#define static_assert(expr, lit) extern char __static_assert_failure[(expr) ? 1:-1]
#endif
typedef long long mstime_t; /* millisecond time type. */ typedef long long mstime_t; /* millisecond time type. */
typedef long long ustime_t; /* microsecond time type. */ typedef long long ustime_t; /* microsecond time type. */
...@@ -698,6 +694,7 @@ typedef enum { ...@@ -698,6 +694,7 @@ typedef enum {
#define OBJ_SET 2 /* Set object. */ #define OBJ_SET 2 /* Set object. */
#define OBJ_ZSET 3 /* Sorted set object. */ #define OBJ_ZSET 3 /* Sorted set object. */
#define OBJ_HASH 4 /* Hash object. */ #define OBJ_HASH 4 /* Hash object. */
#define OBJ_TYPE_BASIC_MAX 5 /* Max number of basic object types. */
/* The "module" object type is a special one that signals that the object /* The "module" object type is a special one that signals that the object
* is one directly managed by a Redis module. In this case the value points * is one directly managed by a Redis module. In this case the value points
...@@ -969,7 +966,7 @@ typedef struct replBufBlock { ...@@ -969,7 +966,7 @@ typedef struct replBufBlock {
* by integers from 0 (the default database) up to the max configured * by integers from 0 (the default database) up to the max configured
* database. The database number is the 'id' field in the structure. */ * database. The database number is the 'id' field in the structure. */
typedef struct redisDb { typedef struct redisDb {
kvstore *keys; /* The keyspace for this DB */ kvstore *keys; /* The keyspace for this DB. As metadata, holds keysizes histogram */
kvstore *expires; /* Timeout of keys with a timeout set */ kvstore *expires; /* Timeout of keys with a timeout set */
ebuckets hexpires; /* Hash expiration DS. Single TTL per hash (of next min field to expire) */ ebuckets hexpires; /* Hash expiration DS. Single TTL per hash (of next min field to expire) */
dict *blocking_keys; /* Keys with clients waiting for data (BLPOP)*/ dict *blocking_keys; /* Keys with clients waiting for data (BLPOP)*/
...@@ -2799,6 +2796,7 @@ int isSdsRepresentableAsLongLong(sds s, long long *llval); ...@@ -2799,6 +2796,7 @@ int isSdsRepresentableAsLongLong(sds s, long long *llval);
int isObjectRepresentableAsLongLong(robj *o, long long *llongval); int isObjectRepresentableAsLongLong(robj *o, long long *llongval);
robj *tryObjectEncoding(robj *o); robj *tryObjectEncoding(robj *o);
robj *tryObjectEncodingEx(robj *o, int try_trim); robj *tryObjectEncodingEx(robj *o, int try_trim);
size_t getObjectLength(robj *o);
robj *getDecodedObject(robj *o); robj *getDecodedObject(robj *o);
size_t stringObjectLen(robj *o); size_t stringObjectLen(robj *o);
robj *createStringObjectFromLongLong(long long value); robj *createStringObjectFromLongLong(long long value);
...@@ -3363,6 +3361,7 @@ long long getModuleNumericConfig(ModuleConfig *module_config); ...@@ -3363,6 +3361,7 @@ long long getModuleNumericConfig(ModuleConfig *module_config);
int setModuleNumericConfig(ModuleConfig *config, long long val, const char **err); int setModuleNumericConfig(ModuleConfig *config, long long val, const char **err);
/* db.c -- Keyspace access API */ /* db.c -- Keyspace access API */
void updateKeysizesHist(redisDb *db, int didx, uint32_t type, uint64_t oldLen, uint64_t newLen);
int removeExpire(redisDb *db, robj *key); int removeExpire(redisDb *db, robj *key);
void deleteExpiredKeyAndPropagate(redisDb *db, robj *keyobj); void deleteExpiredKeyAndPropagate(redisDb *db, robj *keyobj);
void deleteEvictedKeyAndPropagate(redisDb *db, robj *keyobj, long long *key_mem_freed); void deleteEvictedKeyAndPropagate(redisDb *db, robj *keyobj, long long *key_mem_freed);
......
...@@ -422,8 +422,13 @@ void listpackExExpire(redisDb *db, robj *o, ExpireInfo *info) { ...@@ -422,8 +422,13 @@ void listpackExExpire(redisDb *db, robj *o, ExpireInfo *info) {
expired++; expired++;
} }
if (expired) if (expired) {
lpt->lp = lpDeleteRange(lpt->lp, 0, expired * 3); lpt->lp = lpDeleteRange(lpt->lp, 0, expired * 3);
/* update keysizes */
unsigned long l = lpLength(lpt->lp) / 3;
updateKeysizesHist(db, getKeySlot(lpt->key), OBJ_HASH, l + expired, l);
}
min = hashTypeGetMinExpire(o, 1 /*accurate*/); min = hashTypeGetMinExpire(o, 1 /*accurate*/);
info->nextExpireTime = min; info->nextExpireTime = min;
...@@ -546,6 +551,11 @@ SetExRes hashTypeSetExpiryListpack(HashTypeSetEx *ex, sds field, ...@@ -546,6 +551,11 @@ SetExRes hashTypeSetExpiryListpack(HashTypeSetEx *ex, sds field,
if (unlikely(checkAlreadyExpired(expireAt))) { if (unlikely(checkAlreadyExpired(expireAt))) {
propagateHashFieldDeletion(ex->db, ex->key->ptr, field, sdslen(field)); propagateHashFieldDeletion(ex->db, ex->key->ptr, field, sdslen(field));
hashTypeDelete(ex->hashObj, field, 1); hashTypeDelete(ex->hashObj, field, 1);
/* get listpack length */
listpackEx *lpt = ((listpackEx *) ex->hashObj->ptr);
unsigned long length = lpLength(lpt->lp) / 3;
updateKeysizesHist(ex->db, getKeySlot(ex->key->ptr), OBJ_HASH, length+1, length);
server.stat_expired_subkeys++; server.stat_expired_subkeys++;
ex->fieldDeleted++; ex->fieldDeleted++;
return HSETEX_DELETED; return HSETEX_DELETED;
...@@ -1042,6 +1052,8 @@ SetExRes hashTypeSetExpiryHT(HashTypeSetEx *exInfo, sds field, uint64_t expireAt ...@@ -1042,6 +1052,8 @@ SetExRes hashTypeSetExpiryHT(HashTypeSetEx *exInfo, sds field, uint64_t expireAt
/* If expired, then delete the field and propagate the deletion. /* If expired, then delete the field and propagate the deletion.
* If replica, continue like the field is valid */ * If replica, continue like the field is valid */
if (unlikely(checkAlreadyExpired(expireAt))) { if (unlikely(checkAlreadyExpired(expireAt))) {
unsigned long length = dictSize(ht);
updateKeysizesHist(exInfo->db, getKeySlot(exInfo->key->ptr), OBJ_HASH, length, length-1);
/* replicas should not initiate deletion of fields */ /* replicas should not initiate deletion of fields */
propagateHashFieldDeletion(exInfo->db, exInfo->key->ptr, field, sdslen(field)); propagateHashFieldDeletion(exInfo->db, exInfo->key->ptr, field, sdslen(field));
hashTypeDelete(exInfo->hashObj, field, 1); hashTypeDelete(exInfo->hashObj, field, 1);
...@@ -2132,6 +2144,7 @@ ebuckets *hashTypeGetDictMetaHFE(dict *d) { ...@@ -2132,6 +2144,7 @@ ebuckets *hashTypeGetDictMetaHFE(dict *d) {
*----------------------------------------------------------------------------*/ *----------------------------------------------------------------------------*/
void hsetnxCommand(client *c) { void hsetnxCommand(client *c) {
unsigned long hlen;
int isHashDeleted; int isHashDeleted;
robj *o; robj *o;
if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return; if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return;
...@@ -2152,6 +2165,8 @@ void hsetnxCommand(client *c) { ...@@ -2152,6 +2165,8 @@ void hsetnxCommand(client *c) {
addReply(c, shared.cone); addReply(c, shared.cone);
signalModifiedKey(c,c->db,c->argv[1]); signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id);
hlen = hashTypeLength(o, 0);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, hlen - 1, hlen);
server.dirty++; server.dirty++;
} }
...@@ -2180,6 +2195,8 @@ void hsetCommand(client *c) { ...@@ -2180,6 +2195,8 @@ void hsetCommand(client *c) {
addReply(c, shared.ok); addReply(c, shared.ok);
} }
signalModifiedKey(c,c->db,c->argv[1]); signalModifiedKey(c,c->db,c->argv[1]);
unsigned long l = hashTypeLength(o, 0);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, l - created, l);
notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id);
server.dirty += (c->argc - 2)/2; server.dirty += (c->argc - 2)/2;
} }
...@@ -2205,11 +2222,14 @@ void hincrbyCommand(client *c) { ...@@ -2205,11 +2222,14 @@ void hincrbyCommand(client *c) {
} /* Else hashTypeGetValue() already stored it into &value */ } /* Else hashTypeGetValue() already stored it into &value */
} else if ((res == GETF_NOT_FOUND) || (res == GETF_EXPIRED)) { } else if ((res == GETF_NOT_FOUND) || (res == GETF_EXPIRED)) {
value = 0; value = 0;
unsigned long l = hashTypeLength(o, 0);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, l, l + 1);
} else { } else {
/* Field expired and in turn hash deleted. Create new one! */ /* Field expired and in turn hash deleted. Create new one! */
o = createHashObject(); o = createHashObject();
dbAdd(c->db,c->argv[1],o); dbAdd(c->db,c->argv[1],o);
value = 0; value = 0;
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, 0, 1);
} }
oldvalue = value; oldvalue = value;
...@@ -2254,11 +2274,14 @@ void hincrbyfloatCommand(client *c) { ...@@ -2254,11 +2274,14 @@ void hincrbyfloatCommand(client *c) {
} }
} else if ((res == GETF_NOT_FOUND) || (res == GETF_EXPIRED)) { } else if ((res == GETF_NOT_FOUND) || (res == GETF_EXPIRED)) {
value = 0; value = 0;
unsigned long l = hashTypeLength(o, 0);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, l, l + 1);
} else { } else {
/* Field expired and in turn hash deleted. Create new one! */ /* Field expired and in turn hash deleted. Create new one! */
o = createHashObject(); o = createHashObject();
dbAdd(c->db,c->argv[1],o); dbAdd(c->db,c->argv[1],o);
value = 0; value = 0;
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, 0, 1);
} }
value += incr; value += incr;
...@@ -2356,6 +2379,8 @@ void hdelCommand(client *c) { ...@@ -2356,6 +2379,8 @@ void hdelCommand(client *c) {
if ((o = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL || if ((o = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL ||
checkType(c,o,OBJ_HASH)) return; checkType(c,o,OBJ_HASH)) return;
unsigned long oldLen = hashTypeLength(o, 0);
/* Hash field expiration is optimized to avoid frequent update global HFE DS for /* Hash field expiration is optimized to avoid frequent update global HFE DS for
* each field deletion. Eventually active-expiration will run and update or remove * each field deletion. Eventually active-expiration will run and update or remove
* the hash from global HFE DS gracefully. Nevertheless, statistic "subexpiry" * the hash from global HFE DS gracefully. Nevertheless, statistic "subexpiry"
...@@ -2375,6 +2400,8 @@ void hdelCommand(client *c) { ...@@ -2375,6 +2400,8 @@ void hdelCommand(client *c) {
} }
} }
if (deleted) { if (deleted) {
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_HASH, oldLen, oldLen - deleted);
signalModifiedKey(c,c->db,c->argv[1]); signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_HASH,"hdel",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_HASH,"hdel",c->argv[1],c->db->id);
if (keyremoved) { if (keyremoved) {
...@@ -2943,6 +2970,11 @@ static ExpireAction onFieldExpire(eItem item, void *ctx) { ...@@ -2943,6 +2970,11 @@ static ExpireAction onFieldExpire(eItem item, void *ctx) {
dict *d = expCtx->hashObj->ptr; dict *d = expCtx->hashObj->ptr;
dictExpireMetadata *dictExpireMeta = (dictExpireMetadata *) dictMetadata(d); dictExpireMetadata *dictExpireMeta = (dictExpireMetadata *) dictMetadata(d);
propagateHashFieldDeletion(expCtx->db, dictExpireMeta->key, hf, hfieldlen(hf)); propagateHashFieldDeletion(expCtx->db, dictExpireMeta->key, hf, hfieldlen(hf));
/* update keysizes */
unsigned long l = hashTypeLength(expCtx->hashObj, 0);
updateKeysizesHist(expCtx->db, getKeySlot(dictExpireMeta->key), OBJ_HASH, l, l - 1);
serverAssert(hashTypeDelete(expCtx->hashObj, hf, 0) == 1); serverAssert(hashTypeDelete(expCtx->hashObj, hf, 0) == 1);
server.stat_expired_subkeys++; server.stat_expired_subkeys++;
return ACT_REMOVE_EXP_ITEM; return ACT_REMOVE_EXP_ITEM;
......
...@@ -7,6 +7,7 @@ ...@@ -7,6 +7,7 @@
*/ */
#include "server.h" #include "server.h"
#include "util.h"
/*----------------------------------------------------------------------------- /*-----------------------------------------------------------------------------
* List API * List API
...@@ -462,6 +463,7 @@ void listTypeDelRange(robj *subject, long start, long count) { ...@@ -462,6 +463,7 @@ void listTypeDelRange(robj *subject, long start, long count) {
/* Implements LPUSH/RPUSH/LPUSHX/RPUSHX. /* Implements LPUSH/RPUSH/LPUSHX/RPUSHX.
* 'xx': push if key exists. */ * 'xx': push if key exists. */
void pushGenericCommand(client *c, int where, int xx) { void pushGenericCommand(client *c, int where, int xx) {
unsigned long llen;
int j; int j;
robj *lobj = lookupKeyWrite(c->db, c->argv[1]); robj *lobj = lookupKeyWrite(c->db, c->argv[1]);
...@@ -482,11 +484,13 @@ void pushGenericCommand(client *c, int where, int xx) { ...@@ -482,11 +484,13 @@ void pushGenericCommand(client *c, int where, int xx) {
server.dirty++; server.dirty++;
} }
addReplyLongLong(c, listTypeLength(lobj)); llen = listTypeLength(lobj);
addReplyLongLong(c, llen);
char *event = (where == LIST_HEAD) ? "lpush" : "rpush"; char *event = (where == LIST_HEAD) ? "lpush" : "rpush";
signalModifiedKey(c,c->db,c->argv[1]); signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_LIST,event,c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_LIST,event,c->argv[1],c->db->id);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_LIST, llen - (c->argc - 2), llen);
} }
/* LPUSH <key> <element> [<element> ...] */ /* LPUSH <key> <element> [<element> ...] */
...@@ -553,6 +557,8 @@ void linsertCommand(client *c) { ...@@ -553,6 +557,8 @@ void linsertCommand(client *c) {
notifyKeyspaceEvent(NOTIFY_LIST,"linsert", notifyKeyspaceEvent(NOTIFY_LIST,"linsert",
c->argv[1],c->db->id); c->argv[1],c->db->id);
server.dirty++; server.dirty++;
unsigned long ll = listTypeLength(subject);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_LIST, ll-1, ll);
} else { } else {
/* Notify client of a failed insert */ /* Notify client of a failed insert */
addReplyLongLong(c,-1); addReplyLongLong(c,-1);
...@@ -736,9 +742,11 @@ void addListRangeReply(client *c, robj *o, long start, long end, int reverse) { ...@@ -736,9 +742,11 @@ void addListRangeReply(client *c, robj *o, long start, long end, int reverse) {
* if the key got deleted by this function. */ * if the key got deleted by this function. */
void listElementsRemoved(client *c, robj *key, int where, robj *o, long count, int signal, int *deleted) { void listElementsRemoved(client *c, robj *key, int where, robj *o, long count, int signal, int *deleted) {
char *event = (where == LIST_HEAD) ? "lpop" : "rpop"; char *event = (where == LIST_HEAD) ? "lpop" : "rpop";
unsigned long llen = listTypeLength(o);
notifyKeyspaceEvent(NOTIFY_LIST, event, key, c->db->id); notifyKeyspaceEvent(NOTIFY_LIST, event, key, c->db->id);
if (listTypeLength(o) == 0) { updateKeysizesHist(c->db, getKeySlot(key->ptr), OBJ_LIST, llen + count, llen);
if (llen == 0) {
if (deleted) *deleted = 1; if (deleted) *deleted = 1;
dbDelete(c->db, key); dbDelete(c->db, key);
...@@ -870,7 +878,7 @@ void lrangeCommand(client *c) { ...@@ -870,7 +878,7 @@ void lrangeCommand(client *c) {
/* LTRIM <key> <start> <stop> */ /* LTRIM <key> <start> <stop> */
void ltrimCommand(client *c) { void ltrimCommand(client *c) {
robj *o; robj *o;
long start, end, llen, ltrim, rtrim; long start, end, llen, ltrim, rtrim, llenNew;;
if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != C_OK) || if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != C_OK) ||
(getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != C_OK)) return; (getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != C_OK)) return;
...@@ -908,12 +916,13 @@ void ltrimCommand(client *c) { ...@@ -908,12 +916,13 @@ void ltrimCommand(client *c) {
} }
notifyKeyspaceEvent(NOTIFY_LIST,"ltrim",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_LIST,"ltrim",c->argv[1],c->db->id);
if (listTypeLength(o) == 0) { if ((llenNew = listTypeLength(o)) == 0) {
dbDelete(c->db,c->argv[1]); dbDelete(c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id);
} else { } else {
listTypeTryConversion(o,LIST_CONV_SHRINKING,NULL,NULL); listTypeTryConversion(o,LIST_CONV_SHRINKING,NULL,NULL);
} }
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_LIST, llen, llenNew);
signalModifiedKey(c,c->db,c->argv[1]); signalModifiedKey(c,c->db,c->argv[1]);
server.dirty += (ltrim + rtrim); server.dirty += (ltrim + rtrim);
addReply(c,shared.ok); addReply(c,shared.ok);
...@@ -1066,8 +1075,11 @@ void lremCommand(client *c) { ...@@ -1066,8 +1075,11 @@ void lremCommand(client *c) {
listTypeReleaseIterator(li); listTypeReleaseIterator(li);
if (removed) { if (removed) {
long ll = listTypeLength(subject);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_LIST, ll + removed, ll);
notifyKeyspaceEvent(NOTIFY_LIST,"lrem",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_LIST,"lrem",c->argv[1],c->db->id);
if (listTypeLength(subject) == 0) {
if (ll == 0) {
dbDelete(c->db,c->argv[1]); dbDelete(c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id);
} else { } else {
...@@ -1089,6 +1101,10 @@ void lmoveHandlePush(client *c, robj *dstkey, robj *dstobj, robj *value, ...@@ -1089,6 +1101,10 @@ void lmoveHandlePush(client *c, robj *dstkey, robj *dstobj, robj *value,
listTypeTryConversionAppend(dstobj,&value,0,0,NULL,NULL); listTypeTryConversionAppend(dstobj,&value,0,0,NULL,NULL);
listTypePush(dstobj,value,where); listTypePush(dstobj,value,where);
signalModifiedKey(c,c->db,dstkey); signalModifiedKey(c,c->db,dstkey);
long ll = listTypeLength(dstobj);
updateKeysizesHist(c->db, getKeySlot(dstkey->ptr), OBJ_LIST, ll - 1, ll);
notifyKeyspaceEvent(NOTIFY_LIST, notifyKeyspaceEvent(NOTIFY_LIST,
where == LIST_HEAD ? "lpush" : "rpush", where == LIST_HEAD ? "lpush" : "rpush",
dstkey, dstkey,
......
...@@ -603,6 +603,8 @@ void saddCommand(client *c) { ...@@ -603,6 +603,8 @@ void saddCommand(client *c) {
if (setTypeAdd(set,c->argv[j]->ptr)) added++; if (setTypeAdd(set,c->argv[j]->ptr)) added++;
} }
if (added) { if (added) {
unsigned long size = setTypeSize(set);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_SET, size - added, size);
signalModifiedKey(c,c->db,c->argv[1]); signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_SET,"sadd",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_SET,"sadd",c->argv[1],c->db->id);
} }
...@@ -617,6 +619,8 @@ void sremCommand(client *c) { ...@@ -617,6 +619,8 @@ void sremCommand(client *c) {
if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL || if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL ||
checkType(c,set,OBJ_SET)) return; checkType(c,set,OBJ_SET)) return;
unsigned long oldSize = setTypeSize(set);
for (j = 2; j < c->argc; j++) { for (j = 2; j < c->argc; j++) {
if (setTypeRemove(set,c->argv[j]->ptr)) { if (setTypeRemove(set,c->argv[j]->ptr)) {
deleted++; deleted++;
...@@ -628,6 +632,8 @@ void sremCommand(client *c) { ...@@ -628,6 +632,8 @@ void sremCommand(client *c) {
} }
} }
if (deleted) { if (deleted) {
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_SET, oldSize, oldSize - deleted);
signalModifiedKey(c,c->db,c->argv[1]); signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_SET,"srem",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_SET,"srem",c->argv[1],c->db->id);
if (keyremoved) if (keyremoved)
...@@ -669,8 +675,12 @@ void smoveCommand(client *c) { ...@@ -669,8 +675,12 @@ void smoveCommand(client *c) {
} }
notifyKeyspaceEvent(NOTIFY_SET,"srem",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_SET,"srem",c->argv[1],c->db->id);
/* Update keysizes histogram */
unsigned long srcLen = setTypeSize(srcset);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_SET, srcLen + 1, srcLen);
/* Remove the src set from the database when empty */ /* Remove the src set from the database when empty */
if (setTypeSize(srcset) == 0) { if (srcLen == 0) {
dbDelete(c->db,c->argv[1]); dbDelete(c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",c->argv[1],c->db->id);
} }
...@@ -686,6 +696,8 @@ void smoveCommand(client *c) { ...@@ -686,6 +696,8 @@ void smoveCommand(client *c) {
/* An extra key has changed when ele was successfully added to dstset */ /* An extra key has changed when ele was successfully added to dstset */
if (setTypeAdd(dstset,ele->ptr)) { if (setTypeAdd(dstset,ele->ptr)) {
unsigned long dstLen = setTypeSize(dstset);
updateKeysizesHist(c->db, getKeySlot(c->argv[2]->ptr), OBJ_SET, dstLen - 1, dstLen);
server.dirty++; server.dirty++;
signalModifiedKey(c,c->db,c->argv[2]); signalModifiedKey(c,c->db,c->argv[2]);
notifyKeyspaceEvent(NOTIFY_SET,"sadd",c->argv[2],c->db->id); notifyKeyspaceEvent(NOTIFY_SET,"sadd",c->argv[2],c->db->id);
...@@ -743,7 +755,7 @@ void scardCommand(client *c) { ...@@ -743,7 +755,7 @@ void scardCommand(client *c) {
void spopWithCountCommand(client *c) { void spopWithCountCommand(client *c) {
long l; long l;
unsigned long count, size; unsigned long count, size, toRemove;
robj *set; robj *set;
/* Get the count argument */ /* Get the count argument */
...@@ -763,10 +775,12 @@ void spopWithCountCommand(client *c) { ...@@ -763,10 +775,12 @@ void spopWithCountCommand(client *c) {
} }
size = setTypeSize(set); size = setTypeSize(set);
toRemove = (count >= size) ? size : count;
/* Generate an SPOP keyspace notification */ /* Generate an SPOP keyspace notification */
notifyKeyspaceEvent(NOTIFY_SET,"spop",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_SET,"spop",c->argv[1],c->db->id);
server.dirty += (count >= size) ? size : count; server.dirty += toRemove;
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_SET, size, size - toRemove);
/* CASE 1: /* CASE 1:
* The number of requested elements is greater than or equal to * The number of requested elements is greater than or equal to
...@@ -949,6 +963,7 @@ void spopWithCountCommand(client *c) { ...@@ -949,6 +963,7 @@ void spopWithCountCommand(client *c) {
} }
void spopCommand(client *c) { void spopCommand(client *c) {
unsigned long size;
robj *set, *ele; robj *set, *ele;
if (c->argc == 3) { if (c->argc == 3) {
...@@ -964,6 +979,9 @@ void spopCommand(client *c) { ...@@ -964,6 +979,9 @@ void spopCommand(client *c) {
if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp])) if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp]))
== NULL || checkType(c,set,OBJ_SET)) return; == NULL || checkType(c,set,OBJ_SET)) return;
size = setTypeSize(set);
updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_SET, size, size-1);
/* Pop a random element from the set */ /* Pop a random element from the set */
ele = setTypePopRandom(set); ele = setTypePopRandom(set);
......
...@@ -420,6 +420,7 @@ void getsetCommand(client *c) { ...@@ -420,6 +420,7 @@ void getsetCommand(client *c) {
} }
void setrangeCommand(client *c) { void setrangeCommand(client *c) {
size_t oldLen = 0, newLen;
robj *o; robj *o;
long offset; long offset;
sds value = c->argv[3]->ptr; sds value = c->argv[3]->ptr;
...@@ -449,16 +450,14 @@ void setrangeCommand(client *c) { ...@@ -449,16 +450,14 @@ void setrangeCommand(client *c) {
o = createObject(OBJ_STRING,sdsnewlen(NULL, offset+value_len)); o = createObject(OBJ_STRING,sdsnewlen(NULL, offset+value_len));
dbAdd(c->db,c->argv[1],o); dbAdd(c->db,c->argv[1],o);
} else { } else {
size_t olen;
/* Key exists, check type */ /* Key exists, check type */
if (checkType(c,o,OBJ_STRING)) if (checkType(c,o,OBJ_STRING))
return; return;
/* Return existing string length when setting nothing */ /* Return existing string length when setting nothing */
olen = stringObjectLen(o); oldLen = stringObjectLen(o);
if (value_len == 0) { if (value_len == 0) {
addReplyLongLong(c,olen); addReplyLongLong(c, oldLen);
return; return;
} }
...@@ -478,7 +477,10 @@ void setrangeCommand(client *c) { ...@@ -478,7 +477,10 @@ void setrangeCommand(client *c) {
"setrange",c->argv[1],c->db->id); "setrange",c->argv[1],c->db->id);
server.dirty++; server.dirty++;
} }
addReplyLongLong(c,sdslen(o->ptr));
newLen = sdslen(o->ptr);
updateKeysizesHist(c->db,getKeySlot(c->argv[1]->ptr),OBJ_STRING,oldLen,newLen);
addReplyLongLong(c,newLen);
} }
void getrangeCommand(client *c) { void getrangeCommand(client *c) {
...@@ -669,7 +671,7 @@ void incrbyfloatCommand(client *c) { ...@@ -669,7 +671,7 @@ void incrbyfloatCommand(client *c) {
} }
void appendCommand(client *c) { void appendCommand(client *c) {
size_t totlen; size_t totlen, append_len;
robj *o, *append; robj *o, *append;
dictEntry *de; dictEntry *de;
...@@ -679,7 +681,7 @@ void appendCommand(client *c) { ...@@ -679,7 +681,7 @@ void appendCommand(client *c) {
c->argv[2] = tryObjectEncoding(c->argv[2]); c->argv[2] = tryObjectEncoding(c->argv[2]);
dbAdd(c->db,c->argv[1],c->argv[2]); dbAdd(c->db,c->argv[1],c->argv[2]);
incrRefCount(c->argv[2]); incrRefCount(c->argv[2]);
totlen = stringObjectLen(c->argv[2]); append_len = totlen = stringObjectLen(c->argv[2]);
} else { } else {
/* Key exists, check type */ /* Key exists, check type */
if (checkType(c,o,OBJ_STRING)) if (checkType(c,o,OBJ_STRING))
...@@ -687,7 +689,7 @@ void appendCommand(client *c) { ...@@ -687,7 +689,7 @@ void appendCommand(client *c) {
/* "append" is an argument, so always an sds */ /* "append" is an argument, so always an sds */
append = c->argv[2]; append = c->argv[2];
const size_t append_len = sdslen(append->ptr); append_len = sdslen(append->ptr);
if (checkStringLength(c,stringObjectLen(o),append_len) != C_OK) if (checkStringLength(c,stringObjectLen(o),append_len) != C_OK)
return; return;
...@@ -699,6 +701,7 @@ void appendCommand(client *c) { ...@@ -699,6 +701,7 @@ void appendCommand(client *c) {
signalModifiedKey(c,c->db,c->argv[1]); signalModifiedKey(c,c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_STRING,"append",c->argv[1],c->db->id); notifyKeyspaceEvent(NOTIFY_STRING,"append",c->argv[1],c->db->id);
server.dirty++; server.dirty++;
updateKeysizesHist(c->db,getKeySlot(c->argv[1]->ptr),OBJ_STRING, totlen - append_len, totlen);
addReplyLongLong(c,totlen); addReplyLongLong(c,totlen);
} }
......
...@@ -1843,6 +1843,7 @@ void zaddGenericCommand(client *c, int flags) { ...@@ -1843,6 +1843,7 @@ void zaddGenericCommand(client *c, int flags) {
zsetTypeMaybeConvert(zobj, elements); zsetTypeMaybeConvert(zobj, elements);
} }
unsigned long llen = zsetLength(zobj);
for (j = 0; j < elements; j++) { for (j = 0; j < elements; j++) {
double newscore; double newscore;
score = scores[j]; score = scores[j];
...@@ -1860,6 +1861,7 @@ void zaddGenericCommand(client *c, int flags) { ...@@ -1860,6 +1861,7 @@ void zaddGenericCommand(client *c, int flags) {
score = newscore; score = newscore;
} }
server.dirty += (added+updated); server.dirty += (added+updated);
updateKeysizesHist(c->db, getKeySlot(key->ptr), OBJ_ZSET, llen, llen+added);
reply_to_client: reply_to_client:
if (incr) { /* ZINCRBY or INCR option. */ if (incr) { /* ZINCRBY or INCR option. */
...@@ -1907,8 +1909,13 @@ void zremCommand(client *c) { ...@@ -1907,8 +1909,13 @@ void zremCommand(client *c) {
if (deleted) { if (deleted) {
notifyKeyspaceEvent(NOTIFY_ZSET,"zrem",key,c->db->id); notifyKeyspaceEvent(NOTIFY_ZSET,"zrem",key,c->db->id);
if (keyremoved) if (keyremoved) {
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id); notifyKeyspaceEvent(NOTIFY_GENERIC, "del", key, c->db->id);
/* No need updateKeysizesHist(). dbDelete() done it already. */
} else {
unsigned long len = zsetLength(zobj);
updateKeysizesHist(c->db, getKeySlot(key->ptr), OBJ_ZSET, len + deleted, len);
}
signalModifiedKey(c,c->db,key); signalModifiedKey(c,c->db,key);
server.dirty += deleted; server.dirty += deleted;
} }
...@@ -2023,8 +2030,13 @@ void zremrangeGenericCommand(client *c, zrange_type rangetype) { ...@@ -2023,8 +2030,13 @@ void zremrangeGenericCommand(client *c, zrange_type rangetype) {
if (deleted) { if (deleted) {
signalModifiedKey(c,c->db,key); signalModifiedKey(c,c->db,key);
notifyKeyspaceEvent(NOTIFY_ZSET,notify_type,key,c->db->id); notifyKeyspaceEvent(NOTIFY_ZSET,notify_type,key,c->db->id);
if (keyremoved) if (keyremoved) {
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id); notifyKeyspaceEvent(NOTIFY_GENERIC, "del", key, c->db->id);
/* No need updateKeysizesHist(). dbDelete() done it already. */
} else {
unsigned long len = zsetLength(zobj);
updateKeysizesHist(c->db, getKeySlot(key->ptr), OBJ_ZSET, len + deleted, len);
}
} }
server.dirty += deleted; server.dirty += deleted;
addReplyLongLong(c,deleted); addReplyLongLong(c,deleted);
...@@ -4031,6 +4043,9 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey ...@@ -4031,6 +4043,9 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey
dbDelete(c->db,key); dbDelete(c->db,key);
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id); notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
/* No need updateKeysizesHist(). dbDelete() done it already. */
} else {
updateKeysizesHist(c->db, getKeySlot(key->ptr), OBJ_ZSET, llen, llen - result_count);
} }
signalModifiedKey(c,c->db,key); signalModifiedKey(c,c->db,key);
......
...@@ -54,6 +54,13 @@ ...@@ -54,6 +54,13 @@
#define UNUSED(x) ((void)(x)) #define UNUSED(x) ((void)(x))
/* Selectively define static_assert. Attempt to avoid include server.h in this file. */
#ifndef static_assert
#define static_assert(expr, lit) extern char __static_assert_failure[(expr) ? 1:-1]
#endif
static_assert(UINTPTR_MAX == 0xffffffffffffffff || UINTPTR_MAX == 0xffffffff, "Unsupported pointer size");
/* Glob-style pattern matching. */ /* Glob-style pattern matching. */
static int stringmatchlen_impl(const char *pattern, int patternLen, static int stringmatchlen_impl(const char *pattern, int patternLen,
const char *string, int stringLen, int nocase, int *skipLongerMatches, int nesting) const char *string, int stringLen, int nocase, int *skipLongerMatches, int nesting)
......
...@@ -79,6 +79,19 @@ int snprintf_async_signal_safe(char *to, size_t n, const char *fmt, ...); ...@@ -79,6 +79,19 @@ int snprintf_async_signal_safe(char *to, size_t n, const char *fmt, ...);
size_t redis_strlcpy(char *dst, const char *src, size_t dsize); size_t redis_strlcpy(char *dst, const char *src, size_t dsize);
size_t redis_strlcat(char *dst, const char *src, size_t dsize); size_t redis_strlcat(char *dst, const char *src, size_t dsize);
/* to keep it opt without conditions Works only for: 0 < x < 2^63 */
static inline int log2ceil(size_t x) {
#if UINTPTR_MAX == 0xffffffffffffffff
return 63 - __builtin_clzll(x);
#else
return 31 - __builtin_clz(x);
#endif
}
#ifndef static_assert
#define static_assert(expr, lit) extern char __static_assert_failure[(expr) ? 1:-1]
#endif
#ifdef REDIS_TEST #ifdef REDIS_TEST
int utilTest(int argc, char **argv, int flags); int utilTest(int argc, char **argv, int flags);
#endif #endif
......
This diff is collapsed.
...@@ -306,7 +306,7 @@ run_solo {defrag} { ...@@ -306,7 +306,7 @@ run_solo {defrag} {
r set "{bigstream}smallitem" val r set "{bigstream}smallitem" val
set expected_frag 1.7 set expected_frag 1.5
if {$::accurate} { if {$::accurate} {
# scale the hash to 1m fields in order to have a measurable the latency # scale the hash to 1m fields in order to have a measurable the latency
for {set j 10000} {$j < 1000000} {incr j} { for {set j 10000} {$j < 1000000} {incr j} {
...@@ -601,7 +601,7 @@ run_solo {defrag} { ...@@ -601,7 +601,7 @@ run_solo {defrag} {
# create big keys with 10k items # create big keys with 10k items
set rd [redis_deferring_client] set rd [redis_deferring_client]
set expected_frag 1.7 set expected_frag 1.5
# add a mass of list nodes to two lists (allocations are interlaced) # add a mass of list nodes to two lists (allocations are interlaced)
set val [string repeat A 100] ;# 5 items of 100 bytes puts us in the 640 bytes bin, which has 32 regs, so high potential for fragmentation set val [string repeat A 100] ;# 5 items of 100 bytes puts us in the 640 bytes bin, which has 32 regs, so high potential for fragmentation
set elements 500000 set elements 500000
......
...@@ -508,18 +508,25 @@ start_server {tags {"other external:skip"}} { ...@@ -508,18 +508,25 @@ start_server {tags {"other external:skip"}} {
test "Redis can resize empty dict" { test "Redis can resize empty dict" {
# Write and then delete 128 keys, creating an empty dict # Write and then delete 128 keys, creating an empty dict
r flushall r flushall
# Add one key to the db just to create the dict and get its initial size
r set x 1
set initial_size [dict get [r memory stats] db.9 overhead.hashtable.main]
# Now add 128 keys and then delete them
for {set j 1} {$j <= 128} {incr j} { for {set j 1} {$j <= 128} {incr j} {
r set $j{b} a r set $j{b} a
} }
for {set j 1} {$j <= 128} {incr j} { for {set j 1} {$j <= 128} {incr j} {
r del $j{b} r del $j{b}
} }
# The dict containing 128 keys must have expanded,
# its hash table itself takes a lot more than 400 bytes # dict must have expanded. Verify it eventually shrinks back to its initial size.
wait_for_condition 100 50 { wait_for_condition 100 50 {
[dict get [r memory stats] db.9 overhead.hashtable.main] < 400 [dict get [r memory stats] db.9 overhead.hashtable.main] == $initial_size
} else { } else {
fail "dict did not resize in time" fail "dict did not resize in time to its initial size"
} }
} }
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment