Commit 297922e6 authored by Vitaly Arbuzov's avatar Vitaly Arbuzov
Browse files

Add ability to use dbIterator as dictIterator

parent 72935b9d
...@@ -2235,7 +2235,6 @@ werr: ...@@ -2235,7 +2235,6 @@ werr:
} }
int rewriteAppendOnlyFileRio(rio *aof) { int rewriteAppendOnlyFileRio(rio *aof) {
dictIterator *di = NULL;
dictEntry *de; dictEntry *de;
int j; int j;
long key_count = 0; long key_count = 0;
...@@ -2256,86 +2255,78 @@ int rewriteAppendOnlyFileRio(rio *aof) { ...@@ -2256,86 +2255,78 @@ int rewriteAppendOnlyFileRio(rio *aof) {
if (rioWrite(aof, selectcmd, sizeof(selectcmd) - 1) == 0) goto werr; if (rioWrite(aof, selectcmd, sizeof(selectcmd) - 1) == 0) goto werr;
if (rioWriteBulkLongLong(aof, j) == 0) goto werr; if (rioWriteBulkLongLong(aof, j) == 0) goto werr;
redisDb *db = server.db + j; redisDb *db = server.db + j;
dict *d;
dbIterator dbit; dbIterator dbit;
dbIteratorInit(&dbit, db); dbIteratorInit(&dbit, db);
while ((d = dbIteratorNextDict(&dbit))) { /* Iterate this DB writing every entry */
if (dictSize(d) == 0) continue; while ((de = dbIteratorNext(&dbit)) != NULL) {
di = dictGetSafeIterator(d); sds keystr;
/* Iterate this DB writing every entry */ robj key, *o;
while ((de = dictNext(di)) != NULL) { long long expiretime;
sds keystr; size_t aof_bytes_before_key = aof->processed_bytes;
robj key, *o;
long long expiretime; keystr = dictGetKey(de);
size_t aof_bytes_before_key = aof->processed_bytes; o = dictGetVal(de);
initStaticStringObject(key, keystr);
keystr = dictGetKey(de);
o = dictGetVal(de); expiretime = getExpire(db, &key);
initStaticStringObject(key, keystr);
/* Save the key and associated value */
expiretime = getExpire(db, &key); if (o->type == OBJ_STRING) {
/* Emit a SET command */
/* Save the key and associated value */ char cmd[] = "*3\r\n$3\r\nSET\r\n";
if (o->type == OBJ_STRING) { if (rioWrite(aof, cmd, sizeof(cmd) - 1) == 0) goto werr;
/* Emit a SET command */ /* Key and value */
char cmd[] = "*3\r\n$3\r\nSET\r\n"; if (rioWriteBulkObject(aof, &key) == 0) goto werr;
if (rioWrite(aof, cmd, sizeof(cmd) - 1) == 0) goto werr; if (rioWriteBulkObject(aof, o) == 0) goto werr;
/* Key and value */ } else if (o->type == OBJ_LIST) {
if (rioWriteBulkObject(aof, &key) == 0) goto werr; if (rewriteListObject(aof, &key, o) == 0) goto werr;
if (rioWriteBulkObject(aof, o) == 0) goto werr; } else if (o->type == OBJ_SET) {
} else if (o->type == OBJ_LIST) { if (rewriteSetObject(aof, &key, o) == 0) goto werr;
if (rewriteListObject(aof, &key, o) == 0) goto werr; } else if (o->type == OBJ_ZSET) {
} else if (o->type == OBJ_SET) { if (rewriteSortedSetObject(aof, &key, o) == 0) goto werr;
if (rewriteSetObject(aof, &key, o) == 0) goto werr; } else if (o->type == OBJ_HASH) {
} else if (o->type == OBJ_ZSET) { if (rewriteHashObject(aof, &key, o) == 0) goto werr;
if (rewriteSortedSetObject(aof, &key, o) == 0) goto werr; } else if (o->type == OBJ_STREAM) {
} else if (o->type == OBJ_HASH) { if (rewriteStreamObject(aof, &key, o) == 0) goto werr;
if (rewriteHashObject(aof, &key, o) == 0) goto werr; } else if (o->type == OBJ_MODULE) {
} else if (o->type == OBJ_STREAM) { if (rewriteModuleObject(aof, &key, o, j) == 0) goto werr;
if (rewriteStreamObject(aof, &key, o) == 0) goto werr; } else {
} else if (o->type == OBJ_MODULE) { serverPanic("Unknown object type");
if (rewriteModuleObject(aof, &key, o, j) == 0) goto werr; }
} else {
serverPanic("Unknown object type");
}
/* In fork child process, we can try to release memory back to the /* In fork child process, we can try to release memory back to the
* OS and possibly avoid or decrease COW. We give the dismiss * OS and possibly avoid or decrease COW. We give the dismiss
* mechanism a hint about an estimated size of the object we stored. */ * mechanism a hint about an estimated size of the object we stored. */
size_t dump_size = aof->processed_bytes - aof_bytes_before_key; size_t dump_size = aof->processed_bytes - aof_bytes_before_key;
if (server.in_fork_child) dismissObject(o, dump_size); if (server.in_fork_child) dismissObject(o, dump_size);
/* Save the expire time */ /* Save the expire time */
if (expiretime != -1) { if (expiretime != -1) {
char cmd[] = "*3\r\n$9\r\nPEXPIREAT\r\n"; char cmd[] = "*3\r\n$9\r\nPEXPIREAT\r\n";
if (rioWrite(aof, cmd, sizeof(cmd) - 1) == 0) goto werr; if (rioWrite(aof, cmd, sizeof(cmd) - 1) == 0) goto werr;
if (rioWriteBulkObject(aof, &key) == 0) goto werr; if (rioWriteBulkObject(aof, &key) == 0) goto werr;
if (rioWriteBulkLongLong(aof, expiretime) == 0) goto werr; if (rioWriteBulkLongLong(aof, expiretime) == 0) goto werr;
} }
/* Update info every 1 second (approximately). /* Update info every 1 second (approximately).
* in order to avoid calling mstime() on each iteration, we will * in order to avoid calling mstime() on each iteration, we will
* check the diff every 1024 keys */ * check the diff every 1024 keys */
if ((key_count++ & 1023) == 0) { if ((key_count++ & 1023) == 0) {
long long now = mstime(); long long now = mstime();
if (now - updated_time >= 1000) { if (now - updated_time >= 1000) {
sendChildInfo(CHILD_INFO_TYPE_CURRENT_INFO, key_count, "AOF rewrite"); sendChildInfo(CHILD_INFO_TYPE_CURRENT_INFO, key_count, "AOF rewrite");
updated_time = now; updated_time = now;
}
} }
/* Delay before next key if required (for testing) */
if (server.rdb_key_save_delay)
debugDelay(server.rdb_key_save_delay);
} }
dictReleaseIterator(di);
di = NULL; /* Delay before next key if required (for testing) */
if (server.rdb_key_save_delay)
debugDelay(server.rdb_key_save_delay);
} }
} }
return C_OK; return C_OK;
werr: werr:
if (di) dictReleaseIterator(di);
return C_ERR; return C_ERR;
} }
......
...@@ -61,6 +61,23 @@ dict *dbIteratorNextDict(dbIterator *dbit) { ...@@ -61,6 +61,23 @@ dict *dbIteratorNextDict(dbIterator *dbit) {
return dbit->cur_slot >= 0 ? dbit->db->dict[dbit->cur_slot] : NULL; return dbit->cur_slot >= 0 ? dbit->db->dict[dbit->cur_slot] : NULL;
} }
/* Returns next entry from the multi slot db. */
dictEntry *dbIteratorNext(dbIterator *dbit) {
if (!dbit->di.d) { /* No current dict, need to get a new one. */
dict *d = dbIteratorNextDict(dbit);
if (!d) return NULL;
dictInitSafeIterator(&dbit->di, d);
}
dictEntry *de = dictNext(&dbit->di);
if (!de) { /* Reached the end of the dictionary, check if there are more. */
dict *d = dbIteratorNextDict(dbit);
if (!d) return NULL;
dictInitSafeIterator(&dbit->di, d);
return dictNext(&dbit->di);
}
return de;
}
/* Returns DB iterator that can be used to iterate through sub-dictionaries. /* Returns DB iterator that can be used to iterate through sub-dictionaries.
* Primary database contains only one dictionary when node runs without cluster mode, * Primary database contains only one dictionary when node runs without cluster mode,
* or 16k dictionaries (one per slot) when node runs with cluster mode enabled. */ * or 16k dictionaries (one per slot) when node runs with cluster mode enabled. */
...@@ -68,6 +85,7 @@ void dbIteratorInit(dbIterator *dbit, redisDb *db) { ...@@ -68,6 +85,7 @@ void dbIteratorInit(dbIterator *dbit, redisDb *db) {
dbit->db = db; dbit->db = db;
dbit->index = 0; dbit->index = 0;
dbit->cur_slot = -1; dbit->cur_slot = -1;
dictInitSafeIterator(&dbit->di, NULL);
} }
/* Returns next non-empty dictionary strictly after provided slot and updates slot id in the supplied reference. */ /* Returns next non-empty dictionary strictly after provided slot and updates slot id in the supplied reference. */
...@@ -846,34 +864,28 @@ void randomkeyCommand(client *c) { ...@@ -846,34 +864,28 @@ void randomkeyCommand(client *c) {
} }
void keysCommand(client *c) { void keysCommand(client *c) {
dictIterator *di;
dictEntry *de; dictEntry *de;
sds pattern = c->argv[1]->ptr; sds pattern = c->argv[1]->ptr;
int plen = sdslen(pattern), allkeys; int plen = sdslen(pattern), allkeys;
long numkeys = 0; long numkeys = 0;
void *replylen = addReplyDeferredLen(c); void *replylen = addReplyDeferredLen(c);
dict *d;
dbIterator dbit; dbIterator dbit;
dbIteratorInit(&dbit, c->db); dbIteratorInit(&dbit, c->db);
while ((d = dbIteratorNextDict(&dbit))) { allkeys = (pattern[0] == '*' && plen == 1);
if (dictSize(d) == 0) continue; while ((de = dbIteratorNext(&dbit)) != NULL) {
di = dictGetSafeIterator(d); sds key = dictGetKey(de);
allkeys = (pattern[0] == '*' && plen == 1); robj *keyobj;
while((de = dictNext(di)) != NULL) {
sds key = dictGetKey(de); if (allkeys || stringmatchlen(pattern, plen, key, sdslen(key), 0)) {
robj *keyobj; keyobj = createStringObject(key, sdslen(key));
if (!keyIsExpired(c->db, keyobj)) {
if (allkeys || stringmatchlen(pattern,plen,key,sdslen(key),0)) { addReplyBulk(c, keyobj);
keyobj = createStringObject(key,sdslen(key)); numkeys++;
if (!keyIsExpired(c->db,keyobj)) {
addReplyBulk(c,keyobj);
numkeys++;
}
decrRefCount(keyobj);
} }
decrRefCount(keyobj);
}
if (c->flags & CLIENT_CLOSE_ASAP) if (c->flags & CLIENT_CLOSE_ASAP)
break;} break;
dictReleaseIterator(di);
} }
setDeferredArrayLen(c,replylen,numkeys); setDeferredArrayLen(c,replylen,numkeys);
} }
......
...@@ -277,7 +277,6 @@ void xorObjectDigest(redisDb *db, robj *keyobj, unsigned char *digest, robj *o) ...@@ -277,7 +277,6 @@ void xorObjectDigest(redisDb *db, robj *keyobj, unsigned char *digest, robj *o)
* a different digest. */ * a different digest. */
void computeDatasetDigest(unsigned char *final) { void computeDatasetDigest(unsigned char *final) {
unsigned char digest[20]; unsigned char digest[20];
dictIterator *di = NULL;
dictEntry *de; dictEntry *de;
int j; int j;
uint32_t aux; uint32_t aux;
...@@ -286,39 +285,31 @@ void computeDatasetDigest(unsigned char *final) { ...@@ -286,39 +285,31 @@ void computeDatasetDigest(unsigned char *final) {
for (j = 0; j < server.dbnum; j++) { for (j = 0; j < server.dbnum; j++) {
redisDb *db = server.db+j; redisDb *db = server.db+j;
int hasEntries = 0; if (dbSize(db) == 0) continue;
dict *d;
dbIterator dbit; dbIterator dbit;
dbIteratorInit(&dbit, db); dbIteratorInit(&dbit, db);
while ((d = dbIteratorNextDict(&dbit))) {
if (dictSize(d) == 0) continue;
hasEntries = 1;
di = dictGetSafeIterator(d);
/* Iterate this DB writing every entry */ /* hash the DB id, so the same dataset moved in a different DB will lead to a different digest */
while ((de = dictNext(di)) != NULL) { aux = htonl(j);
sds key; mixDigest(final, &aux, sizeof(aux));
robj *keyobj, *o;
memset(digest, 0, 20); /* This key-val digest */ /* Iterate this DB writing every entry */
key = dictGetKey(de); while ((de = dbIteratorNext(&dbit)) != NULL) {
keyobj = createStringObject(key, sdslen(key)); sds key;
robj *keyobj, *o;
mixDigest(digest, key, sdslen(key)); memset(digest, 0, 20); /* This key-val digest */
key = dictGetKey(de);
keyobj = createStringObject(key, sdslen(key));
o = dictGetVal(de); mixDigest(digest, key, sdslen(key));
xorObjectDigest(db, keyobj, digest, o);
/* We can finally xor the key-val digest to the final digest */ o = dictGetVal(de);
xorDigest(final, digest, 20); xorObjectDigest(db, keyobj, digest, o);
decrRefCount(keyobj);
} /* We can finally xor the key-val digest to the final digest */
dictReleaseIterator(di); xorDigest(final, digest, 20);
} decrRefCount(keyobj);
if (hasEntries) {
/* hash the DB id, so the same dataset moved in a different DB will lead to a different digest */
aux = htonl(j);
mixDigest(final, &aux, sizeof(aux));
} }
} }
} }
......
...@@ -2439,12 +2439,13 @@ typedef struct dbIterator { ...@@ -2439,12 +2439,13 @@ typedef struct dbIterator {
redisDb *db; redisDb *db;
int index; int index;
int cur_slot; int cur_slot;
dictIterator di;
} dbIterator; } dbIterator;
/* DB iterator specific functions */ /* DB iterator specific functions */
void dbIteratorInit(dbIterator *dbit, redisDb *db); void dbIteratorInit(dbIterator *dbit, redisDb *db);
dict *dbIteratorNextDict(dbIterator *dbit); dict *dbIteratorNextDict(dbIterator *dbit);
int dbIterCurSlot(dbIterator *dbit); dictEntry *dbIteratorNext(dbIterator *iter);
dict *dbGetNextNonEmptySlot(redisDb *db, int *slot); dict *dbGetNextNonEmptySlot(redisDb *db, int *slot);
/* SCAN specific commands for easy cursor manipulation, shared between main code and modules. */ /* SCAN specific commands for easy cursor manipulation, shared between main code and modules. */
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment