Commit 1ac30300 authored by Guy Korland's avatar Guy Korland
Browse files

Merge branch 'unstable' of github.com:antirez/redis into unstable

parents c1455dc0 673c9d70
...@@ -47,7 +47,7 @@ int je_get_defrag_hint(void* ptr, int *bin_util, int *run_util); ...@@ -47,7 +47,7 @@ int je_get_defrag_hint(void* ptr, int *bin_util, int *run_util);
/* forward declarations*/ /* forward declarations*/
void defragDictBucketCallback(void *privdata, dictEntry **bucketref); void defragDictBucketCallback(void *privdata, dictEntry **bucketref);
dictEntry* replaceSateliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sds newkey, unsigned int hash, long *defragged); dictEntry* replaceSateliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sds newkey, uint64_t hash, long *defragged);
/* Defrag helper for generic allocations. /* Defrag helper for generic allocations.
* *
...@@ -355,7 +355,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) { ...@@ -355,7 +355,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) {
sdsele = ln->value; sdsele = ln->value;
if ((newsds = activeDefragSds(sdsele))) { if ((newsds = activeDefragSds(sdsele))) {
/* When defragging an sds value, we need to update the dict key */ /* When defragging an sds value, we need to update the dict key */
unsigned int hash = dictGetHash(d, sdsele); uint64_t hash = dictGetHash(d, sdsele);
replaceSateliteDictKeyPtrAndOrDefragDictEntry(d, sdsele, newsds, hash, &defragged); replaceSateliteDictKeyPtrAndOrDefragDictEntry(d, sdsele, newsds, hash, &defragged);
ln->value = newsds; ln->value = newsds;
defragged++; defragged++;
...@@ -374,7 +374,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) { ...@@ -374,7 +374,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) {
if ((newele = activeDefragStringOb(ele, &defragged))) if ((newele = activeDefragStringOb(ele, &defragged)))
de->v.val = newele, defragged++; de->v.val = newele, defragged++;
} else if (dict_val_type == DEFRAG_SDS_DICT_VAL_VOID_PTR) { } else if (dict_val_type == DEFRAG_SDS_DICT_VAL_VOID_PTR) {
void *newptr, *ptr = ln->value; void *newptr, *ptr = dictGetVal(de);
if ((newptr = activeDefragAlloc(ptr))) if ((newptr = activeDefragAlloc(ptr)))
ln->value = newptr, defragged++; ln->value = newptr, defragged++;
} }
...@@ -392,7 +392,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) { ...@@ -392,7 +392,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) {
* moved. Return value is the the dictEntry if found, or NULL if not found. * moved. Return value is the the dictEntry if found, or NULL if not found.
* NOTE: this is very ugly code, but it let's us avoid the complication of * NOTE: this is very ugly code, but it let's us avoid the complication of
* doing a scan on another dict. */ * doing a scan on another dict. */
dictEntry* replaceSateliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sds newkey, unsigned int hash, long *defragged) { dictEntry* replaceSateliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sds newkey, uint64_t hash, long *defragged) {
dictEntry **deref = dictFindEntryRefByPtrAndHash(d, oldkey, hash); dictEntry **deref = dictFindEntryRefByPtrAndHash(d, oldkey, hash);
if (deref) { if (deref) {
dictEntry *de = *deref; dictEntry *de = *deref;
...@@ -1039,7 +1039,7 @@ void activeDefragCycle(void) { ...@@ -1039,7 +1039,7 @@ void activeDefragCycle(void) {
mstime_t latency; mstime_t latency;
int quit = 0; int quit = 0;
if (server.aof_child_pid!=-1 || server.rdb_child_pid!=-1) if (hasActiveChildProcess())
return; /* Defragging memory while there's a fork will just do damage. */ return; /* Defragging memory while there's a fork will just do damage. */
/* Once a second, check if we the fragmentation justfies starting a scan /* Once a second, check if we the fragmentation justfies starting a scan
......
...@@ -78,7 +78,7 @@ unsigned int getLRUClock(void) { ...@@ -78,7 +78,7 @@ unsigned int getLRUClock(void) {
unsigned int LRU_CLOCK(void) { unsigned int LRU_CLOCK(void) {
unsigned int lruclock; unsigned int lruclock;
if (1000/server.hz <= LRU_CLOCK_RESOLUTION) { if (1000/server.hz <= LRU_CLOCK_RESOLUTION) {
atomicGet(server.lruclock,lruclock); lruclock = server.lruclock;
} else { } else {
lruclock = getLRUClock(); lruclock = getLRUClock();
} }
...@@ -444,6 +444,7 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev ...@@ -444,6 +444,7 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev
* Otehrwise if we are over the memory limit, but not enough memory * Otehrwise if we are over the memory limit, but not enough memory
* was freed to return back under the limit, the function returns C_ERR. */ * was freed to return back under the limit, the function returns C_ERR. */
int freeMemoryIfNeeded(void) { int freeMemoryIfNeeded(void) {
int keys_freed = 0;
/* By default replicas should ignore maxmemory /* By default replicas should ignore maxmemory
* and just be masters exact copies. */ * and just be masters exact copies. */
if (server.masterhost && server.repl_slave_ignore_maxmemory) return C_OK; if (server.masterhost && server.repl_slave_ignore_maxmemory) return C_OK;
...@@ -467,7 +468,7 @@ int freeMemoryIfNeeded(void) { ...@@ -467,7 +468,7 @@ int freeMemoryIfNeeded(void) {
latencyStartMonitor(latency); latencyStartMonitor(latency);
while (mem_freed < mem_tofree) { while (mem_freed < mem_tofree) {
int j, k, i, keys_freed = 0; int j, k, i;
static unsigned int next_db = 0; static unsigned int next_db = 0;
sds bestkey = NULL; sds bestkey = NULL;
int bestdbid; int bestdbid;
...@@ -598,9 +599,7 @@ int freeMemoryIfNeeded(void) { ...@@ -598,9 +599,7 @@ int freeMemoryIfNeeded(void) {
mem_freed = mem_tofree; mem_freed = mem_tofree;
} }
} }
} } else {
if (!keys_freed) {
latencyEndMonitor(latency); latencyEndMonitor(latency);
latencyAddSampleIfNeeded("eviction-cycle",latency); latencyAddSampleIfNeeded("eviction-cycle",latency);
goto cant_free; /* nothing to free... */ goto cant_free; /* nothing to free... */
......
...@@ -64,6 +64,7 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) { ...@@ -64,6 +64,7 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) {
dbSyncDelete(db,keyobj); dbSyncDelete(db,keyobj);
notifyKeyspaceEvent(NOTIFY_EXPIRED, notifyKeyspaceEvent(NOTIFY_EXPIRED,
"expired",keyobj,db->id); "expired",keyobj,db->id);
trackingInvalidateKey(keyobj);
decrRefCount(keyobj); decrRefCount(keyobj);
server.stat_expiredkeys++; server.stat_expiredkeys++;
return 1; return 1;
......
...@@ -466,7 +466,7 @@ void georadiusGeneric(client *c, int flags) { ...@@ -466,7 +466,7 @@ void georadiusGeneric(client *c, int flags) {
/* Look up the requested zset */ /* Look up the requested zset */
robj *zobj = NULL; robj *zobj = NULL;
if ((zobj = lookupKeyReadOrReply(c, key, shared.null[c->resp])) == NULL || if ((zobj = lookupKeyReadOrReply(c, key, shared.emptyarray)) == NULL ||
checkType(c, zobj, OBJ_ZSET)) { checkType(c, zobj, OBJ_ZSET)) {
return; return;
} }
...@@ -566,7 +566,7 @@ void georadiusGeneric(client *c, int flags) { ...@@ -566,7 +566,7 @@ void georadiusGeneric(client *c, int flags) {
/* If no matching results, the user gets an empty reply. */ /* If no matching results, the user gets an empty reply. */
if (ga->used == 0 && storekey == NULL) { if (ga->used == 0 && storekey == NULL) {
addReplyNull(c); addReply(c,shared.emptyarray);
geoArrayFree(ga); geoArrayFree(ga);
return; return;
} }
...@@ -734,14 +734,14 @@ void geohashCommand(client *c) { ...@@ -734,14 +734,14 @@ void geohashCommand(client *c) {
r[1].max = 90; r[1].max = 90;
geohashEncode(&r[0],&r[1],xy[0],xy[1],26,&hash); geohashEncode(&r[0],&r[1],xy[0],xy[1],26,&hash);
char buf[12]; char buf[11];
int i; int i;
for (i = 0; i < 11; i++) { for (i = 0; i < 10; i++) {
int idx = (hash.bits >> (52-((i+1)*5))) & 0x1f; int idx = (hash.bits >> (52-((i+1)*5))) & 0x1f;
buf[i] = geoalphabet[idx]; buf[i] = geoalphabet[idx];
} }
buf[11] = '\0'; buf[10] = '\0';
addReplyBulkCBuffer(c,buf,11); addReplyBulkCBuffer(c,buf,10);
} }
} }
} }
......
...@@ -700,7 +700,7 @@ int hllSparseSet(robj *o, long index, uint8_t count) { ...@@ -700,7 +700,7 @@ int hllSparseSet(robj *o, long index, uint8_t count) {
p += oplen; p += oplen;
first += span; first += span;
} }
if (span == 0) return -1; /* Invalid format. */ if (span == 0 || p >= end) return -1; /* Invalid format. */
next = HLL_SPARSE_IS_XZERO(p) ? p+2 : p+1; next = HLL_SPARSE_IS_XZERO(p) ? p+2 : p+1;
if (next >= end) next = NULL; if (next >= end) next = NULL;
...@@ -1014,8 +1014,8 @@ uint64_t hllCount(struct hllhdr *hdr, int *invalid) { ...@@ -1014,8 +1014,8 @@ uint64_t hllCount(struct hllhdr *hdr, int *invalid) {
double m = HLL_REGISTERS; double m = HLL_REGISTERS;
double E; double E;
int j; int j;
/* Note that reghisto could be just HLL_Q+1, becuase this is the /* Note that reghisto size could be just HLL_Q+2, becuase HLL_Q+1 is
* maximum frequency of the "000...1" sequence the hash function is * the maximum frequency of the "000...1" sequence the hash function is
* able to return. However it is slow to check for sanity of the * able to return. However it is slow to check for sanity of the
* input: instead we history array at a safe size: overflows will * input: instead we history array at a safe size: overflows will
* just write data to wrong, but correctly allocated, places. */ * just write data to wrong, but correctly allocated, places. */
...@@ -1242,7 +1242,7 @@ void pfcountCommand(client *c) { ...@@ -1242,7 +1242,7 @@ void pfcountCommand(client *c) {
if (o == NULL) continue; /* Assume empty HLL for non existing var.*/ if (o == NULL) continue; /* Assume empty HLL for non existing var.*/
if (isHLLObjectOrReply(c,o) != C_OK) return; if (isHLLObjectOrReply(c,o) != C_OK) return;
/* Merge with this HLL with our 'max' HHL by setting max[i] /* Merge with this HLL with our 'max' HLL by setting max[i]
* to MAX(max[i],hll[i]). */ * to MAX(max[i],hll[i]). */
if (hllMerge(registers,o) == C_ERR) { if (hllMerge(registers,o) == C_ERR) {
addReplySds(c,sdsnew(invalid_hll_err)); addReplySds(c,sdsnew(invalid_hll_err));
...@@ -1329,7 +1329,7 @@ void pfmergeCommand(client *c) { ...@@ -1329,7 +1329,7 @@ void pfmergeCommand(client *c) {
hdr = o->ptr; hdr = o->ptr;
if (hdr->encoding == HLL_DENSE) use_dense = 1; if (hdr->encoding == HLL_DENSE) use_dense = 1;
/* Merge with this HLL with our 'max' HHL by setting max[i] /* Merge with this HLL with our 'max' HLL by setting max[i]
* to MAX(max[i],hll[i]). */ * to MAX(max[i],hll[i]). */
if (hllMerge(max,o) == C_ERR) { if (hllMerge(max,o) == C_ERR) {
addReplySds(c,sdsnew(invalid_hll_err)); addReplySds(c,sdsnew(invalid_hll_err));
......
...@@ -599,7 +599,7 @@ NULL ...@@ -599,7 +599,7 @@ NULL
event = dictGetKey(de); event = dictGetKey(de);
graph = latencyCommandGenSparkeline(event,ts); graph = latencyCommandGenSparkeline(event,ts);
addReplyBulkCString(c,graph); addReplyVerbatim(c,graph,sdslen(graph),"txt");
sdsfree(graph); sdsfree(graph);
} else if (!strcasecmp(c->argv[1]->ptr,"latest") && c->argc == 2) { } else if (!strcasecmp(c->argv[1]->ptr,"latest") && c->argc == 2) {
/* LATENCY LATEST */ /* LATENCY LATEST */
...@@ -608,7 +608,7 @@ NULL ...@@ -608,7 +608,7 @@ NULL
/* LATENCY DOCTOR */ /* LATENCY DOCTOR */
sds report = createLatencyReport(); sds report = createLatencyReport();
addReplyBulkCBuffer(c,report,sdslen(report)); addReplyVerbatim(c,report,sdslen(report),"txt");
sdsfree(report); sdsfree(report);
} else if (!strcasecmp(c->argv[1]->ptr,"reset") && c->argc >= 2) { } else if (!strcasecmp(c->argv[1]->ptr,"reset") && c->argc >= 2) {
/* LATENCY RESET */ /* LATENCY RESET */
......
...@@ -34,8 +34,11 @@ ...@@ -34,8 +34,11 @@
*/ */
#include "server.h" #include "server.h"
#include "lolwut.h"
#include <math.h>
void lolwut5Command(client *c); void lolwut5Command(client *c);
void lolwut6Command(client *c);
/* The default target for LOLWUT if no matching version was found. /* The default target for LOLWUT if no matching version was found.
* This is what unstable versions of Redis will display. */ * This is what unstable versions of Redis will display. */
...@@ -43,14 +46,143 @@ void lolwutUnstableCommand(client *c) { ...@@ -43,14 +46,143 @@ void lolwutUnstableCommand(client *c) {
sds rendered = sdsnew("Redis ver. "); sds rendered = sdsnew("Redis ver. ");
rendered = sdscat(rendered,REDIS_VERSION); rendered = sdscat(rendered,REDIS_VERSION);
rendered = sdscatlen(rendered,"\n",1); rendered = sdscatlen(rendered,"\n",1);
addReplyBulkSds(c,rendered); addReplyVerbatim(c,rendered,sdslen(rendered),"txt");
sdsfree(rendered);
} }
/* LOLWUT [VERSION <version>] [... version specific arguments ...] */
void lolwutCommand(client *c) { void lolwutCommand(client *c) {
char *v = REDIS_VERSION; char *v = REDIS_VERSION;
if ((v[0] == '5' && v[1] == '.') || char verstr[64];
if (c->argc >= 3 && !strcasecmp(c->argv[1]->ptr,"version")) {
long ver;
if (getLongFromObjectOrReply(c,c->argv[2],&ver,NULL) != C_OK) return;
snprintf(verstr,sizeof(verstr),"%u.0.0",(unsigned int)ver);
v = verstr;
/* Adjust argv/argc to filter the "VERSION ..." option, since the
* specific LOLWUT version implementations don't know about it
* and expect their arguments. */
c->argv += 2;
c->argc -= 2;
}
if ((v[0] == '5' && v[1] == '.' && v[2] != '9') ||
(v[0] == '4' && v[1] == '.' && v[2] == '9')) (v[0] == '4' && v[1] == '.' && v[2] == '9'))
lolwut5Command(c); lolwut5Command(c);
else if ((v[0] == '6' && v[1] == '.' && v[2] != '9') ||
(v[0] == '5' && v[1] == '.' && v[2] == '9'))
lolwut6Command(c);
else else
lolwutUnstableCommand(c); lolwutUnstableCommand(c);
/* Fix back argc/argv in case of VERSION argument. */
if (v == verstr) {
c->argv -= 2;
c->argc += 2;
}
}
/* ========================== LOLWUT Canvase ===============================
* Many LOWUT versions will likely print some computer art to the screen.
* This is the case with LOLWUT 5 and LOLWUT 6, so here there is a generic
* canvas implementation that can be reused. */
/* Allocate and return a new canvas of the specified size. */
lwCanvas *lwCreateCanvas(int width, int height, int bgcolor) {
lwCanvas *canvas = zmalloc(sizeof(*canvas));
canvas->width = width;
canvas->height = height;
canvas->pixels = zmalloc(width*height);
memset(canvas->pixels,bgcolor,width*height);
return canvas;
}
/* Free the canvas created by lwCreateCanvas(). */
void lwFreeCanvas(lwCanvas *canvas) {
zfree(canvas->pixels);
zfree(canvas);
}
/* Set a pixel to the specified color. Color is 0 or 1, where zero means no
* dot will be displyed, and 1 means dot will be displayed.
* Coordinates are arranged so that left-top corner is 0,0. You can write
* out of the size of the canvas without issues. */
void lwDrawPixel(lwCanvas *canvas, int x, int y, int color) {
if (x < 0 || x >= canvas->width ||
y < 0 || y >= canvas->height) return;
canvas->pixels[x+y*canvas->width] = color;
}
/* Return the value of the specified pixel on the canvas. */
int lwGetPixel(lwCanvas *canvas, int x, int y) {
if (x < 0 || x >= canvas->width ||
y < 0 || y >= canvas->height) return 0;
return canvas->pixels[x+y*canvas->width];
}
/* Draw a line from x1,y1 to x2,y2 using the Bresenham algorithm. */
void lwDrawLine(lwCanvas *canvas, int x1, int y1, int x2, int y2, int color) {
int dx = abs(x2-x1);
int dy = abs(y2-y1);
int sx = (x1 < x2) ? 1 : -1;
int sy = (y1 < y2) ? 1 : -1;
int err = dx-dy, e2;
while(1) {
lwDrawPixel(canvas,x1,y1,color);
if (x1 == x2 && y1 == y2) break;
e2 = err*2;
if (e2 > -dy) {
err -= dy;
x1 += sx;
}
if (e2 < dx) {
err += dx;
y1 += sy;
}
}
}
/* Draw a square centered at the specified x,y coordinates, with the specified
* rotation angle and size. In order to write a rotated square, we use the
* trivial fact that the parametric equation:
*
* x = sin(k)
* y = cos(k)
*
* Describes a circle for values going from 0 to 2*PI. So basically if we start
* at 45 degrees, that is k = PI/4, with the first point, and then we find
* the other three points incrementing K by PI/2 (90 degrees), we'll have the
* points of the square. In order to rotate the square, we just start with
* k = PI/4 + rotation_angle, and we are done.
*
* Of course the vanilla equations above will describe the square inside a
* circle of radius 1, so in order to draw larger squares we'll have to
* multiply the obtained coordinates, and then translate them. However this
* is much simpler than implementing the abstract concept of 2D shape and then
* performing the rotation/translation transformation, so for LOLWUT it's
* a good approach. */
void lwDrawSquare(lwCanvas *canvas, int x, int y, float size, float angle, int color) {
int px[4], py[4];
/* Adjust the desired size according to the fact that the square inscribed
* into a circle of radius 1 has the side of length SQRT(2). This way
* size becomes a simple multiplication factor we can use with our
* coordinates to magnify them. */
size /= 1.4142135623;
size = round(size);
/* Compute the four points. */
float k = M_PI/4 + angle;
for (int j = 0; j < 4; j++) {
px[j] = round(sin(k) * size + x);
py[j] = round(cos(k) * size + y);
k += M_PI/2;
}
/* Draw the square. */
for (int j = 0; j < 4; j++)
lwDrawLine(canvas,px[j],py[j],px[(j+1)%4],py[(j+1)%4],color);
} }
/*
* Copyright (c) 2018-2019, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/* This structure represents our canvas. Drawing functions will take a pointer
* to a canvas to write to it. Later the canvas can be rendered to a string
* suitable to be printed on the screen, using unicode Braille characters. */
/* This represents a very simple generic canvas in order to draw stuff.
* It's up to each LOLWUT versions to translate what they draw to the
* screen, depending on the result to accomplish. */
typedef struct lwCanvas {
int width;
int height;
char *pixels;
} lwCanvas;
/* Drawing functions implemented inside lolwut.c. */
lwCanvas *lwCreateCanvas(int width, int height, int bgcolor);
void lwFreeCanvas(lwCanvas *canvas);
void lwDrawPixel(lwCanvas *canvas, int x, int y, int color);
int lwGetPixel(lwCanvas *canvas, int x, int y);
void lwDrawLine(lwCanvas *canvas, int x1, int y1, int x2, int y2, int color);
void lwDrawSquare(lwCanvas *canvas, int x, int y, float size, float angle, int color);
...@@ -34,17 +34,9 @@ ...@@ -34,17 +34,9 @@
*/ */
#include "server.h" #include "server.h"
#include "lolwut.h"
#include <math.h> #include <math.h>
/* This structure represents our canvas. Drawing functions will take a pointer
* to a canvas to write to it. Later the canvas can be rendered to a string
* suitable to be printed on the screen, using unicode Braille characters. */
typedef struct lwCanvas {
int width;
int height;
char *pixels;
} lwCanvas;
/* Translate a group of 8 pixels (2x4 vertical rectangle) to the corresponding /* Translate a group of 8 pixels (2x4 vertical rectangle) to the corresponding
* braille character. The byte should correspond to the pixels arranged as * braille character. The byte should correspond to the pixels arranged as
* follows, where 0 is the least significant bit, and 7 the most significant * follows, where 0 is the least significant bit, and 7 the most significant
...@@ -69,104 +61,6 @@ void lwTranslatePixelsGroup(int byte, char *output) { ...@@ -69,104 +61,6 @@ void lwTranslatePixelsGroup(int byte, char *output) {
output[2] = 0x80 | (code & 0x3F); /* 10-xxxxxx */ output[2] = 0x80 | (code & 0x3F); /* 10-xxxxxx */
} }
/* Allocate and return a new canvas of the specified size. */
lwCanvas *lwCreateCanvas(int width, int height) {
lwCanvas *canvas = zmalloc(sizeof(*canvas));
canvas->width = width;
canvas->height = height;
canvas->pixels = zmalloc(width*height);
memset(canvas->pixels,0,width*height);
return canvas;
}
/* Free the canvas created by lwCreateCanvas(). */
void lwFreeCanvas(lwCanvas *canvas) {
zfree(canvas->pixels);
zfree(canvas);
}
/* Set a pixel to the specified color. Color is 0 or 1, where zero means no
* dot will be displyed, and 1 means dot will be displayed.
* Coordinates are arranged so that left-top corner is 0,0. You can write
* out of the size of the canvas without issues. */
void lwDrawPixel(lwCanvas *canvas, int x, int y, int color) {
if (x < 0 || x >= canvas->width ||
y < 0 || y >= canvas->height) return;
canvas->pixels[x+y*canvas->width] = color;
}
/* Return the value of the specified pixel on the canvas. */
int lwGetPixel(lwCanvas *canvas, int x, int y) {
if (x < 0 || x >= canvas->width ||
y < 0 || y >= canvas->height) return 0;
return canvas->pixels[x+y*canvas->width];
}
/* Draw a line from x1,y1 to x2,y2 using the Bresenham algorithm. */
void lwDrawLine(lwCanvas *canvas, int x1, int y1, int x2, int y2, int color) {
int dx = abs(x2-x1);
int dy = abs(y2-y1);
int sx = (x1 < x2) ? 1 : -1;
int sy = (y1 < y2) ? 1 : -1;
int err = dx-dy, e2;
while(1) {
lwDrawPixel(canvas,x1,y1,color);
if (x1 == x2 && y1 == y2) break;
e2 = err*2;
if (e2 > -dy) {
err -= dy;
x1 += sx;
}
if (e2 < dx) {
err += dx;
y1 += sy;
}
}
}
/* Draw a square centered at the specified x,y coordinates, with the specified
* rotation angle and size. In order to write a rotated square, we use the
* trivial fact that the parametric equation:
*
* x = sin(k)
* y = cos(k)
*
* Describes a circle for values going from 0 to 2*PI. So basically if we start
* at 45 degrees, that is k = PI/4, with the first point, and then we find
* the other three points incrementing K by PI/2 (90 degrees), we'll have the
* points of the square. In order to rotate the square, we just start with
* k = PI/4 + rotation_angle, and we are done.
*
* Of course the vanilla equations above will describe the square inside a
* circle of radius 1, so in order to draw larger squares we'll have to
* multiply the obtained coordinates, and then translate them. However this
* is much simpler than implementing the abstract concept of 2D shape and then
* performing the rotation/translation transformation, so for LOLWUT it's
* a good approach. */
void lwDrawSquare(lwCanvas *canvas, int x, int y, float size, float angle) {
int px[4], py[4];
/* Adjust the desired size according to the fact that the square inscribed
* into a circle of radius 1 has the side of length SQRT(2). This way
* size becomes a simple multiplication factor we can use with our
* coordinates to magnify them. */
size /= 1.4142135623;
size = round(size);
/* Compute the four points. */
float k = M_PI/4 + angle;
for (int j = 0; j < 4; j++) {
px[j] = round(sin(k) * size + x);
py[j] = round(cos(k) * size + y);
k += M_PI/2;
}
/* Draw the square. */
for (int j = 0; j < 4; j++)
lwDrawLine(canvas,px[j],py[j],px[(j+1)%4],py[(j+1)%4],1);
}
/* Schotter, the output of LOLWUT of Redis 5, is a computer graphic art piece /* Schotter, the output of LOLWUT of Redis 5, is a computer graphic art piece
* generated by Georg Nees in the 60s. It explores the relationship between * generated by Georg Nees in the 60s. It explores the relationship between
* caos and order. * caos and order.
...@@ -180,7 +74,7 @@ lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_ ...@@ -180,7 +74,7 @@ lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_
int padding = canvas_width > 4 ? 2 : 0; int padding = canvas_width > 4 ? 2 : 0;
float square_side = (float)(canvas_width-padding*2) / squares_per_row; float square_side = (float)(canvas_width-padding*2) / squares_per_row;
int canvas_height = square_side * squares_per_col + padding*2; int canvas_height = square_side * squares_per_col + padding*2;
lwCanvas *canvas = lwCreateCanvas(canvas_width, canvas_height); lwCanvas *canvas = lwCreateCanvas(canvas_width, canvas_height, 0);
for (int y = 0; y < squares_per_col; y++) { for (int y = 0; y < squares_per_col; y++) {
for (int x = 0; x < squares_per_row; x++) { for (int x = 0; x < squares_per_row; x++) {
...@@ -200,7 +94,7 @@ lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_ ...@@ -200,7 +94,7 @@ lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_
sx += r2*square_side/3; sx += r2*square_side/3;
sy += r3*square_side/3; sy += r3*square_side/3;
} }
lwDrawSquare(canvas,sx,sy,square_side,angle); lwDrawSquare(canvas,sx,sy,square_side,angle,1);
} }
} }
...@@ -212,7 +106,7 @@ lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_ ...@@ -212,7 +106,7 @@ lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_
* logical canvas. The actual returned string will require a terminal that is * logical canvas. The actual returned string will require a terminal that is
* width/2 large and height/4 tall in order to hold the whole image without * width/2 large and height/4 tall in order to hold the whole image without
* overflowing or scrolling, since each Barille character is 2x4. */ * overflowing or scrolling, since each Barille character is 2x4. */
sds lwRenderCanvas(lwCanvas *canvas) { static sds renderCanvas(lwCanvas *canvas) {
sds text = sdsempty(); sds text = sdsempty();
for (int y = 0; y < canvas->height; y += 4) { for (int y = 0; y < canvas->height; y += 4) {
for (int x = 0; x < canvas->width; x += 2) { for (int x = 0; x < canvas->width; x += 2) {
...@@ -272,11 +166,12 @@ void lolwut5Command(client *c) { ...@@ -272,11 +166,12 @@ void lolwut5Command(client *c) {
/* Generate some computer art and reply. */ /* Generate some computer art and reply. */
lwCanvas *canvas = lwDrawSchotter(cols,squares_per_row,squares_per_col); lwCanvas *canvas = lwDrawSchotter(cols,squares_per_row,squares_per_col);
sds rendered = lwRenderCanvas(canvas); sds rendered = renderCanvas(canvas);
rendered = sdscat(rendered, rendered = sdscat(rendered,
"\nGeorg Nees - schotter, plotter on paper, 1968. Redis ver. "); "\nGeorg Nees - schotter, plotter on paper, 1968. Redis ver. ");
rendered = sdscat(rendered,REDIS_VERSION); rendered = sdscat(rendered,REDIS_VERSION);
rendered = sdscatlen(rendered,"\n",1); rendered = sdscatlen(rendered,"\n",1);
addReplyBulkSds(c,rendered); addReplyVerbatim(c,rendered,sdslen(rendered),"txt");
sdsfree(rendered);
lwFreeCanvas(canvas); lwFreeCanvas(canvas);
} }
/*
* Copyright (c) 2019, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
* ----------------------------------------------------------------------------
*
* This file implements the LOLWUT command. The command should do something
* fun and interesting, and should be replaced by a new implementation at
* each new version of Redis.
*
* Thanks to Michele Hiki Falcone for the original image that ispired
* the image, part of his game, Plaguemon.
*
* Thanks to the Shhh computer art collective for the help in tuning the
* output to have a better artistic effect.
*/
#include "server.h"
#include "lolwut.h"
/* Render the canvas using the four gray levels of the standard color
* terminal: they match very well to the grayscale display of the gameboy. */
static sds renderCanvas(lwCanvas *canvas) {
sds text = sdsempty();
for (int y = 0; y < canvas->height; y++) {
for (int x = 0; x < canvas->width; x++) {
int color = lwGetPixel(canvas,x,y);
char *ce; /* Color escape sequence. */
/* Note that we set both the foreground and background color.
* This way we are able to get a more consistent result among
* different terminals implementations. */
switch(color) {
case 0: ce = "0;30;40m"; break; /* Black */
case 1: ce = "0;90;100m"; break; /* Gray 1 */
case 2: ce = "0;37;47m"; break; /* Gray 2 */
case 3: ce = "0;97;107m"; break; /* White */
}
text = sdscatprintf(text,"\033[%s \033[0m",ce);
}
if (y != canvas->height-1) text = sdscatlen(text,"\n",1);
}
return text;
}
/* Draw a skyscraper on the canvas, according to the parameters in the
* 'skyscraper' structure. Window colors are random and are always one
* of the two grays. */
struct skyscraper {
int xoff; /* X offset. */
int width; /* Pixels width. */
int height; /* Pixels height. */
int windows; /* Draw windows if true. */
int color; /* Color of the skyscraper. */
};
void generateSkyscraper(lwCanvas *canvas, struct skyscraper *si) {
int starty = canvas->height-1;
int endy = starty - si->height + 1;
for (int y = starty; y >= endy; y--) {
for (int x = si->xoff; x < si->xoff+si->width; x++) {
/* The roof is four pixels less wide. */
if (y == endy && (x <= si->xoff+1 || x >= si->xoff+si->width-2))
continue;
int color = si->color;
/* Alter the color if this is a place where we want to
* draw a window. We check that we are in the inner part of the
* skyscraper, so that windows are far from the borders. */
if (si->windows &&
x > si->xoff+1 &&
x < si->xoff+si->width-2 &&
y > endy+1 &&
y < starty-1)
{
/* Calculate the x,y position relative to the start of
* the window area. */
int relx = x - (si->xoff+1);
int rely = y - (endy+1);
/* Note that we want the windows to be two pixels wide
* but just one pixel tall, because terminal "pixels"
* (characters) are not square. */
if (relx/2 % 2 && rely % 2) {
do {
color = 1 + rand() % 2;
} while (color == si->color);
/* Except we want adjacent pixels creating the same
* window to be the same color. */
if (relx % 2) color = lwGetPixel(canvas,x-1,y);
}
}
lwDrawPixel(canvas,x,y,color);
}
}
}
/* Generate a skyline inspired by the parallax backgrounds of 8 bit games. */
void generateSkyline(lwCanvas *canvas) {
struct skyscraper si;
/* First draw the background skyscraper without windows, using the
* two different grays. We use two passes to make sure that the lighter
* ones are always in the background. */
for (int color = 2; color >= 1; color--) {
si.color = color;
for (int offset = -10; offset < canvas->width;) {
offset += rand() % 8;
si.xoff = offset;
si.width = 10 + rand()%9;
if (color == 2)
si.height = canvas->height/2 + rand()%canvas->height/2;
else
si.height = canvas->height/2 + rand()%canvas->height/3;
si.windows = 0;
generateSkyscraper(canvas, &si);
if (color == 2)
offset += si.width/2;
else
offset += si.width+1;
}
}
/* Now draw the foreground skyscraper with the windows. */
si.color = 0;
for (int offset = -10; offset < canvas->width;) {
offset += rand() % 8;
si.xoff = offset;
si.width = 5 + rand()%14;
if (si.width % 4) si.width += (si.width % 3);
si.height = canvas->height/3 + rand()%canvas->height/2;
si.windows = 1;
generateSkyscraper(canvas, &si);
offset += si.width+5;
}
}
/* The LOLWUT 6 command:
*
* LOLWUT [columns] [rows]
*
* By default the command uses 80 columns, 40 squares per row
* per column.
*/
void lolwut6Command(client *c) {
long cols = 80;
long rows = 20;
/* Parse the optional arguments if any. */
if (c->argc > 1 &&
getLongFromObjectOrReply(c,c->argv[1],&cols,NULL) != C_OK)
return;
if (c->argc > 2 &&
getLongFromObjectOrReply(c,c->argv[2],&rows,NULL) != C_OK)
return;
/* Limits. We want LOLWUT to be always reasonably fast and cheap to execute
* so we have maximum number of columns, rows, and output resulution. */
if (cols < 1) cols = 1;
if (cols > 1000) cols = 1000;
if (rows < 1) rows = 1;
if (rows > 1000) rows = 1000;
/* Generate the city skyline and reply. */
lwCanvas *canvas = lwCreateCanvas(cols,rows,3);
generateSkyline(canvas);
sds rendered = renderCanvas(canvas);
rendered = sdscat(rendered,
"\nDedicated to the 8 bit game developers of past and present.\n"
"Original 8 bit image from Plaguemon by hikikomori. Redis ver. ");
rendered = sdscat(rendered,REDIS_VERSION);
rendered = sdscatlen(rendered,"\n",1);
addReplyVerbatim(c,rendered,sdslen(rendered),"txt");
sdsfree(rendered);
lwFreeCanvas(canvas);
}
...@@ -29,7 +29,9 @@ ...@@ -29,7 +29,9 @@
#include "server.h" #include "server.h"
#include "cluster.h" #include "cluster.h"
#include "rdb.h"
#include <dlfcn.h> #include <dlfcn.h>
#include <sys/wait.h>
#define REDISMODULE_CORE 1 #define REDISMODULE_CORE 1
#include "redismodule.h" #include "redismodule.h"
...@@ -40,6 +42,17 @@ ...@@ -40,6 +42,17 @@
* pointers that have an API the module can call with them) * pointers that have an API the module can call with them)
* -------------------------------------------------------------------------- */ * -------------------------------------------------------------------------- */
typedef struct RedisModuleInfoCtx {
struct RedisModule *module;
sds requested_section;
sds info; /* info string we collected so far */
int sections; /* number of sections we collected so far */
int in_section; /* indication if we're in an active section or not */
int in_dict_field; /* indication that we're curreintly appending to a dict */
} RedisModuleInfoCtx;
typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_report);
/* This structure represents a module inside the system. */ /* This structure represents a module inside the system. */
struct RedisModule { struct RedisModule {
void *handle; /* Module dlopen() handle. */ void *handle; /* Module dlopen() handle. */
...@@ -49,6 +62,10 @@ struct RedisModule { ...@@ -49,6 +62,10 @@ struct RedisModule {
list *types; /* Module data types. */ list *types; /* Module data types. */
list *usedby; /* List of modules using APIs from this one. */ list *usedby; /* List of modules using APIs from this one. */
list *using; /* List of modules we use some APIs of. */ list *using; /* List of modules we use some APIs of. */
list *filters; /* List of filters the module has registered. */
int in_call; /* RM_Call() nesting level */
int options; /* Module options and capabilities. */
RedisModuleInfoFunc info_cb; /* callback for module to add INFO fields. */
}; };
typedef struct RedisModule RedisModule; typedef struct RedisModule RedisModule;
...@@ -130,10 +147,14 @@ struct RedisModuleCtx { ...@@ -130,10 +147,14 @@ struct RedisModuleCtx {
int keys_count; int keys_count;
struct RedisModulePoolAllocBlock *pa_head; struct RedisModulePoolAllocBlock *pa_head;
redisOpArray saved_oparray; /* When propagating commands in a callback
we reallocate the "also propagate" op
array. Here we save the old one to
restore it later. */
}; };
typedef struct RedisModuleCtx RedisModuleCtx; typedef struct RedisModuleCtx RedisModuleCtx;
#define REDISMODULE_CTX_INIT {(void*)(unsigned long)&RM_GetApi, NULL, NULL, NULL, NULL, 0, 0, 0, NULL, 0, NULL, NULL, 0, NULL} #define REDISMODULE_CTX_INIT {(void*)(unsigned long)&RM_GetApi, NULL, NULL, NULL, NULL, 0, 0, 0, NULL, 0, NULL, NULL, 0, NULL, {0}}
#define REDISMODULE_CTX_MULTI_EMITTED (1<<0) #define REDISMODULE_CTX_MULTI_EMITTED (1<<0)
#define REDISMODULE_CTX_AUTO_MEMORY (1<<1) #define REDISMODULE_CTX_AUTO_MEMORY (1<<1)
#define REDISMODULE_CTX_KEYS_POS_REQUEST (1<<2) #define REDISMODULE_CTX_KEYS_POS_REQUEST (1<<2)
...@@ -141,6 +162,7 @@ typedef struct RedisModuleCtx RedisModuleCtx; ...@@ -141,6 +162,7 @@ typedef struct RedisModuleCtx RedisModuleCtx;
#define REDISMODULE_CTX_BLOCKED_TIMEOUT (1<<4) #define REDISMODULE_CTX_BLOCKED_TIMEOUT (1<<4)
#define REDISMODULE_CTX_THREAD_SAFE (1<<5) #define REDISMODULE_CTX_THREAD_SAFE (1<<5)
#define REDISMODULE_CTX_BLOCKED_DISCONNECTED (1<<6) #define REDISMODULE_CTX_BLOCKED_DISCONNECTED (1<<6)
#define REDISMODULE_CTX_MODULE_COMMAND_CALL (1<<7)
/* This represents a Redis key opened with RM_OpenKey(). */ /* This represents a Redis key opened with RM_OpenKey(). */
struct RedisModuleKey { struct RedisModuleKey {
...@@ -270,6 +292,37 @@ typedef struct RedisModuleDictIter { ...@@ -270,6 +292,37 @@ typedef struct RedisModuleDictIter {
raxIterator ri; raxIterator ri;
} RedisModuleDictIter; } RedisModuleDictIter;
typedef struct RedisModuleCommandFilterCtx {
RedisModuleString **argv;
int argc;
} RedisModuleCommandFilterCtx;
typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter);
typedef struct RedisModuleCommandFilter {
/* The module that registered the filter */
RedisModule *module;
/* Filter callback function */
RedisModuleCommandFilterFunc callback;
/* REDISMODULE_CMDFILTER_* flags */
int flags;
} RedisModuleCommandFilter;
/* Registered filters */
static list *moduleCommandFilters;
typedef void (*RedisModuleForkDoneHandler) (int exitcode, int bysignal, void *user_data);
static struct RedisModuleForkInfo {
RedisModuleForkDoneHandler done_handler;
void* done_handler_user_data;
} moduleForkInfo = {0};
/* Flags for moduleCreateArgvFromUserFormat(). */
#define REDISMODULE_ARGV_REPLICATE (1<<0)
#define REDISMODULE_ARGV_NO_AOF (1<<1)
#define REDISMODULE_ARGV_NO_REPLICAS (1<<2)
/* -------------------------------------------------------------------------- /* --------------------------------------------------------------------------
* Prototypes * Prototypes
* -------------------------------------------------------------------------- */ * -------------------------------------------------------------------------- */
...@@ -475,8 +528,47 @@ int RM_GetApi(const char *funcname, void **targetPtrPtr) { ...@@ -475,8 +528,47 @@ int RM_GetApi(const char *funcname, void **targetPtrPtr) {
return REDISMODULE_OK; return REDISMODULE_OK;
} }
/* Helper function for when a command callback is called, in order to handle
* details needed to correctly replicate commands. */
void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) {
client *c = ctx->client;
/* We don't need to do anything here if the context was never used
* in order to propagate commands. */
if (!(ctx->flags & REDISMODULE_CTX_MULTI_EMITTED)) return;
if (c->flags & CLIENT_LUA) return;
/* Handle the replication of the final EXEC, since whatever a command
* emits is always wrapped around MULTI/EXEC. */
robj *propargv[1];
propargv[0] = createStringObject("EXEC",4);
alsoPropagate(server.execCommand,c->db->id,propargv,1,
PROPAGATE_AOF|PROPAGATE_REPL);
decrRefCount(propargv[0]);
/* If this is not a module command context (but is instead a simple
* callback context), we have to handle directly the "also propagate"
* array and emit it. In a module command call this will be handled
* directly by call(). */
if (!(ctx->flags & REDISMODULE_CTX_MODULE_COMMAND_CALL) &&
server.also_propagate.numops)
{
for (int j = 0; j < server.also_propagate.numops; j++) {
redisOp *rop = &server.also_propagate.ops[j];
int target = rop->target;
if (target)
propagate(rop->cmd,rop->dbid,rop->argv,rop->argc,target);
}
redisOpArrayFree(&server.also_propagate);
/* Restore the previous oparray in case of nexted use of the API. */
server.also_propagate = ctx->saved_oparray;
}
}
/* Free the context after the user function was called. */ /* Free the context after the user function was called. */
void moduleFreeContext(RedisModuleCtx *ctx) { void moduleFreeContext(RedisModuleCtx *ctx) {
moduleHandlePropagationAfterCommandCallback(ctx);
autoMemoryCollect(ctx); autoMemoryCollect(ctx);
poolAllocRelease(ctx); poolAllocRelease(ctx);
if (ctx->postponed_arrays) { if (ctx->postponed_arrays) {
...@@ -492,34 +584,16 @@ void moduleFreeContext(RedisModuleCtx *ctx) { ...@@ -492,34 +584,16 @@ void moduleFreeContext(RedisModuleCtx *ctx) {
if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) freeClient(ctx->client); if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) freeClient(ctx->client);
} }
/* Helper function for when a command callback is called, in order to handle
* details needed to correctly replicate commands. */
void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) {
client *c = ctx->client;
if (c->flags & CLIENT_LUA) return;
/* Handle the replication of the final EXEC, since whatever a command
* emits is always wrapped around MULTI/EXEC. */
if (ctx->flags & REDISMODULE_CTX_MULTI_EMITTED) {
robj *propargv[1];
propargv[0] = createStringObject("EXEC",4);
alsoPropagate(server.execCommand,c->db->id,propargv,1,
PROPAGATE_AOF|PROPAGATE_REPL);
decrRefCount(propargv[0]);
}
}
/* This Redis command binds the normal Redis command invocation with commands /* This Redis command binds the normal Redis command invocation with commands
* exported by modules. */ * exported by modules. */
void RedisModuleCommandDispatcher(client *c) { void RedisModuleCommandDispatcher(client *c) {
RedisModuleCommandProxy *cp = (void*)(unsigned long)c->cmd->getkeys_proc; RedisModuleCommandProxy *cp = (void*)(unsigned long)c->cmd->getkeys_proc;
RedisModuleCtx ctx = REDISMODULE_CTX_INIT; RedisModuleCtx ctx = REDISMODULE_CTX_INIT;
ctx.flags |= REDISMODULE_CTX_MODULE_COMMAND_CALL;
ctx.module = cp->module; ctx.module = cp->module;
ctx.client = c; ctx.client = c;
cp->func(&ctx,(void**)c->argv,c->argc); cp->func(&ctx,(void**)c->argv,c->argc);
moduleHandlePropagationAfterCommandCallback(&ctx);
moduleFreeContext(&ctx); moduleFreeContext(&ctx);
/* In some cases processMultibulkBuffer uses sdsMakeRoomFor to /* In some cases processMultibulkBuffer uses sdsMakeRoomFor to
...@@ -731,6 +805,8 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api ...@@ -731,6 +805,8 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api
module->types = listCreate(); module->types = listCreate();
module->usedby = listCreate(); module->usedby = listCreate();
module->using = listCreate(); module->using = listCreate();
module->filters = listCreate();
module->in_call = 0;
ctx->module = module; ctx->module = module;
} }
...@@ -748,6 +824,19 @@ long long RM_Milliseconds(void) { ...@@ -748,6 +824,19 @@ long long RM_Milliseconds(void) {
return mstime(); return mstime();
} }
/* Set flags defining capabilities or behavior bit flags.
*
* REDISMODULE_OPTIONS_HANDLE_IO_ERRORS:
* Generally, modules don't need to bother with this, as the process will just
* terminate if a read error happens, however, setting this flag would allow
* repl-diskless-load to work if enabled.
* The module should use RedisModule_IsIOError after reads, before using the
* data that was read, and in case of error, propagate it upwards, and also be
* able to release the partially populated value and all it's allocations. */
void RM_SetModuleOptions(RedisModuleCtx *ctx, int options) {
ctx->module->options = options;
}
/* -------------------------------------------------------------------------- /* --------------------------------------------------------------------------
* Automatic memory management for modules * Automatic memory management for modules
* -------------------------------------------------------------------------- */ * -------------------------------------------------------------------------- */
...@@ -1102,10 +1191,9 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { ...@@ -1102,10 +1191,9 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) {
int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) {
client *c = moduleGetReplyClient(ctx); client *c = moduleGetReplyClient(ctx);
if (c == NULL) return REDISMODULE_OK; if (c == NULL) return REDISMODULE_OK;
sds strmsg = sdsnewlen(prefix,1); addReplyProto(c,prefix,strlen(prefix));
strmsg = sdscat(strmsg,msg); addReplyProto(c,msg,strlen(msg));
strmsg = sdscatlen(strmsg,"\r\n",2); addReplyProto(c,"\r\n",2);
addReplySds(c,strmsg);
return REDISMODULE_OK; return REDISMODULE_OK;
} }
...@@ -1219,6 +1307,17 @@ int RM_ReplyWithStringBuffer(RedisModuleCtx *ctx, const char *buf, size_t len) { ...@@ -1219,6 +1307,17 @@ int RM_ReplyWithStringBuffer(RedisModuleCtx *ctx, const char *buf, size_t len) {
return REDISMODULE_OK; return REDISMODULE_OK;
} }
/* Reply with a bulk string, taking in input a C buffer pointer that is
* assumed to be null-terminated.
*
* The function always returns REDISMODULE_OK. */
int RM_ReplyWithCString(RedisModuleCtx *ctx, const char *buf) {
client *c = moduleGetReplyClient(ctx);
if (c == NULL) return REDISMODULE_OK;
addReplyBulkCString(c,(char*)buf);
return REDISMODULE_OK;
}
/* Reply with a bulk string, taking in input a RedisModuleString object. /* Reply with a bulk string, taking in input a RedisModuleString object.
* *
* The function always returns REDISMODULE_OK. */ * The function always returns REDISMODULE_OK. */
...@@ -1281,9 +1380,16 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { ...@@ -1281,9 +1380,16 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) {
/* If we already emitted MULTI return ASAP. */ /* If we already emitted MULTI return ASAP. */
if (ctx->flags & REDISMODULE_CTX_MULTI_EMITTED) return; if (ctx->flags & REDISMODULE_CTX_MULTI_EMITTED) return;
/* If this is a thread safe context, we do not want to wrap commands /* If this is a thread safe context, we do not want to wrap commands
* executed into MUTLI/EXEC, they are executed as single commands * executed into MULTI/EXEC, they are executed as single commands
* from an external client in essence. */ * from an external client in essence. */
if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) return; if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) return;
/* If this is a callback context, and not a module command execution
* context, we have to setup the op array for the "also propagate" API
* so that RM_Replicate() will work. */
if (!(ctx->flags & REDISMODULE_CTX_MODULE_COMMAND_CALL)) {
ctx->saved_oparray = server.also_propagate;
redisOpArrayInit(&server.also_propagate);
}
execCommandPropagateMulti(ctx->client); execCommandPropagateMulti(ctx->client);
ctx->flags |= REDISMODULE_CTX_MULTI_EMITTED; ctx->flags |= REDISMODULE_CTX_MULTI_EMITTED;
} }
...@@ -1305,6 +1411,24 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { ...@@ -1305,6 +1411,24 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) {
* *
* Please refer to RedisModule_Call() for more information. * Please refer to RedisModule_Call() for more information.
* *
* Using the special "A" and "R" modifiers, the caller can exclude either
* the AOF or the replicas from the propagation of the specified command.
* Otherwise, by default, the command will be propagated in both channels.
*
* ## Note about calling this function from a thread safe context:
*
* Normally when you call this function from the callback implementing a
* module command, or any other callback provided by the Redis Module API,
* Redis will accumulate all the calls to this function in the context of
* the callback, and will propagate all the commands wrapped in a MULTI/EXEC
* transaction. However when calling this function from a threaded safe context
* that can live an undefined amount of time, and can be locked/unlocked in
* at will, the behavior is different: MULTI/EXEC wrapper is not emitted
* and the command specified is inserted in the AOF and replication stream
* immediately.
*
* ## Return value
*
* The command returns REDISMODULE_ERR if the format specifiers are invalid * The command returns REDISMODULE_ERR if the format specifiers are invalid
* or the command name does not belong to a known command. */ * or the command name does not belong to a known command. */
int RM_Replicate(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) { int RM_Replicate(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) {
...@@ -1322,10 +1446,23 @@ int RM_Replicate(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) ...@@ -1322,10 +1446,23 @@ int RM_Replicate(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...)
va_end(ap); va_end(ap);
if (argv == NULL) return REDISMODULE_ERR; if (argv == NULL) return REDISMODULE_ERR;
/* Replicate! */ /* Select the propagation target. Usually is AOF + replicas, however
moduleReplicateMultiIfNeeded(ctx); * the caller can exclude one or the other using the "A" or "R"
alsoPropagate(cmd,ctx->client->db->id,argv,argc, * modifiers. */
PROPAGATE_AOF|PROPAGATE_REPL); int target = 0;
if (!(flags & REDISMODULE_ARGV_NO_AOF)) target |= PROPAGATE_AOF;
if (!(flags & REDISMODULE_ARGV_NO_REPLICAS)) target |= PROPAGATE_REPL;
/* Replicate! When we are in a threaded context, we want to just insert
* the replicated command ASAP, since it is not clear when the context
* will stop being used, so accumulating stuff does not make much sense,
* nor we could easily use the alsoPropagate() API from threads. */
if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) {
propagate(cmd,ctx->client->db->id,argv,argc,target);
} else {
moduleReplicateMultiIfNeeded(ctx);
alsoPropagate(cmd,ctx->client->db->id,argv,argc,target);
}
/* Release the argv. */ /* Release the argv. */
for (j = 0; j < argc; j++) decrRefCount(argv[j]); for (j = 0; j < argc; j++) decrRefCount(argv[j]);
...@@ -1367,7 +1504,15 @@ int RM_ReplicateVerbatim(RedisModuleCtx *ctx) { ...@@ -1367,7 +1504,15 @@ int RM_ReplicateVerbatim(RedisModuleCtx *ctx) {
* are guaranteed to get IDs greater than any past ID previously seen. * are guaranteed to get IDs greater than any past ID previously seen.
* *
* Valid IDs are from 1 to 2^64-1. If 0 is returned it means there is no way * Valid IDs are from 1 to 2^64-1. If 0 is returned it means there is no way
* to fetch the ID in the context the function was currently called. */ * to fetch the ID in the context the function was currently called.
*
* After obtaining the ID, it is possible to check if the command execution
* is actually happening in the context of AOF loading, using this macro:
*
* if (RedisModule_IsAOFClient(RedisModule_GetClientId(ctx)) {
* // Handle it differently.
* }
*/
unsigned long long RM_GetClientId(RedisModuleCtx *ctx) { unsigned long long RM_GetClientId(RedisModuleCtx *ctx) {
if (ctx->client == NULL) return 0; if (ctx->client == NULL) return 0;
return ctx->client->id; return ctx->client->id;
...@@ -1414,6 +1559,21 @@ int RM_GetSelectedDb(RedisModuleCtx *ctx) { ...@@ -1414,6 +1559,21 @@ int RM_GetSelectedDb(RedisModuleCtx *ctx) {
* *
* * REDISMODULE_CTX_FLAGS_OOM_WARNING: Less than 25% of memory remains before * * REDISMODULE_CTX_FLAGS_OOM_WARNING: Less than 25% of memory remains before
* reaching the maxmemory level. * reaching the maxmemory level.
*
* * REDISMODULE_CTX_FLAGS_REPLICA_IS_STALE: No active link with the master.
*
* * REDISMODULE_CTX_FLAGS_REPLICA_IS_CONNECTING: The replica is trying to
* connect with the master.
*
* * REDISMODULE_CTX_FLAGS_REPLICA_IS_TRANSFERRING: Master -> Replica RDB
* transfer is in progress.
*
* * REDISMODULE_CTX_FLAGS_REPLICA_IS_ONLINE: The replica has an active link
* with its master. This is the
* contrary of STALE state.
*
* * REDISMODULE_CTX_FLAGS_ACTIVE_CHILD: There is currently some background
* process active (RDB, AUX or module).
*/ */
int RM_GetContextFlags(RedisModuleCtx *ctx) { int RM_GetContextFlags(RedisModuleCtx *ctx) {
...@@ -1432,6 +1592,9 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) { ...@@ -1432,6 +1592,9 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) {
if (server.cluster_enabled) if (server.cluster_enabled)
flags |= REDISMODULE_CTX_FLAGS_CLUSTER; flags |= REDISMODULE_CTX_FLAGS_CLUSTER;
if (server.loading)
flags |= REDISMODULE_CTX_FLAGS_LOADING;
/* Maxmemory and eviction policy */ /* Maxmemory and eviction policy */
if (server.maxmemory > 0) { if (server.maxmemory > 0) {
flags |= REDISMODULE_CTX_FLAGS_MAXMEMORY; flags |= REDISMODULE_CTX_FLAGS_MAXMEMORY;
...@@ -1453,6 +1616,20 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) { ...@@ -1453,6 +1616,20 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) {
flags |= REDISMODULE_CTX_FLAGS_SLAVE; flags |= REDISMODULE_CTX_FLAGS_SLAVE;
if (server.repl_slave_ro) if (server.repl_slave_ro)
flags |= REDISMODULE_CTX_FLAGS_READONLY; flags |= REDISMODULE_CTX_FLAGS_READONLY;
/* Replica state flags. */
if (server.repl_state == REPL_STATE_CONNECT ||
server.repl_state == REPL_STATE_CONNECTING)
{
flags |= REDISMODULE_CTX_FLAGS_REPLICA_IS_CONNECTING;
} else if (server.repl_state == REPL_STATE_TRANSFER) {
flags |= REDISMODULE_CTX_FLAGS_REPLICA_IS_TRANSFERRING;
} else if (server.repl_state == REPL_STATE_CONNECTED) {
flags |= REDISMODULE_CTX_FLAGS_REPLICA_IS_ONLINE;
}
if (server.repl_state != REPL_STATE_CONNECTED)
flags |= REDISMODULE_CTX_FLAGS_REPLICA_IS_STALE;
} }
/* OOM flag. */ /* OOM flag. */
...@@ -1461,6 +1638,9 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) { ...@@ -1461,6 +1638,9 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) {
if (retval == C_ERR) flags |= REDISMODULE_CTX_FLAGS_OOM; if (retval == C_ERR) flags |= REDISMODULE_CTX_FLAGS_OOM;
if (level > 0.75) flags |= REDISMODULE_CTX_FLAGS_OOM_WARNING; if (level > 0.75) flags |= REDISMODULE_CTX_FLAGS_OOM_WARNING;
/* Presence of children processes. */
if (hasActiveChildProcess()) flags |= REDISMODULE_CTX_FLAGS_ACTIVE_CHILD;
return flags; return flags;
} }
...@@ -2351,7 +2531,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) { ...@@ -2351,7 +2531,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) {
* *
* REDISMODULE_HASH_EXISTS: instead of setting the value of the field * REDISMODULE_HASH_EXISTS: instead of setting the value of the field
* expecting a RedisModuleString pointer to pointer, the function just * expecting a RedisModuleString pointer to pointer, the function just
* reports if the field esists or not and expects an integer pointer * reports if the field exists or not and expects an integer pointer
* as the second element of each pair. * as the second element of each pair.
* *
* Example of REDISMODULE_HASH_CFIELD: * Example of REDISMODULE_HASH_CFIELD:
...@@ -2640,12 +2820,11 @@ RedisModuleString *RM_CreateStringFromCallReply(RedisModuleCallReply *reply) { ...@@ -2640,12 +2820,11 @@ RedisModuleString *RM_CreateStringFromCallReply(RedisModuleCallReply *reply) {
* to special modifiers in "fmt". For now only one exists: * to special modifiers in "fmt". For now only one exists:
* *
* "!" -> REDISMODULE_ARGV_REPLICATE * "!" -> REDISMODULE_ARGV_REPLICATE
* "A" -> REDISMODULE_ARGV_NO_AOF
* "R" -> REDISMODULE_ARGV_NO_REPLICAS
* *
* On error (format specifier error) NULL is returned and nothing is * On error (format specifier error) NULL is returned and nothing is
* allocated. On success the argument vector is returned. */ * allocated. On success the argument vector is returned. */
#define REDISMODULE_ARGV_REPLICATE (1<<0)
robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int *argcp, int *flags, va_list ap) { robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int *argcp, int *flags, va_list ap) {
int argc = 0, argv_size, j; int argc = 0, argv_size, j;
robj **argv = NULL; robj **argv = NULL;
...@@ -2694,6 +2873,10 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int ...@@ -2694,6 +2873,10 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int
} }
} else if (*p == '!') { } else if (*p == '!') {
if (flags) (*flags) |= REDISMODULE_ARGV_REPLICATE; if (flags) (*flags) |= REDISMODULE_ARGV_REPLICATE;
} else if (*p == 'A') {
if (flags) (*flags) |= REDISMODULE_ARGV_NO_AOF;
} else if (*p == 'R') {
if (flags) (*flags) |= REDISMODULE_ARGV_NO_REPLICAS;
} else { } else {
goto fmterr; goto fmterr;
} }
...@@ -2714,7 +2897,10 @@ fmterr: ...@@ -2714,7 +2897,10 @@ fmterr:
* NULL is returned and errno is set to the following values: * NULL is returned and errno is set to the following values:
* *
* EINVAL: command non existing, wrong arity, wrong format specifier. * EINVAL: command non existing, wrong arity, wrong format specifier.
* EPERM: operation in Cluster instance with key in non local slot. */ * EPERM: operation in Cluster instance with key in non local slot.
*
* This API is documented here: https://redis.io/topics/modules-intro
*/
RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) { RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) {
struct redisCommand *cmd; struct redisCommand *cmd;
client *c = NULL; client *c = NULL;
...@@ -2724,15 +2910,9 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch ...@@ -2724,15 +2910,9 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
RedisModuleCallReply *reply = NULL; RedisModuleCallReply *reply = NULL;
int replicate = 0; /* Replicate this command? */ int replicate = 0; /* Replicate this command? */
cmd = lookupCommandByCString((char*)cmdname);
if (!cmd) {
errno = EINVAL;
return NULL;
}
/* Create the client and dispatch the command. */ /* Create the client and dispatch the command. */
va_start(ap, fmt); va_start(ap, fmt);
c = createClient(-1); c = createClient(NULL);
c->user = NULL; /* Root user. */ c->user = NULL; /* Root user. */
argv = moduleCreateArgvFromUserFormat(cmdname,fmt,&argc,&flags,ap); argv = moduleCreateArgvFromUserFormat(cmdname,fmt,&argc,&flags,ap);
replicate = flags & REDISMODULE_ARGV_REPLICATE; replicate = flags & REDISMODULE_ARGV_REPLICATE;
...@@ -2743,11 +2923,25 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch ...@@ -2743,11 +2923,25 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
c->db = ctx->client->db; c->db = ctx->client->db;
c->argv = argv; c->argv = argv;
c->argc = argc; c->argc = argc;
c->cmd = c->lastcmd = cmd; if (ctx->module) ctx->module->in_call++;
/* We handle the above format error only when the client is setup so that /* We handle the above format error only when the client is setup so that
* we can free it normally. */ * we can free it normally. */
if (argv == NULL) goto cleanup; if (argv == NULL) goto cleanup;
/* Call command filters */
moduleCallCommandFilters(c);
/* Lookup command now, after filters had a chance to make modifications
* if necessary.
*/
cmd = lookupCommand(c->argv[0]->ptr);
if (!cmd) {
errno = EINVAL;
goto cleanup;
}
c->cmd = c->lastcmd = cmd;
/* Basic arity checks. */ /* Basic arity checks. */
if ((cmd->arity > 0 && cmd->arity != argc) || (argc < -cmd->arity)) { if ((cmd->arity > 0 && cmd->arity != argc) || (argc < -cmd->arity)) {
errno = EINVAL; errno = EINVAL;
...@@ -2777,8 +2971,10 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch ...@@ -2777,8 +2971,10 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
/* Run the command */ /* Run the command */
int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS; int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS;
if (replicate) { if (replicate) {
call_flags |= CMD_CALL_PROPAGATE_AOF; if (!(flags & REDISMODULE_ARGV_NO_AOF))
call_flags |= CMD_CALL_PROPAGATE_REPL; call_flags |= CMD_CALL_PROPAGATE_AOF;
if (!(flags & REDISMODULE_ARGV_NO_REPLICAS))
call_flags |= CMD_CALL_PROPAGATE_REPL;
} }
call(c,call_flags); call(c,call_flags);
...@@ -2797,6 +2993,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch ...@@ -2797,6 +2993,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
autoMemoryAdd(ctx,REDISMODULE_AM_REPLY,reply); autoMemoryAdd(ctx,REDISMODULE_AM_REPLY,reply);
cleanup: cleanup:
if (ctx->module) ctx->module->in_call--;
freeClient(c); freeClient(c);
return reply; return reply;
} }
...@@ -3032,6 +3229,11 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver, ...@@ -3032,6 +3229,11 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver,
moduleTypeMemUsageFunc mem_usage; moduleTypeMemUsageFunc mem_usage;
moduleTypeDigestFunc digest; moduleTypeDigestFunc digest;
moduleTypeFreeFunc free; moduleTypeFreeFunc free;
struct {
moduleTypeAuxLoadFunc aux_load;
moduleTypeAuxSaveFunc aux_save;
int aux_save_triggers;
} v2;
} *tms = (struct typemethods*) typemethods_ptr; } *tms = (struct typemethods*) typemethods_ptr;
moduleType *mt = zcalloc(sizeof(*mt)); moduleType *mt = zcalloc(sizeof(*mt));
...@@ -3043,6 +3245,11 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver, ...@@ -3043,6 +3245,11 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver,
mt->mem_usage = tms->mem_usage; mt->mem_usage = tms->mem_usage;
mt->digest = tms->digest; mt->digest = tms->digest;
mt->free = tms->free; mt->free = tms->free;
if (tms->version >= 2) {
mt->aux_load = tms->v2.aux_load;
mt->aux_save = tms->v2.aux_save;
mt->aux_save_triggers = tms->v2.aux_save_triggers;
}
memcpy(mt->name,name,sizeof(mt->name)); memcpy(mt->name,name,sizeof(mt->name));
listAddNodeTail(ctx->module->types,mt); listAddNodeTail(ctx->module->types,mt);
return mt; return mt;
...@@ -3093,9 +3300,14 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) { ...@@ -3093,9 +3300,14 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) {
* RDB loading and saving functions * RDB loading and saving functions
* -------------------------------------------------------------------------- */ * -------------------------------------------------------------------------- */
/* Called when there is a load error in the context of a module. This cannot /* Called when there is a load error in the context of a module. On some
* be recovered like for the built-in types. */ * modules this cannot be recovered, but if the module declared capability
* to handle errors, we'll raise a flag rather than exiting. */
void moduleRDBLoadError(RedisModuleIO *io) { void moduleRDBLoadError(RedisModuleIO *io) {
if (io->type->module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) {
io->error = 1;
return;
}
serverLog(LL_WARNING, serverLog(LL_WARNING,
"Error loading data from RDB (short read or EOF). " "Error loading data from RDB (short read or EOF). "
"Read performed by module '%s' about type '%s' " "Read performed by module '%s' about type '%s' "
...@@ -3106,6 +3318,33 @@ void moduleRDBLoadError(RedisModuleIO *io) { ...@@ -3106,6 +3318,33 @@ void moduleRDBLoadError(RedisModuleIO *io) {
exit(1); exit(1);
} }
/* Returns 0 if there's at least one registered data type that did not declare
* REDISMODULE_OPTIONS_HANDLE_IO_ERRORS, in which case diskless loading should
* be avoided since it could cause data loss. */
int moduleAllDatatypesHandleErrors() {
dictIterator *di = dictGetIterator(modules);
dictEntry *de;
while ((de = dictNext(di)) != NULL) {
struct RedisModule *module = dictGetVal(de);
if (listLength(module->types) &&
!(module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS))
{
dictReleaseIterator(di);
return 0;
}
}
dictReleaseIterator(di);
return 1;
}
/* Returns true if any previous IO API failed.
* for Load* APIs the REDISMODULE_OPTIONS_HANDLE_IO_ERRORS flag must be set with
* RediModule_SetModuleOptions first. */
int RM_IsIOError(RedisModuleIO *io) {
return io->error;
}
/* Save an unsigned 64 bit value into the RDB file. This function should only /* Save an unsigned 64 bit value into the RDB file. This function should only
* be called in the context of the rdb_save method of modules implementing new * be called in the context of the rdb_save method of modules implementing new
* data types. */ * data types. */
...@@ -3129,6 +3368,7 @@ saveerr: ...@@ -3129,6 +3368,7 @@ saveerr:
* be called in the context of the rdb_load method of modules implementing * be called in the context of the rdb_load method of modules implementing
* new data types. */ * new data types. */
uint64_t RM_LoadUnsigned(RedisModuleIO *io) { uint64_t RM_LoadUnsigned(RedisModuleIO *io) {
if (io->error) return 0;
if (io->ver == 2) { if (io->ver == 2) {
uint64_t opcode = rdbLoadLen(io->rio,NULL); uint64_t opcode = rdbLoadLen(io->rio,NULL);
if (opcode != RDB_MODULE_OPCODE_UINT) goto loaderr; if (opcode != RDB_MODULE_OPCODE_UINT) goto loaderr;
...@@ -3140,7 +3380,7 @@ uint64_t RM_LoadUnsigned(RedisModuleIO *io) { ...@@ -3140,7 +3380,7 @@ uint64_t RM_LoadUnsigned(RedisModuleIO *io) {
loaderr: loaderr:
moduleRDBLoadError(io); moduleRDBLoadError(io);
return 0; /* Never reached. */ return 0;
} }
/* Like RedisModule_SaveUnsigned() but for signed 64 bit values. */ /* Like RedisModule_SaveUnsigned() but for signed 64 bit values. */
...@@ -3199,6 +3439,7 @@ saveerr: ...@@ -3199,6 +3439,7 @@ saveerr:
/* Implements RM_LoadString() and RM_LoadStringBuffer() */ /* Implements RM_LoadString() and RM_LoadStringBuffer() */
void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) { void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) {
if (io->error) return NULL;
if (io->ver == 2) { if (io->ver == 2) {
uint64_t opcode = rdbLoadLen(io->rio,NULL); uint64_t opcode = rdbLoadLen(io->rio,NULL);
if (opcode != RDB_MODULE_OPCODE_STRING) goto loaderr; if (opcode != RDB_MODULE_OPCODE_STRING) goto loaderr;
...@@ -3210,7 +3451,7 @@ void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) { ...@@ -3210,7 +3451,7 @@ void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) {
loaderr: loaderr:
moduleRDBLoadError(io); moduleRDBLoadError(io);
return NULL; /* Never reached. */ return NULL;
} }
/* In the context of the rdb_load method of a module data type, loads a string /* In the context of the rdb_load method of a module data type, loads a string
...@@ -3231,7 +3472,7 @@ RedisModuleString *RM_LoadString(RedisModuleIO *io) { ...@@ -3231,7 +3472,7 @@ RedisModuleString *RM_LoadString(RedisModuleIO *io) {
* RedisModule_Realloc() or RedisModule_Free(). * RedisModule_Realloc() or RedisModule_Free().
* *
* The size of the string is stored at '*lenptr' if not NULL. * The size of the string is stored at '*lenptr' if not NULL.
* The returned string is not automatically NULL termianted, it is loaded * The returned string is not automatically NULL terminated, it is loaded
* exactly as it was stored inisde the RDB file. */ * exactly as it was stored inisde the RDB file. */
char *RM_LoadStringBuffer(RedisModuleIO *io, size_t *lenptr) { char *RM_LoadStringBuffer(RedisModuleIO *io, size_t *lenptr) {
return moduleLoadString(io,1,lenptr); return moduleLoadString(io,1,lenptr);
...@@ -3259,6 +3500,7 @@ saveerr: ...@@ -3259,6 +3500,7 @@ saveerr:
/* In the context of the rdb_save method of a module data type, loads back the /* In the context of the rdb_save method of a module data type, loads back the
* double value saved by RedisModule_SaveDouble(). */ * double value saved by RedisModule_SaveDouble(). */
double RM_LoadDouble(RedisModuleIO *io) { double RM_LoadDouble(RedisModuleIO *io) {
if (io->error) return 0;
if (io->ver == 2) { if (io->ver == 2) {
uint64_t opcode = rdbLoadLen(io->rio,NULL); uint64_t opcode = rdbLoadLen(io->rio,NULL);
if (opcode != RDB_MODULE_OPCODE_DOUBLE) goto loaderr; if (opcode != RDB_MODULE_OPCODE_DOUBLE) goto loaderr;
...@@ -3270,7 +3512,7 @@ double RM_LoadDouble(RedisModuleIO *io) { ...@@ -3270,7 +3512,7 @@ double RM_LoadDouble(RedisModuleIO *io) {
loaderr: loaderr:
moduleRDBLoadError(io); moduleRDBLoadError(io);
return 0; /* Never reached. */ return 0;
} }
/* In the context of the rdb_save method of a module data type, saves a float /* In the context of the rdb_save method of a module data type, saves a float
...@@ -3295,6 +3537,7 @@ saveerr: ...@@ -3295,6 +3537,7 @@ saveerr:
/* In the context of the rdb_save method of a module data type, loads back the /* In the context of the rdb_save method of a module data type, loads back the
* float value saved by RedisModule_SaveFloat(). */ * float value saved by RedisModule_SaveFloat(). */
float RM_LoadFloat(RedisModuleIO *io) { float RM_LoadFloat(RedisModuleIO *io) {
if (io->error) return 0;
if (io->ver == 2) { if (io->ver == 2) {
uint64_t opcode = rdbLoadLen(io->rio,NULL); uint64_t opcode = rdbLoadLen(io->rio,NULL);
if (opcode != RDB_MODULE_OPCODE_FLOAT) goto loaderr; if (opcode != RDB_MODULE_OPCODE_FLOAT) goto loaderr;
...@@ -3306,7 +3549,37 @@ float RM_LoadFloat(RedisModuleIO *io) { ...@@ -3306,7 +3549,37 @@ float RM_LoadFloat(RedisModuleIO *io) {
loaderr: loaderr:
moduleRDBLoadError(io); moduleRDBLoadError(io);
return 0; /* Never reached. */ return 0;
}
/* Iterate over modules, and trigger rdb aux saving for the ones modules types
* who asked for it. */
ssize_t rdbSaveModulesAux(rio *rdb, int when) {
size_t total_written = 0;
dictIterator *di = dictGetIterator(modules);
dictEntry *de;
while ((de = dictNext(di)) != NULL) {
struct RedisModule *module = dictGetVal(de);
listIter li;
listNode *ln;
listRewind(module->types,&li);
while((ln = listNext(&li))) {
moduleType *mt = ln->value;
if (!mt->aux_save || !(mt->aux_save_triggers & when))
continue;
ssize_t ret = rdbSaveSingleModuleAux(rdb, when, mt);
if (ret==-1) {
dictReleaseIterator(di);
return -1;
}
total_written += ret;
}
}
dictReleaseIterator(di);
return total_written;
} }
/* -------------------------------------------------------------------------- /* --------------------------------------------------------------------------
...@@ -3438,6 +3711,14 @@ RedisModuleCtx *RM_GetContextFromIO(RedisModuleIO *io) { ...@@ -3438,6 +3711,14 @@ RedisModuleCtx *RM_GetContextFromIO(RedisModuleIO *io) {
return io->ctx; return io->ctx;
} }
/* Returns a RedisModuleString with the name of the key currently saving or
* loading, when an IO data type callback is called. There is no guarantee
* that the key name is always available, so this may return NULL.
*/
const RedisModuleString *RM_GetKeyNameFromIO(RedisModuleIO *io) {
return io->key;
}
/* -------------------------------------------------------------------------- /* --------------------------------------------------------------------------
* Logging * Logging
* -------------------------------------------------------------------------- */ * -------------------------------------------------------------------------- */
...@@ -3461,7 +3742,7 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li ...@@ -3461,7 +3742,7 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li
if (level < server.verbosity) return; if (level < server.verbosity) return;
name_len = snprintf(msg, sizeof(msg),"<%s> ", module->name); name_len = snprintf(msg, sizeof(msg),"<%s> ", module? module->name: "module");
vsnprintf(msg + name_len, sizeof(msg) - name_len, fmt, ap); vsnprintf(msg + name_len, sizeof(msg) - name_len, fmt, ap);
serverLogRaw(level,msg); serverLogRaw(level,msg);
} }
...@@ -3479,13 +3760,15 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li ...@@ -3479,13 +3760,15 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li
* There is a fixed limit to the length of the log line this function is able * There is a fixed limit to the length of the log line this function is able
* to emit, this limit is not specified but is guaranteed to be more than * to emit, this limit is not specified but is guaranteed to be more than
* a few lines of text. * a few lines of text.
*
* The ctx argument may be NULL if cannot be provided in the context of the
* caller for instance threads or callbacks, in which case a generic "module"
* will be used instead of the module name.
*/ */
void RM_Log(RedisModuleCtx *ctx, const char *levelstr, const char *fmt, ...) { void RM_Log(RedisModuleCtx *ctx, const char *levelstr, const char *fmt, ...) {
if (!ctx->module) return; /* Can only log if module is initialized */
va_list ap; va_list ap;
va_start(ap, fmt); va_start(ap, fmt);
RM_LogRaw(ctx->module,levelstr,fmt,ap); RM_LogRaw(ctx? ctx->module: NULL,levelstr,fmt,ap);
va_end(ap); va_end(ap);
} }
...@@ -3501,6 +3784,15 @@ void RM_LogIOError(RedisModuleIO *io, const char *levelstr, const char *fmt, ... ...@@ -3501,6 +3784,15 @@ void RM_LogIOError(RedisModuleIO *io, const char *levelstr, const char *fmt, ...
va_end(ap); va_end(ap);
} }
/* Redis-like assert function.
*
* A failed assertion will shut down the server and produce logging information
* that looks identical to information generated by Redis itself.
*/
void RM__Assert(const char *estr, const char *file, int line) {
_serverAssert(estr, file, line);
}
/* -------------------------------------------------------------------------- /* --------------------------------------------------------------------------
* Blocking clients from modules * Blocking clients from modules
* -------------------------------------------------------------------------- */ * -------------------------------------------------------------------------- */
...@@ -3584,7 +3876,7 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc ...@@ -3584,7 +3876,7 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc
bc->disconnect_callback = NULL; /* Set by RM_SetDisconnectCallback() */ bc->disconnect_callback = NULL; /* Set by RM_SetDisconnectCallback() */
bc->free_privdata = free_privdata; bc->free_privdata = free_privdata;
bc->privdata = NULL; bc->privdata = NULL;
bc->reply_client = createClient(-1); bc->reply_client = createClient(NULL);
bc->reply_client->flags |= CLIENT_MODULE; bc->reply_client->flags |= CLIENT_MODULE;
bc->dbid = c->db->id; bc->dbid = c->db->id;
c->bpop.timeout = timeout_ms ? (mstime()+timeout_ms) : 0; c->bpop.timeout = timeout_ms ? (mstime()+timeout_ms) : 0;
...@@ -3707,14 +3999,7 @@ void moduleHandleBlockedClients(void) { ...@@ -3707,14 +3999,7 @@ void moduleHandleBlockedClients(void) {
* replies to send to the client in a thread safe context. * replies to send to the client in a thread safe context.
* We need to glue such replies to the client output buffer and * We need to glue such replies to the client output buffer and
* free the temporary client we just used for the replies. */ * free the temporary client we just used for the replies. */
if (c) { if (c) AddReplyFromClient(c, bc->reply_client);
if (bc->reply_client->bufpos)
addReplyProto(c,bc->reply_client->buf,
bc->reply_client->bufpos);
if (listLength(bc->reply_client->reply))
listJoin(c->reply,bc->reply_client->reply);
c->reply_bytes += bc->reply_client->reply_bytes;
}
freeClient(bc->reply_client); freeClient(bc->reply_client);
if (c != NULL) { if (c != NULL) {
...@@ -3832,8 +4117,11 @@ RedisModuleCtx *RM_GetThreadSafeContext(RedisModuleBlockedClient *bc) { ...@@ -3832,8 +4117,11 @@ RedisModuleCtx *RM_GetThreadSafeContext(RedisModuleBlockedClient *bc) {
* access it safely from another thread, so we create a fake client here * access it safely from another thread, so we create a fake client here
* in order to keep things like the currently selected database and similar * in order to keep things like the currently selected database and similar
* things. */ * things. */
ctx->client = createClient(-1); ctx->client = createClient(NULL);
if (bc) selectDb(ctx->client,bc->dbid); if (bc) {
selectDb(ctx->client,bc->dbid);
ctx->client->id = bc->client->id;
}
return ctx; return ctx;
} }
...@@ -4636,6 +4924,194 @@ int RM_DictCompare(RedisModuleDictIter *di, const char *op, RedisModuleString *k ...@@ -4636,6 +4924,194 @@ int RM_DictCompare(RedisModuleDictIter *di, const char *op, RedisModuleString *k
return res ? REDISMODULE_OK : REDISMODULE_ERR; return res ? REDISMODULE_OK : REDISMODULE_ERR;
} }
/* --------------------------------------------------------------------------
* Modules Info fields
* -------------------------------------------------------------------------- */
int RM_InfoEndDictField(RedisModuleInfoCtx *ctx);
/* Used to start a new section, before adding any fields. the section name will
* be prefixed by "<modulename>_" and must only include A-Z,a-z,0-9.
* NULL or empty string indicates the default section (only <modulename>) is used.
* When return value is REDISMODULE_ERR, the section should and will be skipped. */
int RM_InfoAddSection(RedisModuleInfoCtx *ctx, char *name) {
sds full_name = sdsdup(ctx->module->name);
if (name != NULL && strlen(name) > 0)
full_name = sdscatfmt(full_name, "_%s", name);
/* Implicitly end dicts, instead of returning an error which is likely un checked. */
if (ctx->in_dict_field)
RM_InfoEndDictField(ctx);
/* proceed only if:
* 1) no section was requested (emit all)
* 2) the module name was requested (emit all)
* 3) this specific section was requested. */
if (ctx->requested_section) {
if (strcasecmp(ctx->requested_section, full_name) &&
strcasecmp(ctx->requested_section, ctx->module->name)) {
sdsfree(full_name);
ctx->in_section = 0;
return REDISMODULE_ERR;
}
}
if (ctx->sections++) ctx->info = sdscat(ctx->info,"\r\n");
ctx->info = sdscatfmt(ctx->info, "# %S\r\n", full_name);
ctx->in_section = 1;
sdsfree(full_name);
return REDISMODULE_OK;
}
/* Starts a dict field, similar to the ones in INFO KEYSPACE. Use normal
* RedisModule_InfoAddField* functions to add the items to this field, and
* terminate with RedisModule_InfoEndDictField. */
int RM_InfoBeginDictField(RedisModuleInfoCtx *ctx, char *name) {
if (!ctx->in_section)
return REDISMODULE_ERR;
/* Implicitly end dicts, instead of returning an error which is likely un checked. */
if (ctx->in_dict_field)
RM_InfoEndDictField(ctx);
ctx->info = sdscatfmt(ctx->info,
"%s_%s:",
ctx->module->name,
name);
ctx->in_dict_field = 1;
return REDISMODULE_OK;
}
/* Ends a dict field, see RedisModule_InfoBeginDictField */
int RM_InfoEndDictField(RedisModuleInfoCtx *ctx) {
if (!ctx->in_dict_field)
return REDISMODULE_ERR;
/* trim the last ',' if found. */
if (ctx->info[sdslen(ctx->info)-1]==',')
sdsIncrLen(ctx->info, -1);
ctx->info = sdscat(ctx->info, "\r\n");
ctx->in_dict_field = 0;
return REDISMODULE_OK;
}
/* Used by RedisModuleInfoFunc to add info fields.
* Each field will be automatically prefixed by "<modulename>_".
* Field names or values must not include \r\n of ":" */
int RM_InfoAddFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value) {
if (!ctx->in_section)
return REDISMODULE_ERR;
if (ctx->in_dict_field) {
ctx->info = sdscatfmt(ctx->info,
"%s=%S,",
field,
(sds)value->ptr);
return REDISMODULE_OK;
}
ctx->info = sdscatfmt(ctx->info,
"%s_%s:%S\r\n",
ctx->module->name,
field,
(sds)value->ptr);
return REDISMODULE_OK;
}
int RM_InfoAddFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) {
if (!ctx->in_section)
return REDISMODULE_ERR;
if (ctx->in_dict_field) {
ctx->info = sdscatfmt(ctx->info,
"%s=%s,",
field,
value);
return REDISMODULE_OK;
}
ctx->info = sdscatfmt(ctx->info,
"%s_%s:%s\r\n",
ctx->module->name,
field,
value);
return REDISMODULE_OK;
}
int RM_InfoAddFieldDouble(RedisModuleInfoCtx *ctx, char *field, double value) {
if (!ctx->in_section)
return REDISMODULE_ERR;
if (ctx->in_dict_field) {
ctx->info = sdscatprintf(ctx->info,
"%s=%.17g,",
field,
value);
return REDISMODULE_OK;
}
ctx->info = sdscatprintf(ctx->info,
"%s_%s:%.17g\r\n",
ctx->module->name,
field,
value);
return REDISMODULE_OK;
}
int RM_InfoAddFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long value) {
if (!ctx->in_section)
return REDISMODULE_ERR;
if (ctx->in_dict_field) {
ctx->info = sdscatfmt(ctx->info,
"%s=%I,",
field,
value);
return REDISMODULE_OK;
}
ctx->info = sdscatfmt(ctx->info,
"%s_%s:%I\r\n",
ctx->module->name,
field,
value);
return REDISMODULE_OK;
}
int RM_InfoAddFieldULongLong(RedisModuleInfoCtx *ctx, char *field, unsigned long long value) {
if (!ctx->in_section)
return REDISMODULE_ERR;
if (ctx->in_dict_field) {
ctx->info = sdscatfmt(ctx->info,
"%s=%U,",
field,
value);
return REDISMODULE_OK;
}
ctx->info = sdscatfmt(ctx->info,
"%s_%s:%U\r\n",
ctx->module->name,
field,
value);
return REDISMODULE_OK;
}
int RM_RegisterInfoFunc(RedisModuleCtx *ctx, RedisModuleInfoFunc cb) {
ctx->module->info_cb = cb;
return REDISMODULE_OK;
}
sds modulesCollectInfo(sds info, sds section, int for_crash_report, int sections) {
dictIterator *di = dictGetIterator(modules);
dictEntry *de;
while ((de = dictNext(di)) != NULL) {
struct RedisModule *module = dictGetVal(de);
if (!module->info_cb)
continue;
RedisModuleInfoCtx info_ctx = {module, section, info, sections, 0};
module->info_cb(&info_ctx, for_crash_report);
/* Implicitly end dicts (no way to handle errors, and we must add the newline). */
if (info_ctx.in_dict_field)
RM_InfoEndDictField(&info_ctx);
info = info_ctx.info;
sections = info_ctx.sections;
}
dictReleaseIterator(di);
return info;
}
/* -------------------------------------------------------------------------- /* --------------------------------------------------------------------------
* Modules utility APIs * Modules utility APIs
* -------------------------------------------------------------------------- */ * -------------------------------------------------------------------------- */
...@@ -4770,6 +5246,310 @@ int moduleUnregisterUsedAPI(RedisModule *module) { ...@@ -4770,6 +5246,310 @@ int moduleUnregisterUsedAPI(RedisModule *module) {
return count; return count;
} }
/* Unregister all filters registered by a module.
* This is called when a module is being unloaded.
*
* Returns the number of filters unregistered. */
int moduleUnregisterFilters(RedisModule *module) {
listIter li;
listNode *ln;
int count = 0;
listRewind(module->filters,&li);
while((ln = listNext(&li))) {
RedisModuleCommandFilter *filter = ln->value;
listNode *ln = listSearchKey(moduleCommandFilters,filter);
if (ln) {
listDelNode(moduleCommandFilters,ln);
count++;
}
zfree(filter);
}
return count;
}
/* --------------------------------------------------------------------------
* Module Command Filter API
* -------------------------------------------------------------------------- */
/* Register a new command filter function.
*
* Command filtering makes it possible for modules to extend Redis by plugging
* into the execution flow of all commands.
*
* A registered filter gets called before Redis executes *any* command. This
* includes both core Redis commands and commands registered by any module. The
* filter applies in all execution paths including:
*
* 1. Invocation by a client.
* 2. Invocation through `RedisModule_Call()` by any module.
* 3. Invocation through Lua 'redis.call()`.
* 4. Replication of a command from a master.
*
* The filter executes in a special filter context, which is different and more
* limited than a RedisModuleCtx. Because the filter affects any command, it
* must be implemented in a very efficient way to reduce the performance impact
* on Redis. All Redis Module API calls that require a valid context (such as
* `RedisModule_Call()`, `RedisModule_OpenKey()`, etc.) are not supported in a
* filter context.
*
* The `RedisModuleCommandFilterCtx` can be used to inspect or modify the
* executed command and its arguments. As the filter executes before Redis
* begins processing the command, any change will affect the way the command is
* processed. For example, a module can override Redis commands this way:
*
* 1. Register a `MODULE.SET` command which implements an extended version of
* the Redis `SET` command.
* 2. Register a command filter which detects invocation of `SET` on a specific
* pattern of keys. Once detected, the filter will replace the first
* argument from `SET` to `MODULE.SET`.
* 3. When filter execution is complete, Redis considers the new command name
* and therefore executes the module's own command.
*
* Note that in the above use case, if `MODULE.SET` itself uses
* `RedisModule_Call()` the filter will be applied on that call as well. If
* that is not desired, the `REDISMODULE_CMDFILTER_NOSELF` flag can be set when
* registering the filter.
*
* The `REDISMODULE_CMDFILTER_NOSELF` flag prevents execution flows that
* originate from the module's own `RM_Call()` from reaching the filter. This
* flag is effective for all execution flows, including nested ones, as long as
* the execution begins from the module's command context or a thread-safe
* context that is associated with a blocking command.
*
* Detached thread-safe contexts are *not* associated with the module and cannot
* be protected by this flag.
*
* If multiple filters are registered (by the same or different modules), they
* are executed in the order of registration.
*/
RedisModuleCommandFilter *RM_RegisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilterFunc callback, int flags) {
RedisModuleCommandFilter *filter = zmalloc(sizeof(*filter));
filter->module = ctx->module;
filter->callback = callback;
filter->flags = flags;
listAddNodeTail(moduleCommandFilters, filter);
listAddNodeTail(ctx->module->filters, filter);
return filter;
}
/* Unregister a command filter.
*/
int RM_UnregisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilter *filter) {
listNode *ln;
/* A module can only remove its own filters */
if (filter->module != ctx->module) return REDISMODULE_ERR;
ln = listSearchKey(moduleCommandFilters,filter);
if (!ln) return REDISMODULE_ERR;
listDelNode(moduleCommandFilters,ln);
ln = listSearchKey(ctx->module->filters,filter);
if (!ln) return REDISMODULE_ERR; /* Shouldn't happen */
listDelNode(ctx->module->filters,ln);
zfree(filter);
return REDISMODULE_OK;
}
void moduleCallCommandFilters(client *c) {
if (listLength(moduleCommandFilters) == 0) return;
listIter li;
listNode *ln;
listRewind(moduleCommandFilters,&li);
RedisModuleCommandFilterCtx filter = {
.argv = c->argv,
.argc = c->argc
};
while((ln = listNext(&li))) {
RedisModuleCommandFilter *f = ln->value;
/* Skip filter if REDISMODULE_CMDFILTER_NOSELF is set and module is
* currently processing a command.
*/
if ((f->flags & REDISMODULE_CMDFILTER_NOSELF) && f->module->in_call) continue;
/* Call filter */
f->callback(&filter);
}
c->argv = filter.argv;
c->argc = filter.argc;
}
/* Return the number of arguments a filtered command has. The number of
* arguments include the command itself.
*/
int RM_CommandFilterArgsCount(RedisModuleCommandFilterCtx *fctx)
{
return fctx->argc;
}
/* Return the specified command argument. The first argument (position 0) is
* the command itself, and the rest are user-provided args.
*/
const RedisModuleString *RM_CommandFilterArgGet(RedisModuleCommandFilterCtx *fctx, int pos)
{
if (pos < 0 || pos >= fctx->argc) return NULL;
return fctx->argv[pos];
}
/* Modify the filtered command by inserting a new argument at the specified
* position. The specified RedisModuleString argument may be used by Redis
* after the filter context is destroyed, so it must not be auto-memory
* allocated, freed or used elsewhere.
*/
int RM_CommandFilterArgInsert(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg)
{
int i;
if (pos < 0 || pos > fctx->argc) return REDISMODULE_ERR;
fctx->argv = zrealloc(fctx->argv, (fctx->argc+1)*sizeof(RedisModuleString *));
for (i = fctx->argc; i > pos; i--) {
fctx->argv[i] = fctx->argv[i-1];
}
fctx->argv[pos] = arg;
fctx->argc++;
return REDISMODULE_OK;
}
/* Modify the filtered command by replacing an existing argument with a new one.
* The specified RedisModuleString argument may be used by Redis after the
* filter context is destroyed, so it must not be auto-memory allocated, freed
* or used elsewhere.
*/
int RM_CommandFilterArgReplace(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg)
{
if (pos < 0 || pos >= fctx->argc) return REDISMODULE_ERR;
decrRefCount(fctx->argv[pos]);
fctx->argv[pos] = arg;
return REDISMODULE_OK;
}
/* Modify the filtered command by deleting an argument at the specified
* position.
*/
int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos)
{
int i;
if (pos < 0 || pos >= fctx->argc) return REDISMODULE_ERR;
decrRefCount(fctx->argv[pos]);
for (i = pos; i < fctx->argc-1; i++) {
fctx->argv[i] = fctx->argv[i+1];
}
fctx->argc--;
return REDISMODULE_OK;
}
/* --------------------------------------------------------------------------
* Module fork API
* -------------------------------------------------------------------------- */
/* Create a background child process with the current frozen snaphost of the
* main process where you can do some processing in the background without
* affecting / freezing the traffic and no need for threads and GIL locking.
* Note that Redis allows for only one concurrent fork.
* When the child wants to exit, it should call RedisModule_ExitFromChild.
* If the parent wants to kill the child it should call RedisModule_KillForkChild
* The done handler callback will be executed on the parent process when the
* child existed (but not when killed)
* Return: -1 on failure, on success the parent process will get a positive PID
* of the child, and the child process will get 0.
*/
int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) {
pid_t childpid;
if (hasActiveChildProcess()) {
return -1;
}
openChildInfoPipe();
if ((childpid = redisFork()) == 0) {
/* Child */
redisSetProcTitle("redis-module-fork");
} else if (childpid == -1) {
closeChildInfoPipe();
serverLog(LL_WARNING,"Can't fork for module: %s", strerror(errno));
} else {
/* Parent */
server.module_child_pid = childpid;
moduleForkInfo.done_handler = cb;
moduleForkInfo.done_handler_user_data = user_data;
serverLog(LL_NOTICE, "Module fork started pid: %d ", childpid);
}
return childpid;
}
/* Call from the child process when you want to terminate it.
* retcode will be provided to the done handler executed on the parent process.
*/
int RM_ExitFromChild(int retcode) {
sendChildCOWInfo(CHILD_INFO_TYPE_MODULE, "Module fork");
exitFromChild(retcode);
return REDISMODULE_OK;
}
/* Kill the active module forked child, if there is one active and the
* pid matches, and returns C_OK. Otherwise if there is no active module
* child or the pid does not match, return C_ERR without doing anything. */
int TerminateModuleForkChild(int child_pid, int wait) {
/* Module child should be active and pid should match. */
if (server.module_child_pid == -1 ||
server.module_child_pid != child_pid) return C_ERR;
int statloc;
serverLog(LL_NOTICE,"Killing running module fork child: %ld",
(long) server.module_child_pid);
if (kill(server.module_child_pid,SIGUSR1) != -1 && wait) {
while(wait4(server.module_child_pid,&statloc,0,NULL) !=
server.module_child_pid);
}
/* Reset the buffer accumulating changes while the child saves. */
server.module_child_pid = -1;
moduleForkInfo.done_handler = NULL;
moduleForkInfo.done_handler_user_data = NULL;
closeChildInfoPipe();
updateDictResizePolicy();
return C_OK;
}
/* Can be used to kill the forked child process from the parent process.
* child_pid whould be the return value of RedisModule_Fork. */
int RM_KillForkChild(int child_pid) {
/* Kill module child, wait for child exit. */
if (TerminateModuleForkChild(child_pid,1) == C_OK)
return REDISMODULE_OK;
else
return REDISMODULE_ERR;
}
void ModuleForkDoneHandler(int exitcode, int bysignal) {
serverLog(LL_NOTICE,
"Module fork exited pid: %d, retcode: %d, bysignal: %d",
server.module_child_pid, exitcode, bysignal);
if (moduleForkInfo.done_handler) {
moduleForkInfo.done_handler(exitcode, bysignal,
moduleForkInfo.done_handler_user_data);
}
server.module_child_pid = -1;
moduleForkInfo.done_handler = NULL;
moduleForkInfo.done_handler_user_data = NULL;
}
/* -------------------------------------------------------------------------- /* --------------------------------------------------------------------------
* Modules API internals * Modules API internals
* -------------------------------------------------------------------------- */ * -------------------------------------------------------------------------- */
...@@ -4812,10 +5592,13 @@ void moduleInitModulesSystem(void) { ...@@ -4812,10 +5592,13 @@ void moduleInitModulesSystem(void) {
/* Set up the keyspace notification susbscriber list and static client */ /* Set up the keyspace notification susbscriber list and static client */
moduleKeyspaceSubscribers = listCreate(); moduleKeyspaceSubscribers = listCreate();
moduleFreeContextReusedClient = createClient(-1); moduleFreeContextReusedClient = createClient(NULL);
moduleFreeContextReusedClient->flags |= CLIENT_MODULE; moduleFreeContextReusedClient->flags |= CLIENT_MODULE;
moduleFreeContextReusedClient->user = NULL; /* root user. */ moduleFreeContextReusedClient->user = NULL; /* root user. */
/* Set up filter list */
moduleCommandFilters = listCreate();
moduleRegisterCoreAPI(); moduleRegisterCoreAPI();
if (pipe(server.module_blocked_pipe) == -1) { if (pipe(server.module_blocked_pipe) == -1) {
serverLog(LL_WARNING, serverLog(LL_WARNING,
...@@ -4865,6 +5648,9 @@ void moduleLoadFromQueue(void) { ...@@ -4865,6 +5648,9 @@ void moduleLoadFromQueue(void) {
void moduleFreeModuleStructure(struct RedisModule *module) { void moduleFreeModuleStructure(struct RedisModule *module) {
listRelease(module->types); listRelease(module->types);
listRelease(module->filters);
listRelease(module->usedby);
listRelease(module->using);
sdsfree(module->name); sdsfree(module->name);
zfree(module); zfree(module);
} }
...@@ -4952,10 +5738,28 @@ int moduleUnload(sds name) { ...@@ -4952,10 +5738,28 @@ int moduleUnload(sds name) {
errno = EPERM; errno = EPERM;
return REDISMODULE_ERR; return REDISMODULE_ERR;
} }
/* Give module a chance to clean up. */
int (*onunload)(void *);
onunload = (int (*)(void *))(unsigned long) dlsym(module->handle, "RedisModule_OnUnload");
if (onunload) {
RedisModuleCtx ctx = REDISMODULE_CTX_INIT;
ctx.module = module;
ctx.client = moduleFreeContextReusedClient;
int unload_status = onunload((void*)&ctx);
moduleFreeContext(&ctx);
if (unload_status == REDISMODULE_ERR) {
serverLog(LL_WARNING, "Module %s OnUnload failed. Unload canceled.", name);
errno = ECANCELED;
return REDISMODULE_ERR;
}
}
moduleUnregisterCommands(module); moduleUnregisterCommands(module);
moduleUnregisterSharedAPI(module); moduleUnregisterSharedAPI(module);
moduleUnregisterUsedAPI(module); moduleUnregisterUsedAPI(module);
moduleUnregisterFilters(module);
/* Remove any notification subscribers this module might have */ /* Remove any notification subscribers this module might have */
moduleUnsubscribeNotifications(module); moduleUnsubscribeNotifications(module);
...@@ -4998,6 +5802,62 @@ void addReplyLoadedModules(client *c) { ...@@ -4998,6 +5802,62 @@ void addReplyLoadedModules(client *c) {
dictReleaseIterator(di); dictReleaseIterator(di);
} }
/* Helper for genModulesInfoString(): given a list of modules, return
* am SDS string in the form "[modulename|modulename2|...]" */
sds genModulesInfoStringRenderModulesList(list *l) {
listIter li;
listNode *ln;
listRewind(l,&li);
sds output = sdsnew("[");
while((ln = listNext(&li))) {
RedisModule *module = ln->value;
output = sdscat(output,module->name);
}
output = sdstrim(output,"|");
output = sdscat(output,"]");
return output;
}
/* Helper for genModulesInfoString(): render module options as an SDS string. */
sds genModulesInfoStringRenderModuleOptions(struct RedisModule *module) {
sds output = sdsnew("[");
if (module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS)
output = sdscat(output,"handle-io-errors|");
output = sdstrim(output,"|");
output = sdscat(output,"]");
return output;
}
/* Helper function for the INFO command: adds loaded modules as to info's
* output.
*
* After the call, the passed sds info string is no longer valid and all the
* references must be substituted with the new pointer returned by the call. */
sds genModulesInfoString(sds info) {
dictIterator *di = dictGetIterator(modules);
dictEntry *de;
while ((de = dictNext(di)) != NULL) {
sds name = dictGetKey(de);
struct RedisModule *module = dictGetVal(de);
sds usedby = genModulesInfoStringRenderModulesList(module->usedby);
sds using = genModulesInfoStringRenderModulesList(module->using);
sds options = genModulesInfoStringRenderModuleOptions(module);
info = sdscatfmt(info,
"module:name=%S,ver=%i,api=%i,filters=%i,"
"usedby=%S,using=%S,options=%S\r\n",
name, module->ver, module->apiver,
(int)listLength(module->filters), usedby, using, options);
sdsfree(usedby);
sdsfree(using);
sdsfree(options);
}
dictReleaseIterator(di);
return info;
}
/* Redis MODULE command. /* Redis MODULE command.
* *
* MODULE LOAD <path> [args...] */ * MODULE LOAD <path> [args...] */
...@@ -5083,6 +5943,7 @@ void moduleRegisterCoreAPI(void) { ...@@ -5083,6 +5943,7 @@ void moduleRegisterCoreAPI(void) {
REGISTER_API(ReplySetArrayLength); REGISTER_API(ReplySetArrayLength);
REGISTER_API(ReplyWithString); REGISTER_API(ReplyWithString);
REGISTER_API(ReplyWithStringBuffer); REGISTER_API(ReplyWithStringBuffer);
REGISTER_API(ReplyWithCString);
REGISTER_API(ReplyWithNull); REGISTER_API(ReplyWithNull);
REGISTER_API(ReplyWithCallReply); REGISTER_API(ReplyWithCallReply);
REGISTER_API(ReplyWithDouble); REGISTER_API(ReplyWithDouble);
...@@ -5145,6 +6006,8 @@ void moduleRegisterCoreAPI(void) { ...@@ -5145,6 +6006,8 @@ void moduleRegisterCoreAPI(void) {
REGISTER_API(ModuleTypeSetValue); REGISTER_API(ModuleTypeSetValue);
REGISTER_API(ModuleTypeGetType); REGISTER_API(ModuleTypeGetType);
REGISTER_API(ModuleTypeGetValue); REGISTER_API(ModuleTypeGetValue);
REGISTER_API(IsIOError);
REGISTER_API(SetModuleOptions);
REGISTER_API(SaveUnsigned); REGISTER_API(SaveUnsigned);
REGISTER_API(LoadUnsigned); REGISTER_API(LoadUnsigned);
REGISTER_API(SaveSigned); REGISTER_API(SaveSigned);
...@@ -5160,10 +6023,12 @@ void moduleRegisterCoreAPI(void) { ...@@ -5160,10 +6023,12 @@ void moduleRegisterCoreAPI(void) {
REGISTER_API(EmitAOF); REGISTER_API(EmitAOF);
REGISTER_API(Log); REGISTER_API(Log);
REGISTER_API(LogIOError); REGISTER_API(LogIOError);
REGISTER_API(_Assert);
REGISTER_API(StringAppendBuffer); REGISTER_API(StringAppendBuffer);
REGISTER_API(RetainString); REGISTER_API(RetainString);
REGISTER_API(StringCompare); REGISTER_API(StringCompare);
REGISTER_API(GetContextFromIO); REGISTER_API(GetContextFromIO);
REGISTER_API(GetKeyNameFromIO);
REGISTER_API(BlockClient); REGISTER_API(BlockClient);
REGISTER_API(UnblockClient); REGISTER_API(UnblockClient);
REGISTER_API(IsBlockedReplyRequest); REGISTER_API(IsBlockedReplyRequest);
...@@ -5219,4 +6084,23 @@ void moduleRegisterCoreAPI(void) { ...@@ -5219,4 +6084,23 @@ void moduleRegisterCoreAPI(void) {
REGISTER_API(DictCompare); REGISTER_API(DictCompare);
REGISTER_API(ExportSharedAPI); REGISTER_API(ExportSharedAPI);
REGISTER_API(GetSharedAPI); REGISTER_API(GetSharedAPI);
REGISTER_API(RegisterCommandFilter);
REGISTER_API(UnregisterCommandFilter);
REGISTER_API(CommandFilterArgsCount);
REGISTER_API(CommandFilterArgGet);
REGISTER_API(CommandFilterArgInsert);
REGISTER_API(CommandFilterArgReplace);
REGISTER_API(CommandFilterArgDelete);
REGISTER_API(Fork);
REGISTER_API(ExitFromChild);
REGISTER_API(KillForkChild);
REGISTER_API(RegisterInfoFunc);
REGISTER_API(InfoAddSection);
REGISTER_API(InfoBeginDictField);
REGISTER_API(InfoEndDictField);
REGISTER_API(InfoAddFieldString);
REGISTER_API(InfoAddFieldCString);
REGISTER_API(InfoAddFieldDouble);
REGISTER_API(InfoAddFieldLongLong);
REGISTER_API(InfoAddFieldULongLong);
} }
...@@ -46,7 +46,6 @@ hellotimer.so: hellotimer.xo ...@@ -46,7 +46,6 @@ hellotimer.so: hellotimer.xo
hellodict.xo: ../redismodule.h hellodict.xo: ../redismodule.h
hellodict.so: hellodict.xo hellodict.so: hellodict.xo
$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc
testmodule.xo: ../redismodule.h testmodule.xo: ../redismodule.h
......
...@@ -109,9 +109,9 @@ int TestStringPrintf(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { ...@@ -109,9 +109,9 @@ int TestStringPrintf(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
if (argc < 3) { if (argc < 3) {
return RedisModule_WrongArity(ctx); return RedisModule_WrongArity(ctx);
} }
RedisModuleString *s = RedisModule_CreateStringPrintf(ctx, RedisModuleString *s = RedisModule_CreateStringPrintf(ctx,
"Got %d args. argv[1]: %s, argv[2]: %s", "Got %d args. argv[1]: %s, argv[2]: %s",
argc, argc,
RedisModule_StringPtrLen(argv[1], NULL), RedisModule_StringPtrLen(argv[1], NULL),
RedisModule_StringPtrLen(argv[2], NULL) RedisModule_StringPtrLen(argv[2], NULL)
); );
...@@ -133,7 +133,7 @@ int TestUnlink(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { ...@@ -133,7 +133,7 @@ int TestUnlink(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
RedisModuleKey *k = RedisModule_OpenKey(ctx, RedisModule_CreateStringPrintf(ctx, "unlinked"), REDISMODULE_WRITE | REDISMODULE_READ); RedisModuleKey *k = RedisModule_OpenKey(ctx, RedisModule_CreateStringPrintf(ctx, "unlinked"), REDISMODULE_WRITE | REDISMODULE_READ);
if (!k) return failTest(ctx, "Could not create key"); if (!k) return failTest(ctx, "Could not create key");
if (REDISMODULE_ERR == RedisModule_StringSet(k, RedisModule_CreateStringPrintf(ctx, "Foobar"))) { if (REDISMODULE_ERR == RedisModule_StringSet(k, RedisModule_CreateStringPrintf(ctx, "Foobar"))) {
return failTest(ctx, "Could not set string value"); return failTest(ctx, "Could not set string value");
} }
...@@ -152,7 +152,7 @@ int TestUnlink(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { ...@@ -152,7 +152,7 @@ int TestUnlink(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
return failTest(ctx, "Could not verify key to be unlinked"); return failTest(ctx, "Could not verify key to be unlinked");
} }
return RedisModule_ReplyWithSimpleString(ctx, "OK"); return RedisModule_ReplyWithSimpleString(ctx, "OK");
} }
int NotifyCallback(RedisModuleCtx *ctx, int type, const char *event, int NotifyCallback(RedisModuleCtx *ctx, int type, const char *event,
...@@ -188,6 +188,10 @@ int TestNotifications(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { ...@@ -188,6 +188,10 @@ int TestNotifications(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
RedisModule_Call(ctx, "LPUSH", "cc", "l", "y"); RedisModule_Call(ctx, "LPUSH", "cc", "l", "y");
RedisModule_Call(ctx, "LPUSH", "cc", "l", "y"); RedisModule_Call(ctx, "LPUSH", "cc", "l", "y");
/* Miss some keys intentionally so we will get a "keymiss" notification. */
RedisModule_Call(ctx, "GET", "c", "nosuchkey");
RedisModule_Call(ctx, "SMEMBERS", "c", "nosuchkey");
size_t sz; size_t sz;
const char *rep; const char *rep;
RedisModuleCallReply *r = RedisModule_Call(ctx, "HGET", "cc", "notifications", "foo"); RedisModuleCallReply *r = RedisModule_Call(ctx, "HGET", "cc", "notifications", "foo");
...@@ -225,6 +229,16 @@ int TestNotifications(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { ...@@ -225,6 +229,16 @@ int TestNotifications(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
FAIL("Wrong reply for l"); FAIL("Wrong reply for l");
} }
r = RedisModule_Call(ctx, "HGET", "cc", "notifications", "nosuchkey");
if (r == NULL || RedisModule_CallReplyType(r) != REDISMODULE_REPLY_STRING) {
FAIL("Wrong or no reply for nosuchkey");
} else {
rep = RedisModule_CallReplyStringPtr(r, &sz);
if (sz != 1 || *rep != '2') {
FAIL("Got reply '%.*s'. expected '2'", sz, rep);
}
}
RedisModule_Call(ctx, "FLUSHDB", ""); RedisModule_Call(ctx, "FLUSHDB", "");
return RedisModule_ReplyWithSimpleString(ctx, "OK"); return RedisModule_ReplyWithSimpleString(ctx, "OK");
...@@ -423,7 +437,7 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) ...@@ -423,7 +437,7 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
if (RedisModule_CreateCommand(ctx,"test.ctxflags", if (RedisModule_CreateCommand(ctx,"test.ctxflags",
TestCtxFlags,"readonly",1,1,1) == REDISMODULE_ERR) TestCtxFlags,"readonly",1,1,1) == REDISMODULE_ERR)
return REDISMODULE_ERR; return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx,"test.unlink", if (RedisModule_CreateCommand(ctx,"test.unlink",
TestUnlink,"write deny-oom",1,1,1) == REDISMODULE_ERR) TestUnlink,"write deny-oom",1,1,1) == REDISMODULE_ERR)
return REDISMODULE_ERR; return REDISMODULE_ERR;
...@@ -435,7 +449,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) ...@@ -435,7 +449,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
RedisModule_SubscribeToKeyspaceEvents(ctx, RedisModule_SubscribeToKeyspaceEvents(ctx,
REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_HASH |
REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_SET |
REDISMODULE_NOTIFY_STRING, REDISMODULE_NOTIFY_STRING |
REDISMODULE_NOTIFY_KEY_MISS,
NotifyCallback); NotifyCallback);
if (RedisModule_CreateCommand(ctx,"test.notify", if (RedisModule_CreateCommand(ctx,"test.notify",
TestNotifications,"write deny-oom",1,1,1) == REDISMODULE_ERR) TestNotifications,"write deny-oom",1,1,1) == REDISMODULE_ERR)
......
...@@ -175,7 +175,19 @@ void execCommand(client *c) { ...@@ -175,7 +175,19 @@ void execCommand(client *c) {
must_propagate = 1; must_propagate = 1;
} }
call(c,server.loading ? CMD_CALL_NONE : CMD_CALL_FULL); int acl_retval = ACLCheckCommandPerm(c);
if (acl_retval != ACL_OK) {
addReplyErrorFormat(c,
"-NOPERM ACLs rules changed between the moment the "
"transaction was accumulated and the EXEC call. "
"This command is no longer allowed for the "
"following reason: %s",
(acl_retval == ACL_DENIED_CMD) ?
"no permission to execute the command or subcommand" :
"no permission to touch the specified keys");
} else {
call(c,server.loading ? CMD_CALL_NONE : CMD_CALL_FULL);
}
/* Commands may alter argc/argv, restore mstate. */ /* Commands may alter argc/argv, restore mstate. */
c->mstate.commands[j].argc = c->argc; c->mstate.commands[j].argc = c->argc;
......
...@@ -29,11 +29,13 @@ ...@@ -29,11 +29,13 @@
#include "server.h" #include "server.h"
#include "atomicvar.h" #include "atomicvar.h"
#include <sys/socket.h>
#include <sys/uio.h> #include <sys/uio.h>
#include <math.h> #include <math.h>
#include <ctype.h> #include <ctype.h>
static void setProtocolError(const char *errstr, client *c); static void setProtocolError(const char *errstr, client *c);
int postponeClientRead(client *c);
/* Return the size consumed from the allocator, for the specified SDS string, /* Return the size consumed from the allocator, for the specified SDS string,
* including internal fragmentation. This function is used in order to compute * including internal fragmentation. This function is used in order to compute
...@@ -82,33 +84,27 @@ void linkClient(client *c) { ...@@ -82,33 +84,27 @@ void linkClient(client *c) {
raxInsert(server.clients_index,(unsigned char*)&id,sizeof(id),c,NULL); raxInsert(server.clients_index,(unsigned char*)&id,sizeof(id),c,NULL);
} }
client *createClient(int fd) { client *createClient(connection *conn) {
client *c = zmalloc(sizeof(client)); client *c = zmalloc(sizeof(client));
/* passing -1 as fd it is possible to create a non connected client. /* passing NULL as conn it is possible to create a non connected client.
* This is useful since all the commands needs to be executed * This is useful since all the commands needs to be executed
* in the context of a client. When commands are executed in other * in the context of a client. When commands are executed in other
* contexts (for instance a Lua script) we need a non connected client. */ * contexts (for instance a Lua script) we need a non connected client. */
if (fd != -1) { if (conn) {
anetNonBlock(NULL,fd); connNonBlock(conn);
anetEnableTcpNoDelay(NULL,fd); connEnableTcpNoDelay(conn);
if (server.tcpkeepalive) if (server.tcpkeepalive)
anetKeepAlive(NULL,fd,server.tcpkeepalive); connKeepAlive(conn,server.tcpkeepalive);
if (aeCreateFileEvent(server.el,fd,AE_READABLE, connSetReadHandler(conn, readQueryFromClient);
readQueryFromClient, c) == AE_ERR) connSetPrivateData(conn, c);
{
close(fd);
zfree(c);
return NULL;
}
} }
selectDb(c,0); selectDb(c,0);
uint64_t client_id; uint64_t client_id = ++server.next_client_id;
atomicGetIncr(server.next_client_id,client_id,1);
c->id = client_id; c->id = client_id;
c->resp = 2; c->resp = 2;
c->fd = fd; c->conn = conn;
c->name = NULL; c->name = NULL;
c->bufpos = 0; c->bufpos = 0;
c->qb_pos = 0; c->qb_pos = 0;
...@@ -157,9 +153,10 @@ client *createClient(int fd) { ...@@ -157,9 +153,10 @@ client *createClient(int fd) {
c->pubsub_patterns = listCreate(); c->pubsub_patterns = listCreate();
c->peerid = NULL; c->peerid = NULL;
c->client_list_node = NULL; c->client_list_node = NULL;
c->client_tracking_redirection = 0;
listSetFreeMethod(c->pubsub_patterns,decrRefCountVoid); listSetFreeMethod(c->pubsub_patterns,decrRefCountVoid);
listSetMatchMethod(c->pubsub_patterns,listMatchObjects); listSetMatchMethod(c->pubsub_patterns,listMatchObjects);
if (fd != -1) linkClient(c); if (conn) linkClient(c);
initClientMultiState(c); initClientMultiState(c);
return c; return c;
} }
...@@ -225,7 +222,7 @@ int prepareClientToWrite(client *c) { ...@@ -225,7 +222,7 @@ int prepareClientToWrite(client *c) {
if ((c->flags & CLIENT_MASTER) && if ((c->flags & CLIENT_MASTER) &&
!(c->flags & CLIENT_MASTER_FORCE_REPLY)) return C_ERR; !(c->flags & CLIENT_MASTER_FORCE_REPLY)) return C_ERR;
if (c->fd <= 0) return C_ERR; /* Fake client for AOF loading. */ if (!c->conn) return C_ERR; /* Fake client for AOF loading. */
/* Schedule the client to write the output buffers to the socket, unless /* Schedule the client to write the output buffers to the socket, unless
* it should already be setup to do so (it has already pending data). */ * it should already be setup to do so (it has already pending data). */
...@@ -505,7 +502,7 @@ void addReplyDouble(client *c, double d) { ...@@ -505,7 +502,7 @@ void addReplyDouble(client *c, double d) {
if (c->resp == 2) { if (c->resp == 2) {
addReplyBulkCString(c, d > 0 ? "inf" : "-inf"); addReplyBulkCString(c, d > 0 ? "inf" : "-inf");
} else { } else {
addReplyProto(c, d > 0 ? ",inf\r\n" : "-inf\r\n", addReplyProto(c, d > 0 ? ",inf\r\n" : ",-inf\r\n",
d > 0 ? 6 : 7); d > 0 ? 6 : 7);
} }
} else { } else {
...@@ -744,6 +741,19 @@ void addReplySubcommandSyntaxError(client *c) { ...@@ -744,6 +741,19 @@ void addReplySubcommandSyntaxError(client *c) {
sdsfree(cmd); sdsfree(cmd);
} }
/* Append 'src' client output buffers into 'dst' client output buffers.
* This function clears the output buffers of 'src' */
void AddReplyFromClient(client *dst, client *src) {
if (prepareClientToWrite(dst) != C_OK)
return;
addReplyProto(dst,src->buf, src->bufpos);
if (listLength(src->reply))
listJoin(dst->reply,src->reply);
dst->reply_bytes += src->reply_bytes;
src->reply_bytes = 0;
src->bufpos = 0;
}
/* Copy 'src' client output buffers into 'dst' client output buffers. /* Copy 'src' client output buffers into 'dst' client output buffers.
* The function takes care of freeing the old output buffers of the * The function takes care of freeing the old output buffers of the
* destination client. */ * destination client. */
...@@ -762,28 +772,13 @@ int clientHasPendingReplies(client *c) { ...@@ -762,28 +772,13 @@ int clientHasPendingReplies(client *c) {
return c->bufpos || listLength(c->reply); return c->bufpos || listLength(c->reply);
} }
#define MAX_ACCEPTS_PER_CALL 1000 void clientAcceptHandler(connection *conn) {
static void acceptCommonHandler(int fd, int flags, char *ip) { client *c = connGetPrivateData(conn);
client *c;
if ((c = createClient(fd)) == NULL) {
serverLog(LL_WARNING,
"Error registering fd event for the new client: %s (fd=%d)",
strerror(errno),fd);
close(fd); /* May be already closed, just ignore errors */
return;
}
/* If maxclient directive is set and this is one client more... close the
* connection. Note that we create the client instead to check before
* for this condition, since now the socket is already set in non-blocking
* mode and we can send an error for free using the Kernel I/O */
if (listLength(server.clients) > server.maxclients) {
char *err = "-ERR max number of clients reached\r\n";
/* That's a best effort error message, don't check write errors */ if (connGetState(conn) != CONN_STATE_CONNECTED) {
if (write(c->fd,err,strlen(err)) == -1) { serverLog(LL_WARNING,
/* Nothing to do, Just to avoid the warning... */ "Error accepting a client connection: %s",
} connGetLastError(conn));
server.stat_rejected_conn++;
freeClient(c); freeClient(c);
return; return;
} }
...@@ -795,10 +790,12 @@ static void acceptCommonHandler(int fd, int flags, char *ip) { ...@@ -795,10 +790,12 @@ static void acceptCommonHandler(int fd, int flags, char *ip) {
if (server.protected_mode && if (server.protected_mode &&
server.bindaddr_count == 0 && server.bindaddr_count == 0 &&
DefaultUser->flags & USER_FLAG_NOPASS && DefaultUser->flags & USER_FLAG_NOPASS &&
!(flags & CLIENT_UNIX_SOCKET) && !(c->flags & CLIENT_UNIX_SOCKET))
ip != NULL)
{ {
if (strcmp(ip,"127.0.0.1") && strcmp(ip,"::1")) { char cip[NET_IP_STR_LEN+1] = { 0 };
connPeerToString(conn, cip, sizeof(cip)-1, NULL);
if (strcmp(cip,"127.0.0.1") && strcmp(cip,"::1")) {
char *err = char *err =
"-DENIED Redis is running in protected mode because protected " "-DENIED Redis is running in protected mode because protected "
"mode is enabled, no bind address was specified, no " "mode is enabled, no bind address was specified, no "
...@@ -820,7 +817,7 @@ static void acceptCommonHandler(int fd, int flags, char *ip) { ...@@ -820,7 +817,7 @@ static void acceptCommonHandler(int fd, int flags, char *ip) {
"4) Setup a bind address or an authentication password. " "4) Setup a bind address or an authentication password. "
"NOTE: You only need to do one of the above things in order for " "NOTE: You only need to do one of the above things in order for "
"the server to start accepting connections from the outside.\r\n"; "the server to start accepting connections from the outside.\r\n";
if (write(c->fd,err,strlen(err)) == -1) { if (connWrite(c->conn,err,strlen(err)) == -1) {
/* Nothing to do, Just to avoid the warning... */ /* Nothing to do, Just to avoid the warning... */
} }
server.stat_rejected_conn++; server.stat_rejected_conn++;
...@@ -830,7 +827,63 @@ static void acceptCommonHandler(int fd, int flags, char *ip) { ...@@ -830,7 +827,63 @@ static void acceptCommonHandler(int fd, int flags, char *ip) {
} }
server.stat_numconnections++; server.stat_numconnections++;
}
#define MAX_ACCEPTS_PER_CALL 1000
static void acceptCommonHandler(connection *conn, int flags, char *ip) {
client *c;
UNUSED(ip);
/* Admission control will happen before a client is created and connAccept()
* called, because we don't want to even start transport-level negotiation
* if rejected.
*/
if (listLength(server.clients) >= server.maxclients) {
char *err = "-ERR max number of clients reached\r\n";
/* That's a best effort error message, don't check write errors.
* Note that for TLS connections, no handshake was done yet so nothing is written
* and the connection will just drop.
*/
if (connWrite(conn,err,strlen(err)) == -1) {
/* Nothing to do, Just to avoid the warning... */
}
server.stat_rejected_conn++;
connClose(conn);
return;
}
/* Create connection and client */
if ((c = createClient(conn)) == NULL) {
char conninfo[100];
serverLog(LL_WARNING,
"Error registering fd event for the new client: %s (conn: %s)",
connGetLastError(conn),
connGetInfo(conn, conninfo, sizeof(conninfo)));
connClose(conn); /* May be already closed, just ignore errors */
return;
}
/* Last chance to keep flags */
c->flags |= flags; c->flags |= flags;
/* Initiate accept.
*
* Note that connAccept() is free to do two things here:
* 1. Call clientAcceptHandler() immediately;
* 2. Schedule a future call to clientAcceptHandler().
*
* Because of that, we must do nothing else afterwards.
*/
if (connAccept(conn, clientAcceptHandler) == C_ERR) {
char conninfo[100];
serverLog(LL_WARNING,
"Error accepting a client connection: %s (conn: %s)",
connGetLastError(conn), connGetInfo(conn, conninfo, sizeof(conninfo)));
freeClient(connGetPrivateData(conn));
return;
}
} }
void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) { void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
...@@ -849,7 +902,27 @@ void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) { ...@@ -849,7 +902,27 @@ void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
return; return;
} }
serverLog(LL_VERBOSE,"Accepted %s:%d", cip, cport); serverLog(LL_VERBOSE,"Accepted %s:%d", cip, cport);
acceptCommonHandler(cfd,0,cip); acceptCommonHandler(connCreateAcceptedSocket(cfd),0,cip);
}
}
void acceptTLSHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
int cport, cfd, max = MAX_ACCEPTS_PER_CALL;
char cip[NET_IP_STR_LEN];
UNUSED(el);
UNUSED(mask);
UNUSED(privdata);
while(max--) {
cfd = anetTcpAccept(server.neterr, fd, cip, sizeof(cip), &cport);
if (cfd == ANET_ERR) {
if (errno != EWOULDBLOCK)
serverLog(LL_WARNING,
"Accepting client connection: %s", server.neterr);
return;
}
serverLog(LL_VERBOSE,"Accepted %s:%d", cip, cport);
acceptCommonHandler(connCreateAcceptedTLS(cfd, server.tls_auth_clients),0,cip);
} }
} }
...@@ -868,7 +941,7 @@ void acceptUnixHandler(aeEventLoop *el, int fd, void *privdata, int mask) { ...@@ -868,7 +941,7 @@ void acceptUnixHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
return; return;
} }
serverLog(LL_VERBOSE,"Accepted connection to %s", server.unixsocket); serverLog(LL_VERBOSE,"Accepted connection to %s", server.unixsocket);
acceptCommonHandler(cfd,CLIENT_UNIX_SOCKET,NULL); acceptCommonHandler(connCreateAcceptedSocket(cfd),CLIENT_UNIX_SOCKET,NULL);
} }
} }
...@@ -899,10 +972,10 @@ void unlinkClient(client *c) { ...@@ -899,10 +972,10 @@ void unlinkClient(client *c) {
/* If this is marked as current client unset it. */ /* If this is marked as current client unset it. */
if (server.current_client == c) server.current_client = NULL; if (server.current_client == c) server.current_client = NULL;
/* Certain operations must be done only if the client has an active socket. /* Certain operations must be done only if the client has an active connection.
* If the client was already unlinked or if it's a "fake client" the * If the client was already unlinked or if it's a "fake client" the
* fd is already set to -1. */ * conn is already set to NULL. */
if (c->fd != -1) { if (c->conn) {
/* Remove from the list of active clients. */ /* Remove from the list of active clients. */
if (c->client_list_node) { if (c->client_list_node) {
uint64_t id = htonu64(c->id); uint64_t id = htonu64(c->id);
...@@ -911,11 +984,23 @@ void unlinkClient(client *c) { ...@@ -911,11 +984,23 @@ void unlinkClient(client *c) {
c->client_list_node = NULL; c->client_list_node = NULL;
} }
/* Unregister async I/O handlers and close the socket. */ /* Check if this is a replica waiting for diskless replication (rdb pipe),
aeDeleteFileEvent(server.el,c->fd,AE_READABLE); * in which case it needs to be cleaned from that list */
aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); if (c->flags & CLIENT_SLAVE &&
close(c->fd); c->replstate == SLAVE_STATE_WAIT_BGSAVE_END &&
c->fd = -1; server.rdb_pipe_conns)
{
int i;
for (i=0; i < server.rdb_pipe_numconns; i++) {
if (server.rdb_pipe_conns[i] == c->conn) {
rdbPipeWriteHandlerConnRemoved(c->conn);
server.rdb_pipe_conns[i] = NULL;
break;
}
}
}
connClose(c->conn);
c->conn = NULL;
} }
/* Remove from the list of pending writes if needed. */ /* Remove from the list of pending writes if needed. */
...@@ -926,6 +1011,14 @@ void unlinkClient(client *c) { ...@@ -926,6 +1011,14 @@ void unlinkClient(client *c) {
c->flags &= ~CLIENT_PENDING_WRITE; c->flags &= ~CLIENT_PENDING_WRITE;
} }
/* Remove from the list of pending reads if needed. */
if (c->flags & CLIENT_PENDING_READ) {
ln = listSearchKey(server.clients_pending_read,c);
serverAssert(ln != NULL);
listDelNode(server.clients_pending_read,ln);
c->flags &= ~CLIENT_PENDING_READ;
}
/* When client was just unblocked because of a blocking operation, /* When client was just unblocked because of a blocking operation,
* remove it from the list of unblocked clients. */ * remove it from the list of unblocked clients. */
if (c->flags & CLIENT_UNBLOCKED) { if (c->flags & CLIENT_UNBLOCKED) {
...@@ -934,6 +1027,9 @@ void unlinkClient(client *c) { ...@@ -934,6 +1027,9 @@ void unlinkClient(client *c) {
listDelNode(server.unblocked_clients,ln); listDelNode(server.unblocked_clients,ln);
c->flags &= ~CLIENT_UNBLOCKED; c->flags &= ~CLIENT_UNBLOCKED;
} }
/* Clear the tracking status. */
if (c->flags & CLIENT_TRACKING) disableTracking(c);
} }
void freeClient(client *c) { void freeClient(client *c) {
...@@ -1041,9 +1137,17 @@ void freeClient(client *c) { ...@@ -1041,9 +1137,17 @@ void freeClient(client *c) {
* a context where calling freeClient() is not possible, because the client * a context where calling freeClient() is not possible, because the client
* should be valid for the continuation of the flow of the program. */ * should be valid for the continuation of the flow of the program. */
void freeClientAsync(client *c) { void freeClientAsync(client *c) {
/* We need to handle concurrent access to the server.clients_to_close list
* only in the freeClientAsync() function, since it's the only function that
* may access the list while Redis uses I/O threads. All the other accesses
* are in the context of the main thread while the other threads are
* idle. */
static pthread_mutex_t async_free_queue_mutex = PTHREAD_MUTEX_INITIALIZER;
if (c->flags & CLIENT_CLOSE_ASAP || c->flags & CLIENT_LUA) return; if (c->flags & CLIENT_CLOSE_ASAP || c->flags & CLIENT_LUA) return;
c->flags |= CLIENT_CLOSE_ASAP; c->flags |= CLIENT_CLOSE_ASAP;
pthread_mutex_lock(&async_free_queue_mutex);
listAddNodeTail(server.clients_to_close,c); listAddNodeTail(server.clients_to_close,c);
pthread_mutex_unlock(&async_free_queue_mutex);
} }
void freeClientsInAsyncFreeQueue(void) { void freeClientsInAsyncFreeQueue(void) {
...@@ -1067,15 +1171,21 @@ client *lookupClientByID(uint64_t id) { ...@@ -1067,15 +1171,21 @@ client *lookupClientByID(uint64_t id) {
} }
/* Write data in output buffers to client. Return C_OK if the client /* Write data in output buffers to client. Return C_OK if the client
* is still valid after the call, C_ERR if it was freed. */ * is still valid after the call, C_ERR if it was freed because of some
int writeToClient(int fd, client *c, int handler_installed) { * error. If handler_installed is set, it will attempt to clear the
* write event.
*
* This function is called by threads, but always with handler_installed
* set to 0. So when handler_installed is set to 0 the function must be
* thread safe. */
int writeToClient(client *c, int handler_installed) {
ssize_t nwritten = 0, totwritten = 0; ssize_t nwritten = 0, totwritten = 0;
size_t objlen; size_t objlen;
clientReplyBlock *o; clientReplyBlock *o;
while(clientHasPendingReplies(c)) { while(clientHasPendingReplies(c)) {
if (c->bufpos > 0) { if (c->bufpos > 0) {
nwritten = write(fd,c->buf+c->sentlen,c->bufpos-c->sentlen); nwritten = connWrite(c->conn,c->buf+c->sentlen,c->bufpos-c->sentlen);
if (nwritten <= 0) break; if (nwritten <= 0) break;
c->sentlen += nwritten; c->sentlen += nwritten;
totwritten += nwritten; totwritten += nwritten;
...@@ -1096,7 +1206,7 @@ int writeToClient(int fd, client *c, int handler_installed) { ...@@ -1096,7 +1206,7 @@ int writeToClient(int fd, client *c, int handler_installed) {
continue; continue;
} }
nwritten = write(fd, o->buf + c->sentlen, objlen - c->sentlen); nwritten = connWrite(c->conn, o->buf + c->sentlen, objlen - c->sentlen);
if (nwritten <= 0) break; if (nwritten <= 0) break;
c->sentlen += nwritten; c->sentlen += nwritten;
totwritten += nwritten; totwritten += nwritten;
...@@ -1131,12 +1241,12 @@ int writeToClient(int fd, client *c, int handler_installed) { ...@@ -1131,12 +1241,12 @@ int writeToClient(int fd, client *c, int handler_installed) {
} }
server.stat_net_output_bytes += totwritten; server.stat_net_output_bytes += totwritten;
if (nwritten == -1) { if (nwritten == -1) {
if (errno == EAGAIN) { if (connGetState(c->conn) == CONN_STATE_CONNECTED) {
nwritten = 0; nwritten = 0;
} else { } else {
serverLog(LL_VERBOSE, serverLog(LL_VERBOSE,
"Error writing to client: %s", strerror(errno)); "Error writing to client: %s", connGetLastError(c->conn));
freeClient(c); freeClientAsync(c);
return C_ERR; return C_ERR;
} }
} }
...@@ -1149,11 +1259,15 @@ int writeToClient(int fd, client *c, int handler_installed) { ...@@ -1149,11 +1259,15 @@ int writeToClient(int fd, client *c, int handler_installed) {
} }
if (!clientHasPendingReplies(c)) { if (!clientHasPendingReplies(c)) {
c->sentlen = 0; c->sentlen = 0;
if (handler_installed) aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); /* Note that writeToClient() is called in a threaded way, but
* adDeleteFileEvent() is not thread safe: however writeToClient()
* is always called with handler_installed set to 0 from threads
* so we are fine. */
if (handler_installed) connSetWriteHandler(c->conn, NULL);
/* Close connection after entire reply has been sent. */ /* Close connection after entire reply has been sent. */
if (c->flags & CLIENT_CLOSE_AFTER_REPLY) { if (c->flags & CLIENT_CLOSE_AFTER_REPLY) {
freeClient(c); freeClientAsync(c);
return C_ERR; return C_ERR;
} }
} }
...@@ -1161,10 +1275,9 @@ int writeToClient(int fd, client *c, int handler_installed) { ...@@ -1161,10 +1275,9 @@ int writeToClient(int fd, client *c, int handler_installed) {
} }
/* Write event handler. Just send data to the client. */ /* Write event handler. Just send data to the client. */
void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) { void sendReplyToClient(connection *conn) {
UNUSED(el); client *c = connGetPrivateData(conn);
UNUSED(mask); writeToClient(c,1);
writeToClient(fd,privdata,1);
} }
/* This function is called just before entering the event loop, in the hope /* This function is called just before entering the event loop, in the hope
...@@ -1187,26 +1300,24 @@ int handleClientsWithPendingWrites(void) { ...@@ -1187,26 +1300,24 @@ int handleClientsWithPendingWrites(void) {
if (c->flags & CLIENT_PROTECTED) continue; if (c->flags & CLIENT_PROTECTED) continue;
/* Try to write buffers to the client socket. */ /* Try to write buffers to the client socket. */
if (writeToClient(c->fd,c,0) == C_ERR) continue; if (writeToClient(c,0) == C_ERR) continue;
/* If after the synchronous writes above we still have data to /* If after the synchronous writes above we still have data to
* output to the client, we need to install the writable handler. */ * output to the client, we need to install the writable handler. */
if (clientHasPendingReplies(c)) { if (clientHasPendingReplies(c)) {
int ae_flags = AE_WRITABLE; int ae_barrier = 0;
/* For the fsync=always policy, we want that a given FD is never /* For the fsync=always policy, we want that a given FD is never
* served for reading and writing in the same event loop iteration, * served for reading and writing in the same event loop iteration,
* so that in the middle of receiving the query, and serving it * so that in the middle of receiving the query, and serving it
* to the client, we'll call beforeSleep() that will do the * to the client, we'll call beforeSleep() that will do the
* actual fsync of AOF to disk. AE_BARRIER ensures that. */ * actual fsync of AOF to disk. the write barrier ensures that. */
if (server.aof_state == AOF_ON && if (server.aof_state == AOF_ON &&
server.aof_fsync == AOF_FSYNC_ALWAYS) server.aof_fsync == AOF_FSYNC_ALWAYS)
{ {
ae_flags |= AE_BARRIER; ae_barrier = 1;
} }
if (aeCreateFileEvent(server.el, c->fd, ae_flags, if (connSetWriteHandlerWithBarrier(c->conn, sendReplyToClient, ae_barrier) == C_ERR) {
sendReplyToClient, c) == AE_ERR) freeClientAsync(c);
{
freeClientAsync(c);
} }
} }
} }
...@@ -1252,15 +1363,15 @@ void resetClient(client *c) { ...@@ -1252,15 +1363,15 @@ void resetClient(client *c) {
* path, it is not really released, but only marked for later release. */ * path, it is not really released, but only marked for later release. */
void protectClient(client *c) { void protectClient(client *c) {
c->flags |= CLIENT_PROTECTED; c->flags |= CLIENT_PROTECTED;
aeDeleteFileEvent(server.el,c->fd,AE_READABLE); connSetReadHandler(c->conn,NULL);
aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); connSetWriteHandler(c->conn,NULL);
} }
/* This will undo the client protection done by protectClient() */ /* This will undo the client protection done by protectClient() */
void unprotectClient(client *c) { void unprotectClient(client *c) {
if (c->flags & CLIENT_PROTECTED) { if (c->flags & CLIENT_PROTECTED) {
c->flags &= ~CLIENT_PROTECTED; c->flags &= ~CLIENT_PROTECTED;
aeCreateFileEvent(server.el,c->fd,AE_READABLE,readQueryFromClient,c); connSetReadHandler(c->conn,readQueryFromClient);
if (clientHasPendingReplies(c)) clientInstallWriteHandler(c); if (clientHasPendingReplies(c)) clientInstallWriteHandler(c);
} }
} }
...@@ -1509,13 +1620,47 @@ int processMultibulkBuffer(client *c) { ...@@ -1509,13 +1620,47 @@ int processMultibulkBuffer(client *c) {
return C_ERR; return C_ERR;
} }
/* This function calls processCommand(), but also performs a few sub tasks
* that are useful in that context:
*
* 1. It sets the current client to the client 'c'.
* 2. In the case of master clients, the replication offset is updated.
* 3. The client is reset unless there are reasons to avoid doing it.
*
* The function returns C_ERR in case the client was freed as a side effect
* of processing the command, otherwise C_OK is returned. */
int processCommandAndResetClient(client *c) {
int deadclient = 0;
server.current_client = c;
if (processCommand(c) == C_OK) {
if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) {
/* Update the applied replication offset of our master. */
c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos;
}
/* Don't reset the client structure for clients blocked in a
* module blocking command, so that the reply callback will
* still be able to access the client argv and argc field.
* The client will be reset in unblockClientFromModule(). */
if (!(c->flags & CLIENT_BLOCKED) ||
c->btype != BLOCKED_MODULE)
{
resetClient(c);
}
}
if (server.current_client == NULL) deadclient = 1;
server.current_client = NULL;
/* freeMemoryIfNeeded may flush slave output buffers. This may
* result into a slave, that may be the active client, to be
* freed. */
return deadclient ? C_ERR : C_OK;
}
/* This function is called every time, in the client structure 'c', there is /* This function is called every time, in the client structure 'c', there is
* more query buffer to process, because we read more data from the socket * more query buffer to process, because we read more data from the socket
* or because a client was blocked and later reactivated, so there could be * or because a client was blocked and later reactivated, so there could be
* pending query buffer, already representing a full command, to process. */ * pending query buffer, already representing a full command, to process. */
void processInputBuffer(client *c) { void processInputBuffer(client *c) {
server.current_client = c;
/* Keep processing while there is something in the input buffer */ /* Keep processing while there is something in the input buffer */
while(c->qb_pos < sdslen(c->querybuf)) { while(c->qb_pos < sdslen(c->querybuf)) {
/* Return if clients are paused. */ /* Return if clients are paused. */
...@@ -1524,6 +1669,10 @@ void processInputBuffer(client *c) { ...@@ -1524,6 +1669,10 @@ void processInputBuffer(client *c) {
/* Immediately abort if the client is in the middle of something. */ /* Immediately abort if the client is in the middle of something. */
if (c->flags & CLIENT_BLOCKED) break; if (c->flags & CLIENT_BLOCKED) break;
/* Don't process more buffers from clients that have already pending
* commands to execute in c->argv. */
if (c->flags & CLIENT_PENDING_COMMAND) break;
/* Don't process input from the master while there is a busy script /* Don't process input from the master while there is a busy script
* condition on the slave. We want just to accumulate the replication * condition on the slave. We want just to accumulate the replication
* stream (instead of replying -BUSY like we do with other clients) and * stream (instead of replying -BUSY like we do with other clients) and
...@@ -1569,44 +1718,45 @@ void processInputBuffer(client *c) { ...@@ -1569,44 +1718,45 @@ void processInputBuffer(client *c) {
if (c->argc == 0) { if (c->argc == 0) {
resetClient(c); resetClient(c);
} else { } else {
/* Only reset the client when the command was executed. */ /* If we are in the context of an I/O thread, we can't really
if (processCommand(c) == C_OK) { * execute the command here. All we can do is to flag the client
if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) { * as one that needs to process the command. */
/* Update the applied replication offset of our master. */ if (c->flags & CLIENT_PENDING_READ) {
c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos; c->flags |= CLIENT_PENDING_COMMAND;
} break;
}
/* Don't reset the client structure for clients blocked in a /* We are finally ready to execute the command. */
* module blocking command, so that the reply callback will if (processCommandAndResetClient(c) == C_ERR) {
* still be able to access the client argv and argc field. /* If the client is no longer valid, we avoid exiting this
* The client will be reset in unblockClientFromModule(). */ * loop and trimming the client buffer later. So we return
if (!(c->flags & CLIENT_BLOCKED) || c->btype != BLOCKED_MODULE) * ASAP in that case. */
resetClient(c); return;
} }
/* freeMemoryIfNeeded may flush slave output buffers. This may
* result into a slave, that may be the active client, to be
* freed. */
if (server.current_client == NULL) break;
} }
} }
/* Trim to pos */ /* Trim to pos */
if (server.current_client != NULL && c->qb_pos) { if (c->qb_pos) {
sdsrange(c->querybuf,c->qb_pos,-1); sdsrange(c->querybuf,c->qb_pos,-1);
c->qb_pos = 0; c->qb_pos = 0;
} }
server.current_client = NULL;
} }
/* This is a wrapper for processInputBuffer that also cares about handling /* This is a wrapper for processInputBuffer that also cares about handling
* the replication forwarding to the sub-slaves, in case the client 'c' * the replication forwarding to the sub-replicas, in case the client 'c'
* is flagged as master. Usually you want to call this instead of the * is flagged as master. Usually you want to call this instead of the
* raw processInputBuffer(). */ * raw processInputBuffer(). */
void processInputBufferAndReplicate(client *c) { void processInputBufferAndReplicate(client *c) {
if (!(c->flags & CLIENT_MASTER)) { if (!(c->flags & CLIENT_MASTER)) {
processInputBuffer(c); processInputBuffer(c);
} else { } else {
/* If the client is a master we need to compute the difference
* between the applied offset before and after processing the buffer,
* to understand how much of the replication stream was actually
* applied to the master state: this quantity, and its corresponding
* part of the replication stream, will be propagated to the
* sub-replicas and to the replication backlog. */
size_t prev_offset = c->reploff; size_t prev_offset = c->reploff;
processInputBuffer(c); processInputBuffer(c);
size_t applied = c->reploff - prev_offset; size_t applied = c->reploff - prev_offset;
...@@ -1618,12 +1768,14 @@ void processInputBufferAndReplicate(client *c) { ...@@ -1618,12 +1768,14 @@ void processInputBufferAndReplicate(client *c) {
} }
} }
void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { void readQueryFromClient(connection *conn) {
client *c = (client*) privdata; client *c = connGetPrivateData(conn);
int nread, readlen; int nread, readlen;
size_t qblen; size_t qblen;
UNUSED(el);
UNUSED(mask); /* Check if we want to read from the client later when exiting from
* the event loop. This is the case if threaded I/O is enabled. */
if (postponeClientRead(c)) return;
readlen = PROTO_IOBUF_LEN; readlen = PROTO_IOBUF_LEN;
/* If this is a multi bulk request, and we are processing a bulk reply /* If this is a multi bulk request, and we are processing a bulk reply
...@@ -1645,18 +1797,18 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { ...@@ -1645,18 +1797,18 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) {
qblen = sdslen(c->querybuf); qblen = sdslen(c->querybuf);
if (c->querybuf_peak < qblen) c->querybuf_peak = qblen; if (c->querybuf_peak < qblen) c->querybuf_peak = qblen;
c->querybuf = sdsMakeRoomFor(c->querybuf, readlen); c->querybuf = sdsMakeRoomFor(c->querybuf, readlen);
nread = read(fd, c->querybuf+qblen, readlen); nread = connRead(c->conn, c->querybuf+qblen, readlen);
if (nread == -1) { if (nread == -1) {
if (errno == EAGAIN) { if (connGetState(conn) == CONN_STATE_CONNECTED) {
return; return;
} else { } else {
serverLog(LL_VERBOSE, "Reading from client: %s",strerror(errno)); serverLog(LL_VERBOSE, "Reading from client: %s",connGetLastError(c->conn));
freeClient(c); freeClientAsync(c);
return; return;
} }
} else if (nread == 0) { } else if (nread == 0) {
serverLog(LL_VERBOSE, "Client closed connection"); serverLog(LL_VERBOSE, "Client closed connection");
freeClient(c); freeClientAsync(c);
return; return;
} else if (c->flags & CLIENT_MASTER) { } else if (c->flags & CLIENT_MASTER) {
/* Append the query buffer to the pending (not applied) buffer /* Append the query buffer to the pending (not applied) buffer
...@@ -1677,17 +1829,13 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { ...@@ -1677,17 +1829,13 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) {
serverLog(LL_WARNING,"Closing client that reached max query buffer length: %s (qbuf initial bytes: %s)", ci, bytes); serverLog(LL_WARNING,"Closing client that reached max query buffer length: %s (qbuf initial bytes: %s)", ci, bytes);
sdsfree(ci); sdsfree(ci);
sdsfree(bytes); sdsfree(bytes);
freeClient(c); freeClientAsync(c);
return; return;
} }
/* Time to process the buffer. If the client is a master we need to /* There is more data in the client input buffer, continue parsing it
* compute the difference between the applied offset before and after * in case to check if there is a full command to execute. */
* processing the buffer, to understand how much of the replication stream processInputBufferAndReplicate(c);
* was actually applied to the master state: this quantity, and its
* corresponding part of the replication stream, will be propagated to
* the sub-slaves and to the replication backlog. */
processInputBufferAndReplicate(c);
} }
void getClientsMaxBuffers(unsigned long *longest_output_list, void getClientsMaxBuffers(unsigned long *longest_output_list,
...@@ -1726,7 +1874,7 @@ void genClientPeerId(client *client, char *peerid, ...@@ -1726,7 +1874,7 @@ void genClientPeerId(client *client, char *peerid,
snprintf(peerid,peerid_len,"%s:0",server.unixsocket); snprintf(peerid,peerid_len,"%s:0",server.unixsocket);
} else { } else {
/* TCP client. */ /* TCP client. */
anetFormatPeer(client->fd,peerid,peerid_len); connFormatPeer(client->conn,peerid,peerid_len);
} }
} }
...@@ -1747,8 +1895,7 @@ char *getClientPeerId(client *c) { ...@@ -1747,8 +1895,7 @@ char *getClientPeerId(client *c) {
/* Concatenate a string representing the state of a client in an human /* Concatenate a string representing the state of a client in an human
* readable format, into the sds string 's'. */ * readable format, into the sds string 's'. */
sds catClientInfoString(sds s, client *client) { sds catClientInfoString(sds s, client *client) {
char flags[16], events[3], *p; char flags[16], events[3], conninfo[CONN_INFO_LEN], *p;
int emask;
p = flags; p = flags;
if (client->flags & CLIENT_SLAVE) { if (client->flags & CLIENT_SLAVE) {
...@@ -1761,6 +1908,8 @@ sds catClientInfoString(sds s, client *client) { ...@@ -1761,6 +1908,8 @@ sds catClientInfoString(sds s, client *client) {
if (client->flags & CLIENT_PUBSUB) *p++ = 'P'; if (client->flags & CLIENT_PUBSUB) *p++ = 'P';
if (client->flags & CLIENT_MULTI) *p++ = 'x'; if (client->flags & CLIENT_MULTI) *p++ = 'x';
if (client->flags & CLIENT_BLOCKED) *p++ = 'b'; if (client->flags & CLIENT_BLOCKED) *p++ = 'b';
if (client->flags & CLIENT_TRACKING) *p++ = 't';
if (client->flags & CLIENT_TRACKING_BROKEN_REDIR) *p++ = 'R';
if (client->flags & CLIENT_DIRTY_CAS) *p++ = 'd'; if (client->flags & CLIENT_DIRTY_CAS) *p++ = 'd';
if (client->flags & CLIENT_CLOSE_AFTER_REPLY) *p++ = 'c'; if (client->flags & CLIENT_CLOSE_AFTER_REPLY) *p++ = 'c';
if (client->flags & CLIENT_UNBLOCKED) *p++ = 'u'; if (client->flags & CLIENT_UNBLOCKED) *p++ = 'u';
...@@ -1770,16 +1919,17 @@ sds catClientInfoString(sds s, client *client) { ...@@ -1770,16 +1919,17 @@ sds catClientInfoString(sds s, client *client) {
if (p == flags) *p++ = 'N'; if (p == flags) *p++ = 'N';
*p++ = '\0'; *p++ = '\0';
emask = client->fd == -1 ? 0 : aeGetFileEvents(server.el,client->fd);
p = events; p = events;
if (emask & AE_READABLE) *p++ = 'r'; if (client->conn) {
if (emask & AE_WRITABLE) *p++ = 'w'; if (connHasReadHandler(client->conn)) *p++ = 'r';
if (connHasWriteHandler(client->conn)) *p++ = 'w';
}
*p = '\0'; *p = '\0';
return sdscatfmt(s, return sdscatfmt(s,
"id=%U addr=%s fd=%i name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U obl=%U oll=%U omem=%U events=%s cmd=%s user=%s", "id=%U addr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U obl=%U oll=%U omem=%U events=%s cmd=%s user=%s",
(unsigned long long) client->id, (unsigned long long) client->id,
getClientPeerId(client), getClientPeerId(client),
client->fd, connGetInfo(client->conn, conninfo, sizeof(conninfo)),
client->name ? (char*)client->name->ptr : "", client->name ? (char*)client->name->ptr : "",
(long long)(server.unixtime - client->ctime), (long long)(server.unixtime - client->ctime),
(long long)(server.unixtime - client->lastinteraction), (long long)(server.unixtime - client->lastinteraction),
...@@ -1860,19 +2010,21 @@ void clientCommand(client *c) { ...@@ -1860,19 +2010,21 @@ void clientCommand(client *c) {
if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) {
const char *help[] = { const char *help[] = {
"id -- Return the ID of the current connection.", "ID -- Return the ID of the current connection.",
"getname -- Return the name of the current connection.", "GETNAME -- Return the name of the current connection.",
"kill <ip:port> -- Kill connection made from <ip:port>.", "KILL <ip:port> -- Kill connection made from <ip:port>.",
"kill <option> <value> [option value ...] -- Kill connections. Options are:", "KILL <option> <value> [option value ...] -- Kill connections. Options are:",
" addr <ip:port> -- Kill connection made from <ip:port>", " ADDR <ip:port> -- Kill connection made from <ip:port>",
" type (normal|master|replica|pubsub) -- Kill connections by type.", " TYPE (normal|master|replica|pubsub) -- Kill connections by type.",
" skipme (yes|no) -- Skip killing current connection (default: yes).", " SKIPME (yes|no) -- Skip killing current connection (default: yes).",
"list [options ...] -- Return information about client connections. Options:", "LIST [options ...] -- Return information about client connections. Options:",
" type (normal|master|replica|pubsub) -- Return clients of specified type.", " TYPE (normal|master|replica|pubsub) -- Return clients of specified type.",
"pause <timeout> -- Suspend all Redis clients for <timout> milliseconds.", "PAUSE <timeout> -- Suspend all Redis clients for <timout> milliseconds.",
"reply (on|off|skip) -- Control the replies sent to the current connection.", "REPLY (on|off|skip) -- Control the replies sent to the current connection.",
"setname <name> -- Assign the name <name> to the current connection.", "SETNAME <name> -- Assign the name <name> to the current connection.",
"unblock <clientid> [TIMEOUT|ERROR] -- Unblock the specified blocked client.", "UNBLOCK <clientid> [TIMEOUT|ERROR] -- Unblock the specified blocked client.",
"TRACKING (on|off) [REDIRECT <id>] -- Enable client keys tracking for client side caching.",
"GETREDIR -- Return the client ID we are redirecting to when tracking is enabled.",
NULL NULL
}; };
addReplyHelp(c, help); addReplyHelp(c, help);
...@@ -1894,7 +2046,7 @@ NULL ...@@ -1894,7 +2046,7 @@ NULL
return; return;
} }
sds o = getAllClientsInfoString(type); sds o = getAllClientsInfoString(type);
addReplyBulkCBuffer(c,o,sdslen(o)); addReplyVerbatim(c,o,sdslen(o),"txt");
sdsfree(o); sdsfree(o);
} else if (!strcasecmp(c->argv[1]->ptr,"reply") && c->argc == 3) { } else if (!strcasecmp(c->argv[1]->ptr,"reply") && c->argc == 3) {
/* CLIENT REPLY ON|OFF|SKIP */ /* CLIENT REPLY ON|OFF|SKIP */
...@@ -2029,20 +2181,63 @@ NULL ...@@ -2029,20 +2181,63 @@ NULL
addReply(c,shared.czero); addReply(c,shared.czero);
} }
} else if (!strcasecmp(c->argv[1]->ptr,"setname") && c->argc == 3) { } else if (!strcasecmp(c->argv[1]->ptr,"setname") && c->argc == 3) {
/* CLIENT SETNAME */
if (clientSetNameOrReply(c,c->argv[2]) == C_OK) if (clientSetNameOrReply(c,c->argv[2]) == C_OK)
addReply(c,shared.ok); addReply(c,shared.ok);
} else if (!strcasecmp(c->argv[1]->ptr,"getname") && c->argc == 2) { } else if (!strcasecmp(c->argv[1]->ptr,"getname") && c->argc == 2) {
/* CLIENT GETNAME */
if (c->name) if (c->name)
addReplyBulk(c,c->name); addReplyBulk(c,c->name);
else else
addReplyNull(c); addReplyNull(c);
} else if (!strcasecmp(c->argv[1]->ptr,"pause") && c->argc == 3) { } else if (!strcasecmp(c->argv[1]->ptr,"pause") && c->argc == 3) {
/* CLIENT PAUSE */
long long duration; long long duration;
if (getTimeoutFromObjectOrReply(c,c->argv[2],&duration,UNIT_MILLISECONDS) if (getTimeoutFromObjectOrReply(c,c->argv[2],&duration,
!= C_OK) return; UNIT_MILLISECONDS) != C_OK) return;
pauseClients(duration); pauseClients(duration);
addReply(c,shared.ok); addReply(c,shared.ok);
} else if (!strcasecmp(c->argv[1]->ptr,"tracking") &&
(c->argc == 3 || c->argc == 5))
{
/* CLIENT TRACKING (on|off) [REDIRECT <id>] */
long long redir = 0;
/* Parse the redirection option: we'll require the client with
* the specified ID to exist right now, even if it is possible
* it will get disconnected later. */
if (c->argc == 5) {
if (strcasecmp(c->argv[3]->ptr,"redirect") != 0) {
addReply(c,shared.syntaxerr);
return;
} else {
if (getLongLongFromObjectOrReply(c,c->argv[4],&redir,NULL) !=
C_OK) return;
if (lookupClientByID(redir) == NULL) {
addReplyError(c,"The client ID you want redirect to "
"does not exist");
return;
}
}
}
if (!strcasecmp(c->argv[2]->ptr,"on")) {
enableTracking(c,redir);
} else if (!strcasecmp(c->argv[2]->ptr,"off")) {
disableTracking(c);
} else {
addReply(c,shared.syntaxerr);
return;
}
addReply(c,shared.ok);
} else if (!strcasecmp(c->argv[1]->ptr,"getredir") && c->argc == 2) {
/* CLIENT GETREDIR */
if (c->flags & CLIENT_TRACKING) {
addReplyLongLong(c,c->client_tracking_redirection);
} else {
addReplyLongLong(c,-1);
}
} else { } else {
addReplyErrorFormat(c, "Unknown subcommand or wrong number of arguments for '%s'. Try CLIENT HELP", (char*)c->argv[1]->ptr); addReplyErrorFormat(c, "Unknown subcommand or wrong number of arguments for '%s'. Try CLIENT HELP", (char*)c->argv[1]->ptr);
} }
...@@ -2207,15 +2402,8 @@ void rewriteClientCommandArgument(client *c, int i, robj *newval) { ...@@ -2207,15 +2402,8 @@ void rewriteClientCommandArgument(client *c, int i, robj *newval) {
} }
} }
/* This function returns the number of bytes that Redis is virtually /* This function returns the number of bytes that Redis is
* using to store the reply still not read by the client. * using to store the reply still not read by the client.
* It is "virtual" since the reply output list may contain objects that
* are shared and are not really using additional memory.
*
* The function returns the total sum of the length of all the objects
* stored in the output list, plus the memory used to allocate every
* list node. The static reply buffer is not taken into account since it
* is allocated anyway.
* *
* Note: this function is very fast so can be called as many time as * Note: this function is very fast so can be called as many time as
* the caller wishes. The main usage of this function currently is * the caller wishes. The main usage of this function currently is
...@@ -2313,7 +2501,7 @@ int checkClientOutputBufferLimits(client *c) { ...@@ -2313,7 +2501,7 @@ int checkClientOutputBufferLimits(client *c) {
* called from contexts where the client can't be freed safely, i.e. from the * called from contexts where the client can't be freed safely, i.e. from the
* lower level functions pushing data inside the client output buffers. */ * lower level functions pushing data inside the client output buffers. */
void asyncCloseClientOnOutputBufferLimitReached(client *c) { void asyncCloseClientOnOutputBufferLimitReached(client *c) {
if (c->fd == -1) return; /* It is unsafe to free fake clients. */ if (!c->conn) return; /* It is unsafe to free fake clients. */
serverAssert(c->reply_bytes < SIZE_MAX-(1024*64)); serverAssert(c->reply_bytes < SIZE_MAX-(1024*64));
if (c->reply_bytes == 0 || c->flags & CLIENT_CLOSE_ASAP) return; if (c->reply_bytes == 0 || c->flags & CLIENT_CLOSE_ASAP) return;
if (checkClientOutputBufferLimits(c)) { if (checkClientOutputBufferLimits(c)) {
...@@ -2336,20 +2524,29 @@ void flushSlavesOutputBuffers(void) { ...@@ -2336,20 +2524,29 @@ void flushSlavesOutputBuffers(void) {
listRewind(server.slaves,&li); listRewind(server.slaves,&li);
while((ln = listNext(&li))) { while((ln = listNext(&li))) {
client *slave = listNodeValue(ln); client *slave = listNodeValue(ln);
int events; int can_receive_writes = connHasWriteHandler(slave->conn) ||
(slave->flags & CLIENT_PENDING_WRITE);
/* Note that the following will not flush output buffers of slaves
* in STATE_ONLINE but having put_online_on_ack set to true: in this /* We don't want to send the pending data to the replica in a few
* case the writable event is never installed, since the purpose * cases:
* of put_online_on_ack is to postpone the moment it is installed. *
* This is what we want since slaves in this state should not receive * 1. For some reason there is neither the write handler installed
* writes before the first ACK. */ * nor the client is flagged as to have pending writes: for some
events = aeGetFileEvents(server.el,slave->fd); * reason this replica may not be set to receive data. This is
if (events & AE_WRITABLE && * just for the sake of defensive programming.
slave->replstate == SLAVE_STATE_ONLINE && *
* 2. The put_online_on_ack flag is true. To know why we don't want
* to send data to the replica in this case, please grep for the
* flag for this flag.
*
* 3. Obviously if the slave is not ONLINE.
*/
if (slave->replstate == SLAVE_STATE_ONLINE &&
can_receive_writes &&
!slave->repl_put_online_on_ack &&
clientHasPendingReplies(slave)) clientHasPendingReplies(slave))
{ {
writeToClient(slave->fd,slave,0); writeToClient(slave,0);
} }
} }
} }
...@@ -2428,3 +2625,276 @@ int processEventsWhileBlocked(void) { ...@@ -2428,3 +2625,276 @@ int processEventsWhileBlocked(void) {
} }
return count; return count;
} }
/* ==========================================================================
* Threaded I/O
* ========================================================================== */
int tio_debug = 0;
#define IO_THREADS_MAX_NUM 128
#define IO_THREADS_OP_READ 0
#define IO_THREADS_OP_WRITE 1
pthread_t io_threads[IO_THREADS_MAX_NUM];
pthread_mutex_t io_threads_mutex[IO_THREADS_MAX_NUM];
_Atomic unsigned long io_threads_pending[IO_THREADS_MAX_NUM];
int io_threads_active; /* Are the threads currently spinning waiting I/O? */
int io_threads_op; /* IO_THREADS_OP_WRITE or IO_THREADS_OP_READ. */
list *io_threads_list[IO_THREADS_MAX_NUM];
void *IOThreadMain(void *myid) {
/* The ID is the thread number (from 0 to server.iothreads_num-1), and is
* used by the thread to just manipulate a single sub-array of clients. */
long id = (unsigned long)myid;
while(1) {
/* Wait for start */
for (int j = 0; j < 1000000; j++) {
if (io_threads_pending[id] != 0) break;
}
/* Give the main thread a chance to stop this thread. */
if (io_threads_pending[id] == 0) {
pthread_mutex_lock(&io_threads_mutex[id]);
pthread_mutex_unlock(&io_threads_mutex[id]);
continue;
}
serverAssert(io_threads_pending[id] != 0);
if (tio_debug) printf("[%ld] %d to handle\n", id, (int)listLength(io_threads_list[id]));
/* Process: note that the main thread will never touch our list
* before we drop the pending count to 0. */
listIter li;
listNode *ln;
listRewind(io_threads_list[id],&li);
while((ln = listNext(&li))) {
client *c = listNodeValue(ln);
if (io_threads_op == IO_THREADS_OP_WRITE) {
writeToClient(c,0);
} else if (io_threads_op == IO_THREADS_OP_READ) {
readQueryFromClient(c->conn);
} else {
serverPanic("io_threads_op value is unknown");
}
}
listEmpty(io_threads_list[id]);
io_threads_pending[id] = 0;
if (tio_debug) printf("[%ld] Done\n", id);
}
}
/* Initialize the data structures needed for threaded I/O. */
void initThreadedIO(void) {
io_threads_active = 0; /* We start with threads not active. */
/* Don't spawn any thread if the user selected a single thread:
* we'll handle I/O directly from the main thread. */
if (server.io_threads_num == 1) return;
if (server.io_threads_num > IO_THREADS_MAX_NUM) {
serverLog(LL_WARNING,"Fatal: too many I/O threads configured. "
"The maximum number is %d.", IO_THREADS_MAX_NUM);
exit(1);
}
/* Spawn the I/O threads. */
for (int i = 0; i < server.io_threads_num; i++) {
pthread_t tid;
pthread_mutex_init(&io_threads_mutex[i],NULL);
io_threads_pending[i] = 0;
io_threads_list[i] = listCreate();
pthread_mutex_lock(&io_threads_mutex[i]); /* Thread will be stopped. */
if (pthread_create(&tid,NULL,IOThreadMain,(void*)(long)i) != 0) {
serverLog(LL_WARNING,"Fatal: Can't initialize IO thread.");
exit(1);
}
io_threads[i] = tid;
}
}
void startThreadedIO(void) {
if (tio_debug) { printf("S"); fflush(stdout); }
if (tio_debug) printf("--- STARTING THREADED IO ---\n");
serverAssert(io_threads_active == 0);
for (int j = 0; j < server.io_threads_num; j++)
pthread_mutex_unlock(&io_threads_mutex[j]);
io_threads_active = 1;
}
void stopThreadedIO(void) {
/* We may have still clients with pending reads when this function
* is called: handle them before stopping the threads. */
handleClientsWithPendingReadsUsingThreads();
if (tio_debug) { printf("E"); fflush(stdout); }
if (tio_debug) printf("--- STOPPING THREADED IO [R%d] [W%d] ---\n",
(int) listLength(server.clients_pending_read),
(int) listLength(server.clients_pending_write));
serverAssert(io_threads_active == 1);
for (int j = 0; j < server.io_threads_num; j++)
pthread_mutex_lock(&io_threads_mutex[j]);
io_threads_active = 0;
}
/* This function checks if there are not enough pending clients to justify
* taking the I/O threads active: in that case I/O threads are stopped if
* currently active. We track the pending writes as a measure of clients
* we need to handle in parallel, however the I/O threading is disabled
* globally for reads as well if we have too little pending clients.
*
* The function returns 0 if the I/O threading should be used becuase there
* are enough active threads, otherwise 1 is returned and the I/O threads
* could be possibly stopped (if already active) as a side effect. */
int stopThreadedIOIfNeeded(void) {
int pending = listLength(server.clients_pending_write);
/* Return ASAP if IO threads are disabled (single threaded mode). */
if (server.io_threads_num == 1) return 1;
if (pending < (server.io_threads_num*2)) {
if (io_threads_active) stopThreadedIO();
return 1;
} else {
return 0;
}
}
int handleClientsWithPendingWritesUsingThreads(void) {
int processed = listLength(server.clients_pending_write);
if (processed == 0) return 0; /* Return ASAP if there are no clients. */
/* If we have just a few clients to serve, don't use I/O threads, but the
* boring synchronous code. */
if (stopThreadedIOIfNeeded()) {
return handleClientsWithPendingWrites();
}
/* Start threads if needed. */
if (!io_threads_active) startThreadedIO();
if (tio_debug) printf("%d TOTAL WRITE pending clients\n", processed);
/* Distribute the clients across N different lists. */
listIter li;
listNode *ln;
listRewind(server.clients_pending_write,&li);
int item_id = 0;
while((ln = listNext(&li))) {
client *c = listNodeValue(ln);
c->flags &= ~CLIENT_PENDING_WRITE;
int target_id = item_id % server.io_threads_num;
listAddNodeTail(io_threads_list[target_id],c);
item_id++;
}
/* Give the start condition to the waiting threads, by setting the
* start condition atomic var. */
io_threads_op = IO_THREADS_OP_WRITE;
for (int j = 0; j < server.io_threads_num; j++) {
int count = listLength(io_threads_list[j]);
io_threads_pending[j] = count;
}
/* Wait for all threads to end their work. */
while(1) {
unsigned long pending = 0;
for (int j = 0; j < server.io_threads_num; j++)
pending += io_threads_pending[j];
if (pending == 0) break;
}
if (tio_debug) printf("I/O WRITE All threads finshed\n");
/* Run the list of clients again to install the write handler where
* needed. */
listRewind(server.clients_pending_write,&li);
while((ln = listNext(&li))) {
client *c = listNodeValue(ln);
/* Install the write handler if there are pending writes in some
* of the clients. */
if (clientHasPendingReplies(c) &&
connSetWriteHandler(c->conn, sendReplyToClient) == AE_ERR)
{
freeClientAsync(c);
}
}
listEmpty(server.clients_pending_write);
return processed;
}
/* Return 1 if we want to handle the client read later using threaded I/O.
* This is called by the readable handler of the event loop.
* As a side effect of calling this function the client is put in the
* pending read clients and flagged as such. */
int postponeClientRead(client *c) {
if (io_threads_active &&
server.io_threads_do_reads &&
!(c->flags & (CLIENT_MASTER|CLIENT_SLAVE|CLIENT_PENDING_READ)))
{
c->flags |= CLIENT_PENDING_READ;
listAddNodeHead(server.clients_pending_read,c);
return 1;
} else {
return 0;
}
}
/* When threaded I/O is also enabled for the reading + parsing side, the
* readable handler will just put normal clients into a queue of clients to
* process (instead of serving them synchronously). This function runs
* the queue using the I/O threads, and process them in order to accumulate
* the reads in the buffers, and also parse the first command available
* rendering it in the client structures. */
int handleClientsWithPendingReadsUsingThreads(void) {
if (!io_threads_active || !server.io_threads_do_reads) return 0;
int processed = listLength(server.clients_pending_read);
if (processed == 0) return 0;
if (tio_debug) printf("%d TOTAL READ pending clients\n", processed);
/* Distribute the clients across N different lists. */
listIter li;
listNode *ln;
listRewind(server.clients_pending_read,&li);
int item_id = 0;
while((ln = listNext(&li))) {
client *c = listNodeValue(ln);
int target_id = item_id % server.io_threads_num;
listAddNodeTail(io_threads_list[target_id],c);
item_id++;
}
/* Give the start condition to the waiting threads, by setting the
* start condition atomic var. */
io_threads_op = IO_THREADS_OP_READ;
for (int j = 0; j < server.io_threads_num; j++) {
int count = listLength(io_threads_list[j]);
io_threads_pending[j] = count;
}
/* Wait for all threads to end their work. */
while(1) {
unsigned long pending = 0;
for (int j = 0; j < server.io_threads_num; j++)
pending += io_threads_pending[j];
if (pending == 0) break;
}
if (tio_debug) printf("I/O READ All threads finshed\n");
/* Run the list of clients again to process the new buffers. */
listRewind(server.clients_pending_read,&li);
while((ln = listNext(&li))) {
client *c = listNodeValue(ln);
c->flags &= ~CLIENT_PENDING_READ;
if (c->flags & CLIENT_PENDING_COMMAND) {
c->flags &= ~ CLIENT_PENDING_COMMAND;
processCommandAndResetClient(c);
}
processInputBufferAndReplicate(c);
}
listEmpty(server.clients_pending_read);
return processed;
}
...@@ -55,6 +55,7 @@ int keyspaceEventsStringToFlags(char *classes) { ...@@ -55,6 +55,7 @@ int keyspaceEventsStringToFlags(char *classes) {
case 'K': flags |= NOTIFY_KEYSPACE; break; case 'K': flags |= NOTIFY_KEYSPACE; break;
case 'E': flags |= NOTIFY_KEYEVENT; break; case 'E': flags |= NOTIFY_KEYEVENT; break;
case 't': flags |= NOTIFY_STREAM; break; case 't': flags |= NOTIFY_STREAM; break;
case 'm': flags |= NOTIFY_KEY_MISS; break;
default: return -1; default: return -1;
} }
} }
...@@ -81,6 +82,7 @@ sds keyspaceEventsFlagsToString(int flags) { ...@@ -81,6 +82,7 @@ sds keyspaceEventsFlagsToString(int flags) {
if (flags & NOTIFY_EXPIRED) res = sdscatlen(res,"x",1); if (flags & NOTIFY_EXPIRED) res = sdscatlen(res,"x",1);
if (flags & NOTIFY_EVICTED) res = sdscatlen(res,"e",1); if (flags & NOTIFY_EVICTED) res = sdscatlen(res,"e",1);
if (flags & NOTIFY_STREAM) res = sdscatlen(res,"t",1); if (flags & NOTIFY_STREAM) res = sdscatlen(res,"t",1);
if (flags & NOTIFY_KEY_MISS) res = sdscatlen(res,"m",1);
} }
if (flags & NOTIFY_KEYSPACE) res = sdscatlen(res,"K",1); if (flags & NOTIFY_KEYSPACE) res = sdscatlen(res,"K",1);
if (flags & NOTIFY_KEYEVENT) res = sdscatlen(res,"E",1); if (flags & NOTIFY_KEYEVENT) res = sdscatlen(res,"E",1);
...@@ -100,12 +102,12 @@ void notifyKeyspaceEvent(int type, char *event, robj *key, int dbid) { ...@@ -100,12 +102,12 @@ void notifyKeyspaceEvent(int type, char *event, robj *key, int dbid) {
int len = -1; int len = -1;
char buf[24]; char buf[24];
/* If any modules are interested in events, notify the module system now. /* If any modules are interested in events, notify the module system now.
* This bypasses the notifications configuration, but the module engine * This bypasses the notifications configuration, but the module engine
* will only call event subscribers if the event type matches the types * will only call event subscribers if the event type matches the types
* they are interested in. */ * they are interested in. */
moduleNotifyKeyspaceEvent(type, event, key, dbid); moduleNotifyKeyspaceEvent(type, event, key, dbid);
/* If notifications for this class of events are off, return ASAP. */ /* If notifications for this class of events are off, return ASAP. */
if (!(server.notify_keyspace_events & type)) return; if (!(server.notify_keyspace_events & type)) return;
......
...@@ -467,10 +467,15 @@ robj *tryObjectEncoding(robj *o) { ...@@ -467,10 +467,15 @@ robj *tryObjectEncoding(robj *o) {
incrRefCount(shared.integers[value]); incrRefCount(shared.integers[value]);
return shared.integers[value]; return shared.integers[value];
} else { } else {
if (o->encoding == OBJ_ENCODING_RAW) sdsfree(o->ptr); if (o->encoding == OBJ_ENCODING_RAW) {
o->encoding = OBJ_ENCODING_INT; sdsfree(o->ptr);
o->ptr = (void*) value; o->encoding = OBJ_ENCODING_INT;
return o; o->ptr = (void*) value;
return o;
} else if (o->encoding == OBJ_ENCODING_EMBSTR) {
decrRefCount(o);
return createStringObjectFromLongLongForValue(value);
}
} }
} }
...@@ -834,7 +839,9 @@ size_t objectComputeSize(robj *o, size_t sample_size) { ...@@ -834,7 +839,9 @@ size_t objectComputeSize(robj *o, size_t sample_size) {
d = ((zset*)o->ptr)->dict; d = ((zset*)o->ptr)->dict;
zskiplist *zsl = ((zset*)o->ptr)->zsl; zskiplist *zsl = ((zset*)o->ptr)->zsl;
zskiplistNode *znode = zsl->header->level[0].forward; zskiplistNode *znode = zsl->header->level[0].forward;
asize = sizeof(*o)+sizeof(zset)+(sizeof(struct dictEntry*)*dictSlots(d)); asize = sizeof(*o)+sizeof(zset)+sizeof(zskiplist)+sizeof(dict)+
(sizeof(struct dictEntry*)*dictSlots(d))+
zmalloc_size(zsl->header);
while(znode != NULL && samples < sample_size) { while(znode != NULL && samples < sample_size) {
elesize += sdsAllocSize(znode->ele); elesize += sdsAllocSize(znode->ele);
elesize += sizeof(struct dictEntry) + zmalloc_size(znode); elesize += sizeof(struct dictEntry) + zmalloc_size(znode);
...@@ -1433,30 +1440,20 @@ NULL ...@@ -1433,30 +1440,20 @@ NULL
#if defined(USE_JEMALLOC) #if defined(USE_JEMALLOC)
sds info = sdsempty(); sds info = sdsempty();
je_malloc_stats_print(inputCatSds, &info, NULL); je_malloc_stats_print(inputCatSds, &info, NULL);
addReplyBulkSds(c, info); addReplyVerbatim(c,info,sdslen(info),"txt");
sdsfree(info);
#else #else
addReplyBulkCString(c,"Stats not supported for the current allocator"); addReplyBulkCString(c,"Stats not supported for the current allocator");
#endif #endif
} else if (!strcasecmp(c->argv[1]->ptr,"doctor") && c->argc == 2) { } else if (!strcasecmp(c->argv[1]->ptr,"doctor") && c->argc == 2) {
sds report = getMemoryDoctorReport(); sds report = getMemoryDoctorReport();
addReplyBulkSds(c,report); addReplyVerbatim(c,report,sdslen(report),"txt");
sdsfree(report);
} else if (!strcasecmp(c->argv[1]->ptr,"purge") && c->argc == 2) { } else if (!strcasecmp(c->argv[1]->ptr,"purge") && c->argc == 2) {
#if defined(USE_JEMALLOC) if (jemalloc_purge() == 0)
char tmp[32]; addReply(c, shared.ok);
unsigned narenas = 0; else
size_t sz = sizeof(unsigned); addReplyError(c, "Error purging dirty pages");
if (!je_mallctl("arenas.narenas", &narenas, &sz, NULL, 0)) {
sprintf(tmp, "arena.%d.purge", narenas);
if (!je_mallctl(tmp, NULL, 0, NULL, 0)) {
addReply(c, shared.ok);
return;
}
}
addReplyError(c, "Error purging dirty pages");
#else
addReply(c, shared.ok);
/* Nothing to do for other allocators. */
#endif
} else { } else {
addReplyErrorFormat(c, "Unknown subcommand or wrong number of arguments for '%s'. Try MEMORY HELP", (char*)c->argv[1]->ptr); addReplyErrorFormat(c, "Unknown subcommand or wrong number of arguments for '%s'. Try MEMORY HELP", (char*)c->argv[1]->ptr);
} }
......
...@@ -1791,7 +1791,8 @@ int raxCompare(raxIterator *iter, const char *op, unsigned char *key, size_t key ...@@ -1791,7 +1791,8 @@ int raxCompare(raxIterator *iter, const char *op, unsigned char *key, size_t key
if (eq && key_len == iter->key_len) return 1; if (eq && key_len == iter->key_len) return 1;
else if (lt) return iter->key_len < key_len; else if (lt) return iter->key_len < key_len;
else if (gt) return iter->key_len > key_len; else if (gt) return iter->key_len > key_len;
} if (cmp > 0) { return 0;
} else if (cmp > 0) {
return gt ? 1 : 0; return gt ? 1 : 0;
} else /* (cmp < 0) */ { } else /* (cmp < 0) */ {
return lt ? 1 : 0; return lt ? 1 : 0;
......
...@@ -42,30 +42,41 @@ ...@@ -42,30 +42,41 @@
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/param.h> #include <sys/param.h>
#define rdbExitReportCorruptRDB(...) rdbCheckThenExit(__LINE__,__VA_ARGS__) /* This macro is called when the internal RDB stracture is corrupt */
#define rdbExitReportCorruptRDB(...) rdbReportError(1, __LINE__,__VA_ARGS__)
/* This macro is called when RDB read failed (possibly a short read) */
#define rdbReportReadError(...) rdbReportError(0, __LINE__,__VA_ARGS__)
char* rdbFileBeingLoaded = NULL; /* used for rdb checking on read error */
extern int rdbCheckMode; extern int rdbCheckMode;
void rdbCheckError(const char *fmt, ...); void rdbCheckError(const char *fmt, ...);
void rdbCheckSetError(const char *fmt, ...); void rdbCheckSetError(const char *fmt, ...);
void rdbCheckThenExit(int linenum, char *reason, ...) { void rdbReportError(int corruption_error, int linenum, char *reason, ...) {
va_list ap; va_list ap;
char msg[1024]; char msg[1024];
int len; int len;
len = snprintf(msg,sizeof(msg), len = snprintf(msg,sizeof(msg),
"Internal error in RDB reading function at rdb.c:%d -> ", linenum); "Internal error in RDB reading offset %llu, function at rdb.c:%d -> ",
(unsigned long long)server.loading_loaded_bytes, linenum);
va_start(ap,reason); va_start(ap,reason);
vsnprintf(msg+len,sizeof(msg)-len,reason,ap); vsnprintf(msg+len,sizeof(msg)-len,reason,ap);
va_end(ap); va_end(ap);
if (!rdbCheckMode) { if (!rdbCheckMode) {
serverLog(LL_WARNING, "%s", msg); if (rdbFileBeingLoaded || corruption_error) {
char *argv[2] = {"",server.rdb_filename}; serverLog(LL_WARNING, "%s", msg);
redis_check_rdb_main(2,argv,NULL); char *argv[2] = {"",rdbFileBeingLoaded};
redis_check_rdb_main(2,argv,NULL);
} else {
serverLog(LL_WARNING, "%s. Failure loading rdb format from socket, assuming connection error, resuming operation.", msg);
return;
}
} else { } else {
rdbCheckError("%s",msg); rdbCheckError("%s",msg);
} }
serverLog(LL_WARNING, "Terminating server after rdb file reading failure.");
exit(1); exit(1);
} }
...@@ -75,18 +86,6 @@ static int rdbWriteRaw(rio *rdb, void *p, size_t len) { ...@@ -75,18 +86,6 @@ static int rdbWriteRaw(rio *rdb, void *p, size_t len) {
return len; return len;
} }
/* This is just a wrapper for the low level function rioRead() that will
* automatically abort if it is not possible to read the specified amount
* of bytes. */
void rdbLoadRaw(rio *rdb, void *buf, uint64_t len) {
if (rioRead(rdb,buf,len) == 0) {
rdbExitReportCorruptRDB(
"Impossible to read %llu bytes in rdbLoadRaw()",
(unsigned long long) len);
return; /* Not reached. */
}
}
int rdbSaveType(rio *rdb, unsigned char type) { int rdbSaveType(rio *rdb, unsigned char type) {
return rdbWriteRaw(rdb,&type,1); return rdbWriteRaw(rdb,&type,1);
} }
...@@ -102,10 +101,12 @@ int rdbLoadType(rio *rdb) { ...@@ -102,10 +101,12 @@ int rdbLoadType(rio *rdb) {
/* This is only used to load old databases stored with the RDB_OPCODE_EXPIRETIME /* This is only used to load old databases stored with the RDB_OPCODE_EXPIRETIME
* opcode. New versions of Redis store using the RDB_OPCODE_EXPIRETIME_MS * opcode. New versions of Redis store using the RDB_OPCODE_EXPIRETIME_MS
* opcode. */ * opcode. On error -1 is returned, however this could be a valid time, so
* to check for loading errors the caller should call rioGetReadError() after
* calling this function. */
time_t rdbLoadTime(rio *rdb) { time_t rdbLoadTime(rio *rdb) {
int32_t t32; int32_t t32;
rdbLoadRaw(rdb,&t32,4); if (rioRead(rdb,&t32,4) == 0) return -1;
return (time_t)t32; return (time_t)t32;
} }
...@@ -125,10 +126,14 @@ int rdbSaveMillisecondTime(rio *rdb, long long t) { ...@@ -125,10 +126,14 @@ int rdbSaveMillisecondTime(rio *rdb, long long t) {
* after upgrading to Redis version 5 they will no longer be able to load their * after upgrading to Redis version 5 they will no longer be able to load their
* own old RDB files. Because of that, we instead fix the function only for new * own old RDB files. Because of that, we instead fix the function only for new
* RDB versions, and load older RDB versions as we used to do in the past, * RDB versions, and load older RDB versions as we used to do in the past,
* allowing big endian systems to load their own old RDB files. */ * allowing big endian systems to load their own old RDB files.
*
* On I/O error the function returns LLONG_MAX, however if this is also a
* valid stored value, the caller should use rioGetReadError() to check for
* errors after calling this function. */
long long rdbLoadMillisecondTime(rio *rdb, int rdbver) { long long rdbLoadMillisecondTime(rio *rdb, int rdbver) {
int64_t t64; int64_t t64;
rdbLoadRaw(rdb,&t64,8); if (rioRead(rdb,&t64,8) == 0) return LLONG_MAX;
if (rdbver >= 9) /* Check the top comment of this function. */ if (rdbver >= 9) /* Check the top comment of this function. */
memrev64ifbe(&t64); /* Convert in big endian if the system is BE. */ memrev64ifbe(&t64); /* Convert in big endian if the system is BE. */
return (long long)t64; return (long long)t64;
...@@ -255,7 +260,7 @@ int rdbEncodeInteger(long long value, unsigned char *enc) { ...@@ -255,7 +260,7 @@ int rdbEncodeInteger(long long value, unsigned char *enc) {
/* Loads an integer-encoded object with the specified encoding type "enctype". /* Loads an integer-encoded object with the specified encoding type "enctype".
* The returned value changes according to the flags, see * The returned value changes according to the flags, see
* rdbGenerincLoadStringObject() for more info. */ * rdbGenericLoadStringObject() for more info. */
void *rdbLoadIntegerObject(rio *rdb, int enctype, int flags, size_t *lenptr) { void *rdbLoadIntegerObject(rio *rdb, int enctype, int flags, size_t *lenptr) {
int plain = flags & RDB_LOAD_PLAIN; int plain = flags & RDB_LOAD_PLAIN;
int sds = flags & RDB_LOAD_SDS; int sds = flags & RDB_LOAD_SDS;
...@@ -277,8 +282,8 @@ void *rdbLoadIntegerObject(rio *rdb, int enctype, int flags, size_t *lenptr) { ...@@ -277,8 +282,8 @@ void *rdbLoadIntegerObject(rio *rdb, int enctype, int flags, size_t *lenptr) {
v = enc[0]|(enc[1]<<8)|(enc[2]<<16)|(enc[3]<<24); v = enc[0]|(enc[1]<<8)|(enc[2]<<16)|(enc[3]<<24);
val = (int32_t)v; val = (int32_t)v;
} else { } else {
val = 0; /* anti-warning */
rdbExitReportCorruptRDB("Unknown RDB integer encoding type %d",enctype); rdbExitReportCorruptRDB("Unknown RDB integer encoding type %d",enctype);
return NULL; /* Never reached. */
} }
if (plain || sds) { if (plain || sds) {
char buf[LONG_STR_SIZE], *p; char buf[LONG_STR_SIZE], *p;
...@@ -381,8 +386,7 @@ void *rdbLoadLzfStringObject(rio *rdb, int flags, size_t *lenptr) { ...@@ -381,8 +386,7 @@ void *rdbLoadLzfStringObject(rio *rdb, int flags, size_t *lenptr) {
/* Load the compressed representation and uncompress it to target. */ /* Load the compressed representation and uncompress it to target. */
if (rioRead(rdb,c,clen) == 0) goto err; if (rioRead(rdb,c,clen) == 0) goto err;
if (lzf_decompress(c,clen,val,len) == 0) { if (lzf_decompress(c,clen,val,len) == 0) {
if (rdbCheckMode) rdbCheckSetError("Invalid LZF compressed string"); rdbExitReportCorruptRDB("Invalid LZF compressed string");
goto err;
} }
zfree(c); zfree(c);
...@@ -496,6 +500,7 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags, size_t *lenptr) { ...@@ -496,6 +500,7 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags, size_t *lenptr) {
return rdbLoadLzfStringObject(rdb,flags,lenptr); return rdbLoadLzfStringObject(rdb,flags,lenptr);
default: default:
rdbExitReportCorruptRDB("Unknown RDB string encoding type %d",len); rdbExitReportCorruptRDB("Unknown RDB string encoding type %d",len);
return NULL; /* Never reached. */
} }
} }
...@@ -751,7 +756,7 @@ size_t rdbSaveStreamConsumers(rio *rdb, streamCG *cg) { ...@@ -751,7 +756,7 @@ size_t rdbSaveStreamConsumers(rio *rdb, streamCG *cg) {
/* Save a Redis object. /* Save a Redis object.
* Returns -1 on error, number of bytes written on success. */ * Returns -1 on error, number of bytes written on success. */
ssize_t rdbSaveObject(rio *rdb, robj *o) { ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key) {
ssize_t n = 0, nwritten = 0; ssize_t n = 0, nwritten = 0;
if (o->type == OBJ_STRING) { if (o->type == OBJ_STRING) {
...@@ -966,7 +971,6 @@ ssize_t rdbSaveObject(rio *rdb, robj *o) { ...@@ -966,7 +971,6 @@ ssize_t rdbSaveObject(rio *rdb, robj *o) {
RedisModuleIO io; RedisModuleIO io;
moduleValue *mv = o->ptr; moduleValue *mv = o->ptr;
moduleType *mt = mv->type; moduleType *mt = mv->type;
moduleInitIOContext(io,mt,rdb);
/* Write the "module" identifier as prefix, so that we'll be able /* Write the "module" identifier as prefix, so that we'll be able
* to call the right module during loading. */ * to call the right module during loading. */
...@@ -975,10 +979,13 @@ ssize_t rdbSaveObject(rio *rdb, robj *o) { ...@@ -975,10 +979,13 @@ ssize_t rdbSaveObject(rio *rdb, robj *o) {
io.bytes += retval; io.bytes += retval;
/* Then write the module-specific representation + EOF marker. */ /* Then write the module-specific representation + EOF marker. */
moduleInitIOContext(io,mt,rdb,key);
mt->rdb_save(&io,mv->value); mt->rdb_save(&io,mv->value);
retval = rdbSaveLen(rdb,RDB_MODULE_OPCODE_EOF); retval = rdbSaveLen(rdb,RDB_MODULE_OPCODE_EOF);
if (retval == -1) return -1; if (retval == -1)
io.bytes += retval; io.error = 1;
else
io.bytes += retval;
if (io.ctx) { if (io.ctx) {
moduleFreeContext(io.ctx); moduleFreeContext(io.ctx);
...@@ -996,7 +1003,7 @@ ssize_t rdbSaveObject(rio *rdb, robj *o) { ...@@ -996,7 +1003,7 @@ ssize_t rdbSaveObject(rio *rdb, robj *o) {
* this length with very little changes to the code. In the future * this length with very little changes to the code. In the future
* we could switch to a faster solution. */ * we could switch to a faster solution. */
size_t rdbSavedObjectLen(robj *o) { size_t rdbSavedObjectLen(robj *o) {
ssize_t len = rdbSaveObject(NULL,o); ssize_t len = rdbSaveObject(NULL,o,NULL);
serverAssertWithInfo(NULL,o,len != -1); serverAssertWithInfo(NULL,o,len != -1);
return len; return len;
} }
...@@ -1038,7 +1045,12 @@ int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime) { ...@@ -1038,7 +1045,12 @@ int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime) {
/* Save type, key, value */ /* Save type, key, value */
if (rdbSaveObjectType(rdb,val) == -1) return -1; if (rdbSaveObjectType(rdb,val) == -1) return -1;
if (rdbSaveStringObject(rdb,key) == -1) return -1; if (rdbSaveStringObject(rdb,key) == -1) return -1;
if (rdbSaveObject(rdb,val) == -1) return -1; if (rdbSaveObject(rdb,val,key) == -1) return -1;
/* Delay return if required (for testing) */
if (server.rdb_key_save_delay)
usleep(server.rdb_key_save_delay);
return 1; return 1;
} }
...@@ -1091,6 +1103,45 @@ int rdbSaveInfoAuxFields(rio *rdb, int flags, rdbSaveInfo *rsi) { ...@@ -1091,6 +1103,45 @@ int rdbSaveInfoAuxFields(rio *rdb, int flags, rdbSaveInfo *rsi) {
return 1; return 1;
} }
ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt) {
/* Save a module-specific aux value. */
RedisModuleIO io;
int retval = rdbSaveType(rdb, RDB_OPCODE_MODULE_AUX);
/* Write the "module" identifier as prefix, so that we'll be able
* to call the right module during loading. */
retval = rdbSaveLen(rdb,mt->id);
if (retval == -1) return -1;
io.bytes += retval;
/* write the 'when' so that we can provide it on loading. add a UINT opcode
* for backwards compatibility, everything after the MT needs to be prefixed
* by an opcode. */
retval = rdbSaveLen(rdb,RDB_MODULE_OPCODE_UINT);
if (retval == -1) return -1;
io.bytes += retval;
retval = rdbSaveLen(rdb,when);
if (retval == -1) return -1;
io.bytes += retval;
/* Then write the module-specific representation + EOF marker. */
moduleInitIOContext(io,mt,rdb,NULL);
mt->aux_save(&io,when);
retval = rdbSaveLen(rdb,RDB_MODULE_OPCODE_EOF);
if (retval == -1)
io.error = 1;
else
io.bytes += retval;
if (io.ctx) {
moduleFreeContext(io.ctx);
zfree(io.ctx);
}
if (io.error)
return -1;
return io.bytes;
}
/* Produces a dump of the database in RDB format sending it to the specified /* Produces a dump of the database in RDB format sending it to the specified
* Redis I/O channel. On success C_OK is returned, otherwise C_ERR * Redis I/O channel. On success C_OK is returned, otherwise C_ERR
* is returned and part of the output, or all the output, can be * is returned and part of the output, or all the output, can be
...@@ -1112,6 +1163,7 @@ int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi) { ...@@ -1112,6 +1163,7 @@ int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi) {
snprintf(magic,sizeof(magic),"REDIS%04d",RDB_VERSION); snprintf(magic,sizeof(magic),"REDIS%04d",RDB_VERSION);
if (rdbWriteRaw(rdb,magic,9) == -1) goto werr; if (rdbWriteRaw(rdb,magic,9) == -1) goto werr;
if (rdbSaveInfoAuxFields(rdb,flags,rsi) == -1) goto werr; if (rdbSaveInfoAuxFields(rdb,flags,rsi) == -1) goto werr;
if (rdbSaveModulesAux(rdb, REDISMODULE_AUX_BEFORE_RDB) == -1) goto werr;
for (j = 0; j < server.dbnum; j++) { for (j = 0; j < server.dbnum; j++) {
redisDb *db = server.db+j; redisDb *db = server.db+j;
...@@ -1173,6 +1225,8 @@ int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi) { ...@@ -1173,6 +1225,8 @@ int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi) {
di = NULL; /* So that we don't release it again on error. */ di = NULL; /* So that we don't release it again on error. */
} }
if (rdbSaveModulesAux(rdb, REDISMODULE_AUX_AFTER_RDB) == -1) goto werr;
/* EOF opcode */ /* EOF opcode */
if (rdbSaveType(rdb,RDB_OPCODE_EOF) == -1) goto werr; if (rdbSaveType(rdb,RDB_OPCODE_EOF) == -1) goto werr;
...@@ -1281,40 +1335,25 @@ werr: ...@@ -1281,40 +1335,25 @@ werr:
int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) {
pid_t childpid; pid_t childpid;
long long start;
if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) return C_ERR; if (hasActiveChildProcess()) return C_ERR;
server.dirty_before_bgsave = server.dirty; server.dirty_before_bgsave = server.dirty;
server.lastbgsave_try = time(NULL); server.lastbgsave_try = time(NULL);
openChildInfoPipe(); openChildInfoPipe();
start = ustime(); if ((childpid = redisFork()) == 0) {
if ((childpid = fork()) == 0) {
int retval; int retval;
/* Child */ /* Child */
closeListeningSockets(0);
redisSetProcTitle("redis-rdb-bgsave"); redisSetProcTitle("redis-rdb-bgsave");
retval = rdbSave(filename,rsi); retval = rdbSave(filename,rsi);
if (retval == C_OK) { if (retval == C_OK) {
size_t private_dirty = zmalloc_get_private_dirty(-1); sendChildCOWInfo(CHILD_INFO_TYPE_RDB, "RDB");
if (private_dirty) {
serverLog(LL_NOTICE,
"RDB: %zu MB of memory used by copy-on-write",
private_dirty/(1024*1024));
}
server.child_info_data.cow_size = private_dirty;
sendChildInfo(CHILD_INFO_TYPE_RDB);
} }
exitFromChild((retval == C_OK) ? 0 : 1); exitFromChild((retval == C_OK) ? 0 : 1);
} else { } else {
/* Parent */ /* Parent */
server.stat_fork_time = ustime()-start;
server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */
latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000);
if (childpid == -1) { if (childpid == -1) {
closeChildInfoPipe(); closeChildInfoPipe();
server.lastbgsave_status = C_ERR; server.lastbgsave_status = C_ERR;
...@@ -1326,7 +1365,6 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { ...@@ -1326,7 +1365,6 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) {
server.rdb_save_time_start = time(NULL); server.rdb_save_time_start = time(NULL);
server.rdb_child_pid = childpid; server.rdb_child_pid = childpid;
server.rdb_child_type = RDB_CHILD_TYPE_DISK; server.rdb_child_type = RDB_CHILD_TYPE_DISK;
updateDictResizePolicy();
return C_OK; return C_OK;
} }
return C_OK; /* unreached */ return C_OK; /* unreached */
...@@ -1380,7 +1418,7 @@ robj *rdbLoadCheckModuleValue(rio *rdb, char *modulename) { ...@@ -1380,7 +1418,7 @@ robj *rdbLoadCheckModuleValue(rio *rdb, char *modulename) {
/* Load a Redis object of the specified type from the specified file. /* Load a Redis object of the specified type from the specified file.
* On success a newly allocated object is returned, otherwise NULL. */ * On success a newly allocated object is returned, otherwise NULL. */
robj *rdbLoadObject(int rdbtype, rio *rdb) { robj *rdbLoadObject(int rdbtype, rio *rdb, robj *key) {
robj *o = NULL, *ele, *dec; robj *o = NULL, *ele, *dec;
uint64_t len; uint64_t len;
unsigned int i; unsigned int i;
...@@ -1632,6 +1670,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -1632,6 +1670,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
hashTypeConvert(o, OBJ_ENCODING_HT); hashTypeConvert(o, OBJ_ENCODING_HT);
break; break;
default: default:
/* totally unreachable */
rdbExitReportCorruptRDB("Unknown RDB encoding type %d",rdbtype); rdbExitReportCorruptRDB("Unknown RDB encoding type %d",rdbtype);
break; break;
} }
...@@ -1639,6 +1678,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -1639,6 +1678,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
o = createStreamObject(); o = createStreamObject();
stream *s = o->ptr; stream *s = o->ptr;
uint64_t listpacks = rdbLoadLen(rdb,NULL); uint64_t listpacks = rdbLoadLen(rdb,NULL);
if (listpacks == RDB_LENERR) {
rdbReportReadError("Stream listpacks len loading failed.");
decrRefCount(o);
return NULL;
}
while(listpacks--) { while(listpacks--) {
/* Get the master ID, the one we'll use as key of the radix tree /* Get the master ID, the one we'll use as key of the radix tree
...@@ -1646,7 +1690,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -1646,7 +1690,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
* relatively to this ID. */ * relatively to this ID. */
sds nodekey = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL); sds nodekey = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL);
if (nodekey == NULL) { if (nodekey == NULL) {
rdbExitReportCorruptRDB("Stream master ID loading failed: invalid encoding or I/O error."); rdbReportReadError("Stream master ID loading failed: invalid encoding or I/O error.");
decrRefCount(o);
return NULL;
} }
if (sdslen(nodekey) != sizeof(streamID)) { if (sdslen(nodekey) != sizeof(streamID)) {
rdbExitReportCorruptRDB("Stream node key entry is not the " rdbExitReportCorruptRDB("Stream node key entry is not the "
...@@ -1656,7 +1702,12 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -1656,7 +1702,12 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
/* Load the listpack. */ /* Load the listpack. */
unsigned char *lp = unsigned char *lp =
rdbGenericLoadStringObject(rdb,RDB_LOAD_PLAIN,NULL); rdbGenericLoadStringObject(rdb,RDB_LOAD_PLAIN,NULL);
if (lp == NULL) return NULL; if (lp == NULL) {
rdbReportReadError("Stream listpacks loading failed.");
sdsfree(nodekey);
decrRefCount(o);
return NULL;
}
unsigned char *first = lpFirst(lp); unsigned char *first = lpFirst(lp);
if (first == NULL) { if (first == NULL) {
/* Serialized listpacks should never be empty, since on /* Serialized listpacks should never be empty, since on
...@@ -1674,12 +1725,24 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -1674,12 +1725,24 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
} }
/* Load total number of items inside the stream. */ /* Load total number of items inside the stream. */
s->length = rdbLoadLen(rdb,NULL); s->length = rdbLoadLen(rdb,NULL);
/* Load the last entry ID. */ /* Load the last entry ID. */
s->last_id.ms = rdbLoadLen(rdb,NULL); s->last_id.ms = rdbLoadLen(rdb,NULL);
s->last_id.seq = rdbLoadLen(rdb,NULL); s->last_id.seq = rdbLoadLen(rdb,NULL);
if (rioGetReadError(rdb)) {
rdbReportReadError("Stream object metadata loading failed.");
decrRefCount(o);
return NULL;
}
/* Consumer groups loading */ /* Consumer groups loading */
size_t cgroups_count = rdbLoadLen(rdb,NULL); uint64_t cgroups_count = rdbLoadLen(rdb,NULL);
if (cgroups_count == RDB_LENERR) {
rdbReportReadError("Stream cgroup count loading failed.");
decrRefCount(o);
return NULL;
}
while(cgroups_count--) { while(cgroups_count--) {
/* Get the consumer group name and ID. We can then create the /* Get the consumer group name and ID. We can then create the
* consumer group ASAP and populate its structure as * consumer group ASAP and populate its structure as
...@@ -1687,11 +1750,21 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -1687,11 +1750,21 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
streamID cg_id; streamID cg_id;
sds cgname = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL); sds cgname = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL);
if (cgname == NULL) { if (cgname == NULL) {
rdbExitReportCorruptRDB( rdbReportReadError(
"Error reading the consumer group name from Stream"); "Error reading the consumer group name from Stream");
decrRefCount(o);
return NULL;
} }
cg_id.ms = rdbLoadLen(rdb,NULL); cg_id.ms = rdbLoadLen(rdb,NULL);
cg_id.seq = rdbLoadLen(rdb,NULL); cg_id.seq = rdbLoadLen(rdb,NULL);
if (rioGetReadError(rdb)) {
rdbReportReadError("Stream cgroup ID loading failed.");
sdsfree(cgname);
decrRefCount(o);
return NULL;
}
streamCG *cgroup = streamCreateCG(s,cgname,sdslen(cgname),&cg_id); streamCG *cgroup = streamCreateCG(s,cgname,sdslen(cgname),&cg_id);
if (cgroup == NULL) if (cgroup == NULL)
rdbExitReportCorruptRDB("Duplicated consumer group name %s", rdbExitReportCorruptRDB("Duplicated consumer group name %s",
...@@ -1703,13 +1776,28 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -1703,13 +1776,28 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
* owner, since consumers for this group and their messages will * owner, since consumers for this group and their messages will
* be read as a next step. So for now leave them not resolved * be read as a next step. So for now leave them not resolved
* and later populate it. */ * and later populate it. */
size_t pel_size = rdbLoadLen(rdb,NULL); uint64_t pel_size = rdbLoadLen(rdb,NULL);
if (pel_size == RDB_LENERR) {
rdbReportReadError("Stream PEL size loading failed.");
decrRefCount(o);
return NULL;
}
while(pel_size--) { while(pel_size--) {
unsigned char rawid[sizeof(streamID)]; unsigned char rawid[sizeof(streamID)];
rdbLoadRaw(rdb,rawid,sizeof(rawid)); if (rioRead(rdb,rawid,sizeof(rawid)) == 0) {
rdbReportReadError("Stream PEL ID loading failed.");
decrRefCount(o);
return NULL;
}
streamNACK *nack = streamCreateNACK(NULL); streamNACK *nack = streamCreateNACK(NULL);
nack->delivery_time = rdbLoadMillisecondTime(rdb,RDB_VERSION); nack->delivery_time = rdbLoadMillisecondTime(rdb,RDB_VERSION);
nack->delivery_count = rdbLoadLen(rdb,NULL); nack->delivery_count = rdbLoadLen(rdb,NULL);
if (rioGetReadError(rdb)) {
rdbReportReadError("Stream PEL NACK loading failed.");
decrRefCount(o);
streamFreeNACK(nack);
return NULL;
}
if (!raxInsert(cgroup->pel,rawid,sizeof(rawid),nack,NULL)) if (!raxInsert(cgroup->pel,rawid,sizeof(rawid),nack,NULL))
rdbExitReportCorruptRDB("Duplicated gobal PEL entry " rdbExitReportCorruptRDB("Duplicated gobal PEL entry "
"loading stream consumer group"); "loading stream consumer group");
...@@ -1717,24 +1805,47 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -1717,24 +1805,47 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
/* Now that we loaded our global PEL, we need to load the /* Now that we loaded our global PEL, we need to load the
* consumers and their local PELs. */ * consumers and their local PELs. */
size_t consumers_num = rdbLoadLen(rdb,NULL); uint64_t consumers_num = rdbLoadLen(rdb,NULL);
if (consumers_num == RDB_LENERR) {
rdbReportReadError("Stream consumers num loading failed.");
decrRefCount(o);
return NULL;
}
while(consumers_num--) { while(consumers_num--) {
sds cname = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL); sds cname = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL);
if (cname == NULL) { if (cname == NULL) {
rdbExitReportCorruptRDB( rdbReportReadError(
"Error reading the consumer name from Stream group"); "Error reading the consumer name from Stream group.");
decrRefCount(o);
return NULL;
} }
streamConsumer *consumer = streamLookupConsumer(cgroup,cname, streamConsumer *consumer = streamLookupConsumer(cgroup,cname,
1); 1);
sdsfree(cname); sdsfree(cname);
consumer->seen_time = rdbLoadMillisecondTime(rdb,RDB_VERSION); consumer->seen_time = rdbLoadMillisecondTime(rdb,RDB_VERSION);
if (rioGetReadError(rdb)) {
rdbReportReadError("Stream short read reading seen time.");
decrRefCount(o);
return NULL;
}
/* Load the PEL about entries owned by this specific /* Load the PEL about entries owned by this specific
* consumer. */ * consumer. */
pel_size = rdbLoadLen(rdb,NULL); pel_size = rdbLoadLen(rdb,NULL);
if (pel_size == RDB_LENERR) {
rdbReportReadError(
"Stream consumer PEL num loading failed.");
decrRefCount(o);
return NULL;
}
while(pel_size--) { while(pel_size--) {
unsigned char rawid[sizeof(streamID)]; unsigned char rawid[sizeof(streamID)];
rdbLoadRaw(rdb,rawid,sizeof(rawid)); if (rioRead(rdb,rawid,sizeof(rawid)) == 0) {
rdbReportReadError(
"Stream short read reading PEL streamID.");
decrRefCount(o);
return NULL;
}
streamNACK *nack = raxFind(cgroup->pel,rawid,sizeof(rawid)); streamNACK *nack = raxFind(cgroup->pel,rawid,sizeof(rawid));
if (nack == raxNotFound) if (nack == raxNotFound)
rdbExitReportCorruptRDB("Consumer entry not found in " rdbExitReportCorruptRDB("Consumer entry not found in "
...@@ -1753,6 +1864,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -1753,6 +1864,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
} }
} else if (rdbtype == RDB_TYPE_MODULE || rdbtype == RDB_TYPE_MODULE_2) { } else if (rdbtype == RDB_TYPE_MODULE || rdbtype == RDB_TYPE_MODULE_2) {
uint64_t moduleid = rdbLoadLen(rdb,NULL); uint64_t moduleid = rdbLoadLen(rdb,NULL);
if (rioGetReadError(rdb)) return NULL;
moduleType *mt = moduleTypeLookupModuleByID(moduleid); moduleType *mt = moduleTypeLookupModuleByID(moduleid);
char name[10]; char name[10];
...@@ -1767,7 +1879,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -1767,7 +1879,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
exit(1); exit(1);
} }
RedisModuleIO io; RedisModuleIO io;
moduleInitIOContext(io,mt,rdb); moduleInitIOContext(io,mt,rdb,key);
io.ver = (rdbtype == RDB_TYPE_MODULE) ? 1 : 2; io.ver = (rdbtype == RDB_TYPE_MODULE) ? 1 : 2;
/* Call the rdb_load method of the module providing the 10 bit /* Call the rdb_load method of the module providing the 10 bit
* encoding version in the lower 10 bits of the module ID. */ * encoding version in the lower 10 bits of the module ID. */
...@@ -1780,6 +1892,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -1780,6 +1892,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
/* Module v2 serialization has an EOF mark at the end. */ /* Module v2 serialization has an EOF mark at the end. */
if (io.ver == 2) { if (io.ver == 2) {
uint64_t eof = rdbLoadLen(rdb,NULL); uint64_t eof = rdbLoadLen(rdb,NULL);
if (eof == RDB_LENERR) {
o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */
decrRefCount(o);
return NULL;
}
if (eof != RDB_MODULE_OPCODE_EOF) { if (eof != RDB_MODULE_OPCODE_EOF) {
serverLog(LL_WARNING,"The RDB file contains module data for the module '%s' that is not terminated by the proper module value EOF marker", name); serverLog(LL_WARNING,"The RDB file contains module data for the module '%s' that is not terminated by the proper module value EOF marker", name);
exit(1); exit(1);
...@@ -1793,25 +1910,31 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { ...@@ -1793,25 +1910,31 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
} }
o = createModuleObject(mt,ptr); o = createModuleObject(mt,ptr);
} else { } else {
rdbExitReportCorruptRDB("Unknown RDB encoding type %d",rdbtype); rdbReportReadError("Unknown RDB encoding type %d",rdbtype);
return NULL;
} }
return o; return o;
} }
/* Mark that we are loading in the global state and setup the fields /* Mark that we are loading in the global state and setup the fields
* needed to provide loading stats. */ * needed to provide loading stats. */
void startLoading(FILE *fp) { void startLoading(size_t size) {
struct stat sb;
/* Load the DB */ /* Load the DB */
server.loading = 1; server.loading = 1;
server.loading_start_time = time(NULL); server.loading_start_time = time(NULL);
server.loading_loaded_bytes = 0; server.loading_loaded_bytes = 0;
if (fstat(fileno(fp), &sb) == -1) { server.loading_total_bytes = size;
server.loading_total_bytes = 0; }
} else {
server.loading_total_bytes = sb.st_size; /* Mark that we are loading in the global state and setup the fields
} * needed to provide loading stats.
* 'filename' is optional and used for rdb-check on error */
void startLoadingFile(FILE *fp, char* filename) {
struct stat sb;
if (fstat(fileno(fp), &sb) == -1)
sb.st_size = 0;
rdbFileBeingLoaded = filename;
startLoading(sb.st_size);
} }
/* Refresh the loading progress info */ /* Refresh the loading progress info */
...@@ -1824,6 +1947,7 @@ void loadingProgress(off_t pos) { ...@@ -1824,6 +1947,7 @@ void loadingProgress(off_t pos) {
/* Loading finished */ /* Loading finished */
void stopLoading(void) { void stopLoading(void) {
server.loading = 0; server.loading = 0;
rdbFileBeingLoaded = NULL;
} }
/* Track loading progress in order to serve client's from time to time /* Track loading progress in order to serve client's from time to time
...@@ -1886,11 +2010,13 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { ...@@ -1886,11 +2010,13 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) {
* load the actual type, and continue. */ * load the actual type, and continue. */
expiretime = rdbLoadTime(rdb); expiretime = rdbLoadTime(rdb);
expiretime *= 1000; expiretime *= 1000;
if (rioGetReadError(rdb)) goto eoferr;
continue; /* Read next opcode. */ continue; /* Read next opcode. */
} else if (type == RDB_OPCODE_EXPIRETIME_MS) { } else if (type == RDB_OPCODE_EXPIRETIME_MS) {
/* EXPIRETIME_MS: milliseconds precision expire times introduced /* EXPIRETIME_MS: milliseconds precision expire times introduced
* with RDB v3. Like EXPIRETIME but no with more precision. */ * with RDB v3. Like EXPIRETIME but no with more precision. */
expiretime = rdbLoadMillisecondTime(rdb,rdbver); expiretime = rdbLoadMillisecondTime(rdb,rdbver);
if (rioGetReadError(rdb)) goto eoferr;
continue; /* Read next opcode. */ continue; /* Read next opcode. */
} else if (type == RDB_OPCODE_FREQ) { } else if (type == RDB_OPCODE_FREQ) {
/* FREQ: LFU frequency. */ /* FREQ: LFU frequency. */
...@@ -1991,15 +2117,15 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { ...@@ -1991,15 +2117,15 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) {
decrRefCount(auxval); decrRefCount(auxval);
continue; /* Read type again. */ continue; /* Read type again. */
} else if (type == RDB_OPCODE_MODULE_AUX) { } else if (type == RDB_OPCODE_MODULE_AUX) {
/* This is just for compatibility with the future: we have plans /* Load module data that is not related to the Redis key space.
* to add the ability for modules to store anything in the RDB * Such data can be potentially be stored both before and after the
* file, like data that is not related to the Redis key space. * RDB keys-values section. */
* Such data will potentially be stored both before and after the
* RDB keys-values section. For this reason since RDB version 9,
* we have the ability to read a MODULE_AUX opcode followed by an
* identifier of the module, and a serialized value in "MODULE V2"
* format. */
uint64_t moduleid = rdbLoadLen(rdb,NULL); uint64_t moduleid = rdbLoadLen(rdb,NULL);
int when_opcode = rdbLoadLen(rdb,NULL);
int when = rdbLoadLen(rdb,NULL);
if (rioGetReadError(rdb)) goto eoferr;
if (when_opcode != RDB_MODULE_OPCODE_UINT)
rdbReportReadError("bad when_opcode");
moduleType *mt = moduleTypeLookupModuleByID(moduleid); moduleType *mt = moduleTypeLookupModuleByID(moduleid);
char name[10]; char name[10];
moduleTypeNameByID(name,moduleid); moduleTypeNameByID(name,moduleid);
...@@ -2009,21 +2135,44 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { ...@@ -2009,21 +2135,44 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) {
serverLog(LL_WARNING,"The RDB file contains AUX module data I can't load: no matching module '%s'", name); serverLog(LL_WARNING,"The RDB file contains AUX module data I can't load: no matching module '%s'", name);
exit(1); exit(1);
} else if (!rdbCheckMode && mt != NULL) { } else if (!rdbCheckMode && mt != NULL) {
/* This version of Redis actually does not know what to do if (!mt->aux_load) {
* with modules AUX data... */ /* Module doesn't support AUX. */
serverLog(LL_WARNING,"The RDB file contains AUX module data I can't load for the module '%s'. Probably you want to use a newer version of Redis which implements aux data callbacks", name); serverLog(LL_WARNING,"The RDB file contains module AUX data, but the module '%s' doesn't seem to support it.", name);
exit(1); exit(1);
}
RedisModuleIO io;
moduleInitIOContext(io,mt,rdb,NULL);
io.ver = 2;
/* Call the rdb_load method of the module providing the 10 bit
* encoding version in the lower 10 bits of the module ID. */
if (mt->aux_load(&io,moduleid&1023, when) || io.error) {
moduleTypeNameByID(name,moduleid);
serverLog(LL_WARNING,"The RDB file contains module AUX data for the module type '%s', that the responsible module is not able to load. Check for modules log above for additional clues.", name);
exit(1);
}
if (io.ctx) {
moduleFreeContext(io.ctx);
zfree(io.ctx);
}
uint64_t eof = rdbLoadLen(rdb,NULL);
if (eof != RDB_MODULE_OPCODE_EOF) {
serverLog(LL_WARNING,"The RDB file contains module AUX data for the module '%s' that is not terminated by the proper module value EOF marker", name);
exit(1);
}
continue;
} else { } else {
/* RDB check mode. */ /* RDB check mode. */
robj *aux = rdbLoadCheckModuleValue(rdb,name); robj *aux = rdbLoadCheckModuleValue(rdb,name);
decrRefCount(aux); decrRefCount(aux);
continue; /* Read next opcode. */
} }
} }
/* Read key */ /* Read key */
if ((key = rdbLoadStringObject(rdb)) == NULL) goto eoferr; if ((key = rdbLoadStringObject(rdb)) == NULL) goto eoferr;
/* Read value */ /* Read value */
if ((val = rdbLoadObject(type,rdb)) == NULL) goto eoferr; if ((val = rdbLoadObject(type,rdb,key)) == NULL) goto eoferr;
/* Check if the key already expired. This function is used when loading /* Check if the key already expired. This function is used when loading
* an RDB file from disk, either at startup, or when an RDB was * an RDB file from disk, either at startup, or when an RDB was
* received from the master. In the latter case, the master is * received from the master. In the latter case, the master is
...@@ -2046,6 +2195,8 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { ...@@ -2046,6 +2195,8 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) {
* own reference. */ * own reference. */
decrRefCount(key); decrRefCount(key);
} }
if (server.key_load_delay)
usleep(server.key_load_delay);
/* Reset the state that is key-specified and is populated by /* Reset the state that is key-specified and is populated by
* opcodes before the key, so that we start from scratch again. */ * opcodes before the key, so that we start from scratch again. */
...@@ -2070,10 +2221,15 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { ...@@ -2070,10 +2221,15 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) {
} }
return C_OK; return C_OK;
eoferr: /* unexpected end of file is handled here with a fatal exit */ /* Unexpected end of file is handled here calling rdbReportReadError():
serverLog(LL_WARNING,"Short read or OOM loading DB. Unrecoverable error, aborting now."); * this will in turn either abort Redis in most cases, or if we are loading
rdbExitReportCorruptRDB("Unexpected EOF reading RDB file"); * the RDB file from a socket during initial SYNC (diskless replica mode),
return C_ERR; /* Just to avoid warning */ * we'll report the error to the caller, so that we can retry. */
eoferr:
serverLog(LL_WARNING,
"Short read or OOM loading DB. Unrecoverable error, aborting now.");
rdbReportReadError("Unexpected EOF reading RDB file");
return C_ERR;
} }
/* Like rdbLoadRio() but takes a filename instead of a rio stream. The /* Like rdbLoadRio() but takes a filename instead of a rio stream. The
...@@ -2089,7 +2245,7 @@ int rdbLoad(char *filename, rdbSaveInfo *rsi) { ...@@ -2089,7 +2245,7 @@ int rdbLoad(char *filename, rdbSaveInfo *rsi) {
int retval; int retval;
if ((fp = fopen(filename,"r")) == NULL) return C_ERR; if ((fp = fopen(filename,"r")) == NULL) return C_ERR;
startLoading(fp); startLoadingFile(fp, filename);
rioInitWithFile(&rdb,fp); rioInitWithFile(&rdb,fp);
retval = rdbLoadRio(&rdb,rsi,0); retval = rdbLoadRio(&rdb,rsi,0);
fclose(fp); fclose(fp);
...@@ -2136,8 +2292,6 @@ void backgroundSaveDoneHandlerDisk(int exitcode, int bysignal) { ...@@ -2136,8 +2292,6 @@ void backgroundSaveDoneHandlerDisk(int exitcode, int bysignal) {
* This function covers the case of RDB -> Salves socket transfers for * This function covers the case of RDB -> Salves socket transfers for
* diskless replication. */ * diskless replication. */
void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) { void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) {
uint64_t *ok_slaves;
if (!bysignal && exitcode == 0) { if (!bysignal && exitcode == 0) {
serverLog(LL_NOTICE, serverLog(LL_NOTICE,
"Background RDB transfer terminated with success"); "Background RDB transfer terminated with success");
...@@ -2151,79 +2305,6 @@ void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) { ...@@ -2151,79 +2305,6 @@ void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) {
server.rdb_child_type = RDB_CHILD_TYPE_NONE; server.rdb_child_type = RDB_CHILD_TYPE_NONE;
server.rdb_save_time_start = -1; server.rdb_save_time_start = -1;
/* If the child returns an OK exit code, read the set of slave client
* IDs and the associated status code. We'll terminate all the slaves
* in error state.
*
* If the process returned an error, consider the list of slaves that
* can continue to be empty, so that it's just a special case of the
* normal code path. */
ok_slaves = zmalloc(sizeof(uint64_t)); /* Make space for the count. */
ok_slaves[0] = 0;
if (!bysignal && exitcode == 0) {
int readlen = sizeof(uint64_t);
if (read(server.rdb_pipe_read_result_from_child, ok_slaves, readlen) ==
readlen)
{
readlen = ok_slaves[0]*sizeof(uint64_t)*2;
/* Make space for enough elements as specified by the first
* uint64_t element in the array. */
ok_slaves = zrealloc(ok_slaves,sizeof(uint64_t)+readlen);
if (readlen &&
read(server.rdb_pipe_read_result_from_child, ok_slaves+1,
readlen) != readlen)
{
ok_slaves[0] = 0;
}
}
}
close(server.rdb_pipe_read_result_from_child);
close(server.rdb_pipe_write_result_to_parent);
/* We can continue the replication process with all the slaves that
* correctly received the full payload. Others are terminated. */
listNode *ln;
listIter li;
listRewind(server.slaves,&li);
while((ln = listNext(&li))) {
client *slave = ln->value;
if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END) {
uint64_t j;
int errorcode = 0;
/* Search for the slave ID in the reply. In order for a slave to
* continue the replication process, we need to find it in the list,
* and it must have an error code set to 0 (which means success). */
for (j = 0; j < ok_slaves[0]; j++) {
if (slave->id == ok_slaves[2*j+1]) {
errorcode = ok_slaves[2*j+2];
break; /* Found in slaves list. */
}
}
if (j == ok_slaves[0] || errorcode != 0) {
serverLog(LL_WARNING,
"Closing slave %s: child->slave RDB transfer failed: %s",
replicationGetSlaveName(slave),
(errorcode == 0) ? "RDB transfer child aborted"
: strerror(errorcode));
freeClient(slave);
} else {
serverLog(LL_WARNING,
"Slave %s correctly received the streamed RDB file.",
replicationGetSlaveName(slave));
/* Restore the socket as non-blocking. */
anetNonBlock(NULL,slave->fd);
anetSendTimeout(NULL,slave->fd,0);
}
}
}
zfree(ok_slaves);
updateSlavesWaitingBgsave((!bysignal && exitcode == 0) ? C_OK : C_ERR, RDB_CHILD_TYPE_SOCKET); updateSlavesWaitingBgsave((!bysignal && exitcode == 0) ? C_OK : C_ERR, RDB_CHILD_TYPE_SOCKET);
} }
...@@ -2255,120 +2336,61 @@ void killRDBChild(void) { ...@@ -2255,120 +2336,61 @@ void killRDBChild(void) {
/* Spawn an RDB child that writes the RDB to the sockets of the slaves /* Spawn an RDB child that writes the RDB to the sockets of the slaves
* that are currently in SLAVE_STATE_WAIT_BGSAVE_START state. */ * that are currently in SLAVE_STATE_WAIT_BGSAVE_START state. */
int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
int *fds;
uint64_t *clientids;
int numfds;
listNode *ln; listNode *ln;
listIter li; listIter li;
pid_t childpid; pid_t childpid;
long long start;
int pipefds[2]; int pipefds[2];
if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) return C_ERR; if (hasActiveChildProcess()) return C_ERR;
/* Even if the previous fork child exited, don't start a new one until we
* drained the pipe. */
if (server.rdb_pipe_conns) return C_ERR;
/* Before to fork, create a pipe that will be used in order to /* Before to fork, create a pipe that is used to transfer the rdb bytes to
* send back to the parent the IDs of the slaves that successfully * the parent, we can't let it write directly to the sockets, since in case
* received all the writes. */ * of TLS we must let the parent handle a continuous TLS state when the
* child terminates and parent takes over. */
if (pipe(pipefds) == -1) return C_ERR; if (pipe(pipefds) == -1) return C_ERR;
server.rdb_pipe_read_result_from_child = pipefds[0]; server.rdb_pipe_read = pipefds[0];
server.rdb_pipe_write_result_to_parent = pipefds[1]; server.rdb_pipe_write = pipefds[1];
anetNonBlock(NULL, server.rdb_pipe_read);
/* Collect the file descriptors of the slaves we want to transfer /* Collect the connections of the replicas we want to transfer
* the RDB to, which are i WAIT_BGSAVE_START state. */ * the RDB to, which are i WAIT_BGSAVE_START state. */
fds = zmalloc(sizeof(int)*listLength(server.slaves)); server.rdb_pipe_conns = zmalloc(sizeof(connection *)*listLength(server.slaves));
/* We also allocate an array of corresponding client IDs. This will server.rdb_pipe_numconns = 0;
* be useful for the child process in order to build the report server.rdb_pipe_numconns_writing = 0;
* (sent via unix pipe) that will be sent to the parent. */
clientids = zmalloc(sizeof(uint64_t)*listLength(server.slaves));
numfds = 0;
listRewind(server.slaves,&li); listRewind(server.slaves,&li);
while((ln = listNext(&li))) { while((ln = listNext(&li))) {
client *slave = ln->value; client *slave = ln->value;
if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_START) { if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_START) {
clientids[numfds] = slave->id; server.rdb_pipe_conns[server.rdb_pipe_numconns++] = slave->conn;
fds[numfds++] = slave->fd;
replicationSetupSlaveForFullResync(slave,getPsyncInitialOffset()); replicationSetupSlaveForFullResync(slave,getPsyncInitialOffset());
/* Put the socket in blocking mode to simplify RDB transfer.
* We'll restore it when the children returns (since duped socket
* will share the O_NONBLOCK attribute with the parent). */
anetBlock(NULL,slave->fd);
anetSendTimeout(NULL,slave->fd,server.repl_timeout*1000);
} }
} }
/* Create the child process. */ /* Create the child process. */
openChildInfoPipe(); openChildInfoPipe();
start = ustime(); if ((childpid = redisFork()) == 0) {
if ((childpid = fork()) == 0) {
/* Child */ /* Child */
int retval; int retval;
rio slave_sockets; rio rdb;
rioInitWithFdset(&slave_sockets,fds,numfds); rioInitWithFd(&rdb,server.rdb_pipe_write);
zfree(fds);
closeListeningSockets(0);
redisSetProcTitle("redis-rdb-to-slaves"); redisSetProcTitle("redis-rdb-to-slaves");
retval = rdbSaveRioWithEOFMark(&slave_sockets,NULL,rsi); retval = rdbSaveRioWithEOFMark(&rdb,NULL,rsi);
if (retval == C_OK && rioFlush(&slave_sockets) == 0) if (retval == C_OK && rioFlush(&rdb) == 0)
retval = C_ERR; retval = C_ERR;
if (retval == C_OK) { if (retval == C_OK) {
size_t private_dirty = zmalloc_get_private_dirty(-1); sendChildCOWInfo(CHILD_INFO_TYPE_RDB, "RDB");
if (private_dirty) {
serverLog(LL_NOTICE,
"RDB: %zu MB of memory used by copy-on-write",
private_dirty/(1024*1024));
}
server.child_info_data.cow_size = private_dirty;
sendChildInfo(CHILD_INFO_TYPE_RDB);
/* If we are returning OK, at least one slave was served
* with the RDB file as expected, so we need to send a report
* to the parent via the pipe. The format of the message is:
*
* <len> <slave[0].id> <slave[0].error> ...
*
* len, slave IDs, and slave errors, are all uint64_t integers,
* so basically the reply is composed of 64 bits for the len field
* plus 2 additional 64 bit integers for each entry, for a total
* of 'len' entries.
*
* The 'id' represents the slave's client ID, so that the master
* can match the report with a specific slave, and 'error' is
* set to 0 if the replication process terminated with a success
* or the error code if an error occurred. */
void *msg = zmalloc(sizeof(uint64_t)*(1+2*numfds));
uint64_t *len = msg;
uint64_t *ids = len+1;
int j, msglen;
*len = numfds;
for (j = 0; j < numfds; j++) {
*ids++ = clientids[j];
*ids++ = slave_sockets.io.fdset.state[j];
}
/* Write the message to the parent. If we have no good slaves or
* we are unable to transfer the message to the parent, we exit
* with an error so that the parent will abort the replication
* process with all the childre that were waiting. */
msglen = sizeof(uint64_t)*(1+2*numfds);
if (*len == 0 ||
write(server.rdb_pipe_write_result_to_parent,msg,msglen)
!= msglen)
{
retval = C_ERR;
}
zfree(msg);
} }
zfree(clientids);
rioFreeFdset(&slave_sockets); rioFreeFd(&rdb);
close(server.rdb_pipe_write); /* wake up the reader, tell it we're done. */
exitFromChild((retval == C_OK) ? 0 : 1); exitFromChild((retval == C_OK) ? 0 : 1);
} else { } else {
/* Parent */ /* Parent */
...@@ -2382,32 +2404,28 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { ...@@ -2382,32 +2404,28 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
listRewind(server.slaves,&li); listRewind(server.slaves,&li);
while((ln = listNext(&li))) { while((ln = listNext(&li))) {
client *slave = ln->value; client *slave = ln->value;
int j; if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END) {
slave->replstate = SLAVE_STATE_WAIT_BGSAVE_START;
for (j = 0; j < numfds; j++) {
if (slave->id == clientids[j]) {
slave->replstate = SLAVE_STATE_WAIT_BGSAVE_START;
break;
}
} }
} }
close(pipefds[0]); close(server.rdb_pipe_write);
close(pipefds[1]); close(server.rdb_pipe_read);
zfree(server.rdb_pipe_conns);
server.rdb_pipe_conns = NULL;
server.rdb_pipe_numconns = 0;
server.rdb_pipe_numconns_writing = 0;
closeChildInfoPipe(); closeChildInfoPipe();
} else { } else {
server.stat_fork_time = ustime()-start;
server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */
latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000);
serverLog(LL_NOTICE,"Background RDB transfer started by pid %d", serverLog(LL_NOTICE,"Background RDB transfer started by pid %d",
childpid); childpid);
server.rdb_save_time_start = time(NULL); server.rdb_save_time_start = time(NULL);
server.rdb_child_pid = childpid; server.rdb_child_pid = childpid;
server.rdb_child_type = RDB_CHILD_TYPE_SOCKET; server.rdb_child_type = RDB_CHILD_TYPE_SOCKET;
updateDictResizePolicy(); close(server.rdb_pipe_write); /* close write in parent so that it can detect the close on the child. */
if (aeCreateFileEvent(server.el, server.rdb_pipe_read, AE_READABLE, rdbPipeReadHandler,NULL) == AE_ERR) {
serverPanic("Unrecoverable error creating server.rdb_pipe_read file event.");
}
} }
zfree(clientids);
zfree(fds);
return (childpid == -1) ? C_ERR : C_OK; return (childpid == -1) ? C_ERR : C_OK;
} }
return C_OK; /* Unreached. */ return C_OK; /* Unreached. */
...@@ -2447,15 +2465,15 @@ void bgsaveCommand(client *c) { ...@@ -2447,15 +2465,15 @@ void bgsaveCommand(client *c) {
if (server.rdb_child_pid != -1) { if (server.rdb_child_pid != -1) {
addReplyError(c,"Background save already in progress"); addReplyError(c,"Background save already in progress");
} else if (server.aof_child_pid != -1) { } else if (hasActiveChildProcess()) {
if (schedule) { if (schedule) {
server.rdb_bgsave_scheduled = 1; server.rdb_bgsave_scheduled = 1;
addReplyStatus(c,"Background saving scheduled"); addReplyStatus(c,"Background saving scheduled");
} else { } else {
addReplyError(c, addReplyError(c,
"An AOF log rewriting in progress: can't BGSAVE right now. " "Another child process is active (AOF?): can't BGSAVE right now. "
"Use BGSAVE SCHEDULE in order to schedule a BGSAVE whenever " "Use BGSAVE SCHEDULE in order to schedule a BGSAVE whenever "
"possible."); "possible.");
} }
} else if (rdbSaveBackground(server.rdb_filename,rsiptr) == C_OK) { } else if (rdbSaveBackground(server.rdb_filename,rsiptr) == C_OK) {
addReplyStatus(c,"Background saving started"); addReplyStatus(c,"Background saving started");
......
...@@ -140,11 +140,12 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi); ...@@ -140,11 +140,12 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi);
int rdbSaveToSlavesSockets(rdbSaveInfo *rsi); int rdbSaveToSlavesSockets(rdbSaveInfo *rsi);
void rdbRemoveTempFile(pid_t childpid); void rdbRemoveTempFile(pid_t childpid);
int rdbSave(char *filename, rdbSaveInfo *rsi); int rdbSave(char *filename, rdbSaveInfo *rsi);
ssize_t rdbSaveObject(rio *rdb, robj *o); ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key);
size_t rdbSavedObjectLen(robj *o); size_t rdbSavedObjectLen(robj *o);
robj *rdbLoadObject(int type, rio *rdb); robj *rdbLoadObject(int type, rio *rdb, robj *key);
void backgroundSaveDoneHandler(int exitcode, int bysignal); void backgroundSaveDoneHandler(int exitcode, int bysignal);
int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime); int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime);
ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt);
robj *rdbLoadStringObject(rio *rdb); robj *rdbLoadStringObject(rio *rdb);
ssize_t rdbSaveStringObject(rio *rdb, robj *obj); ssize_t rdbSaveStringObject(rio *rdb, robj *obj);
ssize_t rdbSaveRawString(rio *rdb, unsigned char *s, size_t len); ssize_t rdbSaveRawString(rio *rdb, unsigned char *s, size_t len);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment