Commit 1ac30300 authored by Guy Korland's avatar Guy Korland
Browse files

Merge branch 'unstable' of github.com:antirez/redis into unstable

parents c1455dc0 673c9d70
#!/bin/sh
TCL_VERSIONS="8.5 8.6"
TCLSH=""
for VERSION in $TCL_VERSIONS; do
TCL=`which tclsh$VERSION 2>/dev/null` && TCLSH=$TCL
done
if [ -z $TCLSH ]
then
echo "You need tcl 8.5 or newer in order to run the Redis test"
exit 1
fi
make -C tests/modules && \
$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/fork --single unit/moduleapi/testrdb --single unit/moduleapi/infotest --single unit/moduleapi/propagate "${@}"
...@@ -20,7 +20,7 @@ DEPENDENCY_TARGETS=hiredis linenoise lua ...@@ -20,7 +20,7 @@ DEPENDENCY_TARGETS=hiredis linenoise lua
NODEPS:=clean distclean NODEPS:=clean distclean
# Default settings # Default settings
STD=-std=c99 -pedantic -DREDIS_STATIC='' STD=-std=c11 -pedantic -DREDIS_STATIC=''
ifneq (,$(findstring clang,$(CC))) ifneq (,$(findstring clang,$(CC)))
ifneq (,$(findstring FreeBSD,$(uname_S))) ifneq (,$(findstring FreeBSD,$(uname_S)))
STD+=-Wno-c11-extensions STD+=-Wno-c11-extensions
...@@ -93,6 +93,8 @@ else ...@@ -93,6 +93,8 @@ else
ifeq ($(uname_S),Darwin) ifeq ($(uname_S),Darwin)
# Darwin # Darwin
FINAL_LIBS+= -ldl FINAL_LIBS+= -ldl
OPENSSL_CFLAGS=-I/usr/local/opt/openssl/include
OPENSSL_LDFLAGS=-L/usr/local/opt/openssl/lib
else else
ifeq ($(uname_S),AIX) ifeq ($(uname_S),AIX)
# AIX # AIX
...@@ -145,6 +147,12 @@ ifeq ($(MALLOC),jemalloc) ...@@ -145,6 +147,12 @@ ifeq ($(MALLOC),jemalloc)
FINAL_LIBS := ../deps/jemalloc/lib/libjemalloc.a $(FINAL_LIBS) FINAL_LIBS := ../deps/jemalloc/lib/libjemalloc.a $(FINAL_LIBS)
endif endif
ifeq ($(BUILD_TLS),yes)
FINAL_CFLAGS+=-DUSE_OPENSSL $(OPENSSL_CFLAGS)
FINAL_LDFLAGS+=$(OPENSSL_LDFLAGS)
FINAL_LIBS += ../deps/hiredis/libhiredis_ssl.a -lssl -lcrypto
endif
REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS)
REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS) REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS)
REDIS_INSTALL=$(QUIET_INSTALL)$(INSTALL) REDIS_INSTALL=$(QUIET_INSTALL)$(INSTALL)
...@@ -164,7 +172,7 @@ endif ...@@ -164,7 +172,7 @@ endif
REDIS_SERVER_NAME=redis-server REDIS_SERVER_NAME=redis-server
REDIS_SENTINEL_NAME=redis-sentinel REDIS_SENTINEL_NAME=redis-sentinel
REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o gopher.o REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o
REDIS_CLI_NAME=redis-cli REDIS_CLI_NAME=redis-cli
REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o
REDIS_BENCHMARK_NAME=redis-benchmark REDIS_BENCHMARK_NAME=redis-benchmark
......
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
*/ */
#include "server.h" #include "server.h"
#include "sha256.h"
#include <fcntl.h> #include <fcntl.h>
/* ============================================================================= /* =============================================================================
...@@ -93,6 +94,9 @@ void ACLResetSubcommandsForCommand(user *u, unsigned long id); ...@@ -93,6 +94,9 @@ void ACLResetSubcommandsForCommand(user *u, unsigned long id);
void ACLResetSubcommands(user *u); void ACLResetSubcommands(user *u);
void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub); void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub);
/* The length of the string representation of a hashed password. */
#define HASH_PASSWORD_LEN SHA256_BLOCK_SIZE*2
/* ============================================================================= /* =============================================================================
* Helper functions for the rest of the ACL implementation * Helper functions for the rest of the ACL implementation
* ==========================================================================*/ * ==========================================================================*/
...@@ -139,6 +143,25 @@ int time_independent_strcmp(char *a, char *b) { ...@@ -139,6 +143,25 @@ int time_independent_strcmp(char *a, char *b) {
return diff; /* If zero strings are the same. */ return diff; /* If zero strings are the same. */
} }
/* Given an SDS string, returns the SHA256 hex representation as a
* new SDS string. */
sds ACLHashPassword(unsigned char *cleartext, size_t len) {
SHA256_CTX ctx;
unsigned char hash[SHA256_BLOCK_SIZE];
char hex[HASH_PASSWORD_LEN];
char *cset = "0123456789abcdef";
sha256_init(&ctx);
sha256_update(&ctx,(unsigned char*)cleartext,len);
sha256_final(&ctx,hash);
for (int j = 0; j < SHA256_BLOCK_SIZE; j++) {
hex[j*2] = cset[((hash[j]&0xF0)>>4)];
hex[j*2+1] = cset[(hash[j]&0xF)];
}
return sdsnewlen(hex,HASH_PASSWORD_LEN);
}
/* ============================================================================= /* =============================================================================
* Low level ACL API * Low level ACL API
* ==========================================================================*/ * ==========================================================================*/
...@@ -295,7 +318,7 @@ int ACLGetCommandBitCoordinates(uint64_t id, uint64_t *word, uint64_t *bit) { ...@@ -295,7 +318,7 @@ int ACLGetCommandBitCoordinates(uint64_t id, uint64_t *word, uint64_t *bit) {
* Note that this function does not check the ALLCOMMANDS flag of the user * Note that this function does not check the ALLCOMMANDS flag of the user
* but just the lowlevel bitmask. * but just the lowlevel bitmask.
* *
* If the bit overflows the user internal represetation, zero is returned * If the bit overflows the user internal representation, zero is returned
* in order to disallow the execution of the command in such edge case. */ * in order to disallow the execution of the command in such edge case. */
int ACLGetUserCommandBit(user *u, unsigned long id) { int ACLGetUserCommandBit(user *u, unsigned long id) {
uint64_t word, bit; uint64_t word, bit;
...@@ -311,7 +334,7 @@ int ACLUserCanExecuteFutureCommands(user *u) { ...@@ -311,7 +334,7 @@ int ACLUserCanExecuteFutureCommands(user *u) {
} }
/* Set the specified command bit for the specified user to 'value' (0 or 1). /* Set the specified command bit for the specified user to 'value' (0 or 1).
* If the bit overflows the user internal represetation, no operation * If the bit overflows the user internal representation, no operation
* is performed. As a side effect of calling this function with a value of * is performed. As a side effect of calling this function with a value of
* zero, the user flag ALLCOMMANDS is cleared since it is no longer possible * zero, the user flag ALLCOMMANDS is cleared since it is no longer possible
* to skip the command bit explicit test. */ * to skip the command bit explicit test. */
...@@ -350,7 +373,7 @@ int ACLSetUserCommandBitsForCategory(user *u, const char *category, int value) { ...@@ -350,7 +373,7 @@ int ACLSetUserCommandBitsForCategory(user *u, const char *category, int value) {
/* Return the number of commands allowed (on) and denied (off) for the user 'u' /* Return the number of commands allowed (on) and denied (off) for the user 'u'
* in the subset of commands flagged with the specified category name. * in the subset of commands flagged with the specified category name.
* If the categoty name is not valid, C_ERR is returend, otherwise C_OK is * If the category name is not valid, C_ERR is returned, otherwise C_OK is
* returned and on and off are populated by reference. */ * returned and on and off are populated by reference. */
int ACLCountCategoryBitsForUser(user *u, unsigned long *on, unsigned long *off, int ACLCountCategoryBitsForUser(user *u, unsigned long *on, unsigned long *off,
const char *category) const char *category)
...@@ -502,7 +525,7 @@ sds ACLDescribeUser(user *u) { ...@@ -502,7 +525,7 @@ sds ACLDescribeUser(user *u) {
listRewind(u->passwords,&li); listRewind(u->passwords,&li);
while((ln = listNext(&li))) { while((ln = listNext(&li))) {
sds thispass = listNodeValue(ln); sds thispass = listNodeValue(ln);
res = sdscatlen(res,">",1); res = sdscatlen(res,"#",1);
res = sdscatsds(res,thispass); res = sdscatsds(res,thispass);
res = sdscatlen(res," ",1); res = sdscatlen(res," ",1);
} }
...@@ -542,6 +565,8 @@ struct redisCommand *ACLLookupCommand(const char *name) { ...@@ -542,6 +565,8 @@ struct redisCommand *ACLLookupCommand(const char *name) {
* and command ID. */ * and command ID. */
void ACLResetSubcommandsForCommand(user *u, unsigned long id) { void ACLResetSubcommandsForCommand(user *u, unsigned long id) {
if (u->allowed_subcommands && u->allowed_subcommands[id]) { if (u->allowed_subcommands && u->allowed_subcommands[id]) {
for (int i = 0; u->allowed_subcommands[id][i]; i++)
sdsfree(u->allowed_subcommands[id][i]);
zfree(u->allowed_subcommands[id]); zfree(u->allowed_subcommands[id]);
u->allowed_subcommands[id] = NULL; u->allowed_subcommands[id] = NULL;
} }
...@@ -624,10 +649,17 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { ...@@ -624,10 +649,17 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) {
* It is possible to specify multiple patterns. * It is possible to specify multiple patterns.
* allkeys Alias for ~* * allkeys Alias for ~*
* resetkeys Flush the list of allowed keys patterns. * resetkeys Flush the list of allowed keys patterns.
* ><password> Add this passowrd to the list of valid password for the user. * ><password> Add this password to the list of valid password for the user.
* For example >mypass will add "mypass" to the list. * For example >mypass will add "mypass" to the list.
* This directive clears the "nopass" flag (see later). * This directive clears the "nopass" flag (see later).
* #<hash> Add this password hash to the list of valid hashes for
* the user. This is useful if you have previously computed
* the hash, and don't want to store it in plaintext.
* This directive clears the "nopass" flag (see later).
* <<password> Remove this password from the list of valid passwords. * <<password> Remove this password from the list of valid passwords.
* !<hash> Remove this hashed password from the list of valid passwords.
* This is useful when you want to remove a password just by
* hash without knowing its plaintext version at all.
* nopass All the set passwords of the user are removed, and the user * nopass All the set passwords of the user are removed, and the user
* is flagged as requiring no password: it means that every * is flagged as requiring no password: it means that every
* password will work against this user. If this directive is * password will work against this user. If this directive is
...@@ -663,6 +695,7 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { ...@@ -663,6 +695,7 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) {
* EEXIST: You are adding a key pattern after "*" was already added. This is * EEXIST: You are adding a key pattern after "*" was already added. This is
* almost surely an error on the user side. * almost surely an error on the user side.
* ENODEV: The password you are trying to remove from the user does not exist. * ENODEV: The password you are trying to remove from the user does not exist.
* EBADMSG: The hash you are trying to add is not a valid hash.
*/ */
int ACLSetUser(user *u, const char *op, ssize_t oplen) { int ACLSetUser(user *u, const char *op, ssize_t oplen) {
if (oplen == -1) oplen = strlen(op); if (oplen == -1) oplen = strlen(op);
...@@ -698,14 +731,48 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { ...@@ -698,14 +731,48 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
} else if (!strcasecmp(op,"resetpass")) { } else if (!strcasecmp(op,"resetpass")) {
u->flags &= ~USER_FLAG_NOPASS; u->flags &= ~USER_FLAG_NOPASS;
listEmpty(u->passwords); listEmpty(u->passwords);
} else if (op[0] == '>') { } else if (op[0] == '>' || op[0] == '#') {
sds newpass = sdsnewlen(op+1,oplen-1); sds newpass;
if (op[0] == '>') {
newpass = ACLHashPassword((unsigned char*)op+1,oplen-1);
} else {
if (oplen != HASH_PASSWORD_LEN + 1) {
errno = EBADMSG;
return C_ERR;
}
/* Password hashes can only be characters that represent
* hexadecimal values, which are numbers and lowercase
* characters 'a' through 'f'.
*/
for(int i = 1; i < HASH_PASSWORD_LEN + 1; i++) {
char c = op[i];
if ((c < 'a' || c > 'f') && (c < '0' || c > '9')) {
errno = EBADMSG;
return C_ERR;
}
}
newpass = sdsnewlen(op+1,oplen-1);
}
listNode *ln = listSearchKey(u->passwords,newpass); listNode *ln = listSearchKey(u->passwords,newpass);
/* Avoid re-adding the same password multiple times. */ /* Avoid re-adding the same password multiple times. */
if (ln == NULL) listAddNodeTail(u->passwords,newpass); if (ln == NULL)
listAddNodeTail(u->passwords,newpass);
else
sdsfree(newpass);
u->flags &= ~USER_FLAG_NOPASS; u->flags &= ~USER_FLAG_NOPASS;
} else if (op[0] == '<') { } else if (op[0] == '<' || op[0] == '!') {
sds delpass = sdsnewlen(op+1,oplen-1); sds delpass;
if (op[0] == '<') {
delpass = ACLHashPassword((unsigned char*)op+1,oplen-1);
} else {
if (oplen != HASH_PASSWORD_LEN + 1) {
errno = EBADMSG;
return C_ERR;
}
delpass = sdsnewlen(op+1,oplen-1);
}
listNode *ln = listSearchKey(u->passwords,delpass); listNode *ln = listSearchKey(u->passwords,delpass);
sdsfree(delpass); sdsfree(delpass);
if (ln) { if (ln) {
...@@ -722,7 +789,10 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { ...@@ -722,7 +789,10 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) {
sds newpat = sdsnewlen(op+1,oplen-1); sds newpat = sdsnewlen(op+1,oplen-1);
listNode *ln = listSearchKey(u->patterns,newpat); listNode *ln = listSearchKey(u->patterns,newpat);
/* Avoid re-adding the same pattern multiple times. */ /* Avoid re-adding the same pattern multiple times. */
if (ln == NULL) listAddNodeTail(u->patterns,newpat); if (ln == NULL)
listAddNodeTail(u->patterns,newpat);
else
sdsfree(newpat);
u->flags &= ~USER_FLAG_ALLKEYS; u->flags &= ~USER_FLAG_ALLKEYS;
} else if (op[0] == '+' && op[1] != '@') { } else if (op[0] == '+' && op[1] != '@') {
if (strchr(op,'|') == NULL) { if (strchr(op,'|') == NULL) {
...@@ -820,6 +890,9 @@ char *ACLSetUserStringError(void) { ...@@ -820,6 +890,9 @@ char *ACLSetUserStringError(void) {
else if (errno == ENODEV) else if (errno == ENODEV)
errmsg = "The password you are trying to remove from the user does " errmsg = "The password you are trying to remove from the user does "
"not exist"; "not exist";
else if (errno == EBADMSG)
errmsg = "The password hash must be exactly 64 characters and contain "
"only lowercase hexadecimal characters";
return errmsg; return errmsg;
} }
...@@ -877,11 +950,15 @@ int ACLCheckUserCredentials(robj *username, robj *password) { ...@@ -877,11 +950,15 @@ int ACLCheckUserCredentials(robj *username, robj *password) {
listIter li; listIter li;
listNode *ln; listNode *ln;
listRewind(u->passwords,&li); listRewind(u->passwords,&li);
sds hashed = ACLHashPassword(password->ptr,sdslen(password->ptr));
while((ln = listNext(&li))) { while((ln = listNext(&li))) {
sds thispass = listNodeValue(ln); sds thispass = listNodeValue(ln);
if (!time_independent_strcmp(password->ptr, thispass)) if (!time_independent_strcmp(hashed, thispass)) {
sdsfree(hashed);
return C_OK; return C_OK;
}
} }
sdsfree(hashed);
/* If we reached this point, no password matched. */ /* If we reached this point, no password matched. */
errno = EINVAL; errno = EINVAL;
...@@ -947,9 +1024,9 @@ user *ACLGetUserByName(const char *name, size_t namelen) { ...@@ -947,9 +1024,9 @@ user *ACLGetUserByName(const char *name, size_t namelen) {
return myuser; return myuser;
} }
/* Check if the command ready to be excuted in the client 'c', and already /* Check if the command is ready to be executed in the client 'c', already
* referenced by c->cmd, can be executed by this client according to the * referenced by c->cmd, and can be executed by this client according to the
* ACls associated to the client user c->user. * ACLs associated to the client user c->user.
* *
* If the user can execute the command ACL_OK is returned, otherwise * If the user can execute the command ACL_OK is returned, otherwise
* ACL_DENIED_CMD or ACL_DENIED_KEY is returned: the first in case the * ACL_DENIED_CMD or ACL_DENIED_KEY is returned: the first in case the
...@@ -1120,7 +1197,7 @@ int ACLLoadConfiguredUsers(void) { ...@@ -1120,7 +1197,7 @@ int ACLLoadConfiguredUsers(void) {
} }
/* This function loads the ACL from the specified filename: every line /* This function loads the ACL from the specified filename: every line
* is validated and shold be either empty or in the format used to specify * is validated and should be either empty or in the format used to specify
* users in the redis.conf configuration or in the ACL file, that is: * users in the redis.conf configuration or in the ACL file, that is:
* *
* user <username> ... rules ... * user <username> ... rules ...
...@@ -1170,7 +1247,7 @@ sds ACLLoadFromFile(const char *filename) { ...@@ -1170,7 +1247,7 @@ sds ACLLoadFromFile(const char *filename) {
* to the real user mentioned in the ACL line. */ * to the real user mentioned in the ACL line. */
user *fakeuser = ACLCreateUnlinkedUser(); user *fakeuser = ACLCreateUnlinkedUser();
/* We do all the loading in a fresh insteance of the Users radix tree, /* We do all the loading in a fresh instance of the Users radix tree,
* so if there are errors loading the ACL file we can rollback to the * so if there are errors loading the ACL file we can rollback to the
* old version. */ * old version. */
rax *old_users = Users; rax *old_users = Users;
...@@ -1246,7 +1323,7 @@ sds ACLLoadFromFile(const char *filename) { ...@@ -1246,7 +1323,7 @@ sds ACLLoadFromFile(const char *filename) {
} }
/* Note that the same rules already applied to the fake user, so /* Note that the same rules already applied to the fake user, so
* we just assert that everything goess well: it should. */ * we just assert that everything goes well: it should. */
for (j = 2; j < argc; j++) for (j = 2; j < argc; j++)
serverAssert(ACLSetUser(u,argv[j],sdslen(argv[j])) == C_OK); serverAssert(ACLSetUser(u,argv[j],sdslen(argv[j])) == C_OK);
...@@ -1609,7 +1686,7 @@ void addReplyCommandCategories(client *c, struct redisCommand *cmd) { ...@@ -1609,7 +1686,7 @@ void addReplyCommandCategories(client *c, struct redisCommand *cmd) {
setDeferredSetLen(c, flaglen, flagcount); setDeferredSetLen(c, flaglen, flagcount);
} }
/* AUTH <passowrd> /* AUTH <password>
* AUTH <username> <password> (Redis >= 6.0 form) * AUTH <username> <password> (Redis >= 6.0 form)
* *
* When the user is omitted it means that we are trying to authenticate * When the user is omitted it means that we are trying to authenticate
......
...@@ -76,6 +76,7 @@ aeEventLoop *aeCreateEventLoop(int setsize) { ...@@ -76,6 +76,7 @@ aeEventLoop *aeCreateEventLoop(int setsize) {
eventLoop->maxfd = -1; eventLoop->maxfd = -1;
eventLoop->beforesleep = NULL; eventLoop->beforesleep = NULL;
eventLoop->aftersleep = NULL; eventLoop->aftersleep = NULL;
eventLoop->flags = 0;
if (aeApiCreate(eventLoop) == -1) goto err; if (aeApiCreate(eventLoop) == -1) goto err;
/* Events with mask == AE_NONE are not set. So let's initialize the /* Events with mask == AE_NONE are not set. So let's initialize the
* vector with it. */ * vector with it. */
...@@ -97,6 +98,14 @@ int aeGetSetSize(aeEventLoop *eventLoop) { ...@@ -97,6 +98,14 @@ int aeGetSetSize(aeEventLoop *eventLoop) {
return eventLoop->setsize; return eventLoop->setsize;
} }
/* Tells the next iteration/s of the event processing to set timeout of 0. */
void aeSetDontWait(aeEventLoop *eventLoop, int noWait) {
if (noWait)
eventLoop->flags |= AE_DONT_WAIT;
else
eventLoop->flags &= ~AE_DONT_WAIT;
}
/* Resize the maximum set size of the event loop. /* Resize the maximum set size of the event loop.
* If the requested set size is smaller than the current set size, but * If the requested set size is smaller than the current set size, but
* there is already a file descriptor in use that is >= the requested * there is already a file descriptor in use that is >= the requested
...@@ -406,6 +415,11 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) ...@@ -406,6 +415,11 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
} }
} }
if (eventLoop->flags & AE_DONT_WAIT) {
tv.tv_sec = tv.tv_usec = 0;
tvp = &tv;
}
/* Call the multiplexing API, will return only on timeout or when /* Call the multiplexing API, will return only on timeout or when
* some event fires. */ * some event fires. */
numevents = aeApiPoll(eventLoop, tvp); numevents = aeApiPoll(eventLoop, tvp);
......
...@@ -106,6 +106,7 @@ typedef struct aeEventLoop { ...@@ -106,6 +106,7 @@ typedef struct aeEventLoop {
void *apidata; /* This is used for polling API specific data */ void *apidata; /* This is used for polling API specific data */
aeBeforeSleepProc *beforesleep; aeBeforeSleepProc *beforesleep;
aeBeforeSleepProc *aftersleep; aeBeforeSleepProc *aftersleep;
int flags;
} aeEventLoop; } aeEventLoop;
/* Prototypes */ /* Prototypes */
...@@ -128,5 +129,6 @@ void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep ...@@ -128,5 +129,6 @@ void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep
void aeSetAfterSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *aftersleep); void aeSetAfterSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *aftersleep);
int aeGetSetSize(aeEventLoop *eventLoop); int aeGetSetSize(aeEventLoop *eventLoop);
int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); int aeResizeSetSize(aeEventLoop *eventLoop, int setsize);
void aeSetDontWait(aeEventLoop *eventLoop, int noWait);
#endif #endif
...@@ -121,8 +121,8 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { ...@@ -121,8 +121,8 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
if (e->events & EPOLLIN) mask |= AE_READABLE; if (e->events & EPOLLIN) mask |= AE_READABLE;
if (e->events & EPOLLOUT) mask |= AE_WRITABLE; if (e->events & EPOLLOUT) mask |= AE_WRITABLE;
if (e->events & EPOLLERR) mask |= AE_WRITABLE; if (e->events & EPOLLERR) mask |= AE_WRITABLE|AE_READABLE;
if (e->events & EPOLLHUP) mask |= AE_WRITABLE; if (e->events & EPOLLHUP) mask |= AE_WRITABLE|AE_READABLE;
eventLoop->fired[j].fd = e->data.fd; eventLoop->fired[j].fd = e->data.fd;
eventLoop->fired[j].mask = mask; eventLoop->fired[j].mask = mask;
} }
......
...@@ -193,6 +193,20 @@ int anetSendTimeout(char *err, int fd, long long ms) { ...@@ -193,6 +193,20 @@ int anetSendTimeout(char *err, int fd, long long ms) {
return ANET_OK; return ANET_OK;
} }
/* Set the socket receive timeout (SO_RCVTIMEO socket option) to the specified
* number of milliseconds, or disable it if the 'ms' argument is zero. */
int anetRecvTimeout(char *err, int fd, long long ms) {
struct timeval tv;
tv.tv_sec = ms/1000;
tv.tv_usec = (ms%1000)*1000;
if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) == -1) {
anetSetError(err, "setsockopt SO_RCVTIMEO: %s", strerror(errno));
return ANET_ERR;
}
return ANET_OK;
}
/* anetGenericResolve() is called by anetResolve() and anetResolveIP() to /* anetGenericResolve() is called by anetResolve() and anetResolveIP() to
* do the actual work. It resolves the hostname "host" and set the string * do the actual work. It resolves the hostname "host" and set the string
* representation of the IP address into the buffer pointed by "ipbuf". * representation of the IP address into the buffer pointed by "ipbuf".
...@@ -265,8 +279,8 @@ static int anetCreateSocket(char *err, int domain) { ...@@ -265,8 +279,8 @@ static int anetCreateSocket(char *err, int domain) {
#define ANET_CONNECT_NONE 0 #define ANET_CONNECT_NONE 0
#define ANET_CONNECT_NONBLOCK 1 #define ANET_CONNECT_NONBLOCK 1
#define ANET_CONNECT_BE_BINDING 2 /* Best effort binding. */ #define ANET_CONNECT_BE_BINDING 2 /* Best effort binding. */
static int anetTcpGenericConnect(char *err, char *addr, int port, static int anetTcpGenericConnect(char *err, const char *addr, int port,
char *source_addr, int flags) const char *source_addr, int flags)
{ {
int s = ANET_ERR, rv; int s = ANET_ERR, rv;
char portstr[6]; /* strlen("65535") + 1; */ char portstr[6]; /* strlen("65535") + 1; */
...@@ -345,31 +359,31 @@ end: ...@@ -345,31 +359,31 @@ end:
} }
} }
int anetTcpConnect(char *err, char *addr, int port) int anetTcpConnect(char *err, const char *addr, int port)
{ {
return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONE); return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONE);
} }
int anetTcpNonBlockConnect(char *err, char *addr, int port) int anetTcpNonBlockConnect(char *err, const char *addr, int port)
{ {
return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONBLOCK); return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONBLOCK);
} }
int anetTcpNonBlockBindConnect(char *err, char *addr, int port, int anetTcpNonBlockBindConnect(char *err, const char *addr, int port,
char *source_addr) const char *source_addr)
{ {
return anetTcpGenericConnect(err,addr,port,source_addr, return anetTcpGenericConnect(err,addr,port,source_addr,
ANET_CONNECT_NONBLOCK); ANET_CONNECT_NONBLOCK);
} }
int anetTcpNonBlockBestEffortBindConnect(char *err, char *addr, int port, int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port,
char *source_addr) const char *source_addr)
{ {
return anetTcpGenericConnect(err,addr,port,source_addr, return anetTcpGenericConnect(err,addr,port,source_addr,
ANET_CONNECT_NONBLOCK|ANET_CONNECT_BE_BINDING); ANET_CONNECT_NONBLOCK|ANET_CONNECT_BE_BINDING);
} }
int anetUnixGenericConnect(char *err, char *path, int flags) int anetUnixGenericConnect(char *err, const char *path, int flags)
{ {
int s; int s;
struct sockaddr_un sa; struct sockaddr_un sa;
...@@ -397,12 +411,12 @@ int anetUnixGenericConnect(char *err, char *path, int flags) ...@@ -397,12 +411,12 @@ int anetUnixGenericConnect(char *err, char *path, int flags)
return s; return s;
} }
int anetUnixConnect(char *err, char *path) int anetUnixConnect(char *err, const char *path)
{ {
return anetUnixGenericConnect(err,path,ANET_CONNECT_NONE); return anetUnixGenericConnect(err,path,ANET_CONNECT_NONE);
} }
int anetUnixNonBlockConnect(char *err, char *path) int anetUnixNonBlockConnect(char *err, const char *path)
{ {
return anetUnixGenericConnect(err,path,ANET_CONNECT_NONBLOCK); return anetUnixGenericConnect(err,path,ANET_CONNECT_NONBLOCK);
} }
......
...@@ -49,12 +49,12 @@ ...@@ -49,12 +49,12 @@
#undef ip_len #undef ip_len
#endif #endif
int anetTcpConnect(char *err, char *addr, int port); int anetTcpConnect(char *err, const char *addr, int port);
int anetTcpNonBlockConnect(char *err, char *addr, int port); int anetTcpNonBlockConnect(char *err, const char *addr, int port);
int anetTcpNonBlockBindConnect(char *err, char *addr, int port, char *source_addr); int anetTcpNonBlockBindConnect(char *err, const char *addr, int port, const char *source_addr);
int anetTcpNonBlockBestEffortBindConnect(char *err, char *addr, int port, char *source_addr); int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, const char *source_addr);
int anetUnixConnect(char *err, char *path); int anetUnixConnect(char *err, const char *path);
int anetUnixNonBlockConnect(char *err, char *path); int anetUnixNonBlockConnect(char *err, const char *path);
int anetRead(int fd, char *buf, int count); int anetRead(int fd, char *buf, int count);
int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len); int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len);
int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len); int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len);
...@@ -70,6 +70,7 @@ int anetEnableTcpNoDelay(char *err, int fd); ...@@ -70,6 +70,7 @@ int anetEnableTcpNoDelay(char *err, int fd);
int anetDisableTcpNoDelay(char *err, int fd); int anetDisableTcpNoDelay(char *err, int fd);
int anetTcpKeepAlive(char *err, int fd); int anetTcpKeepAlive(char *err, int fd);
int anetSendTimeout(char *err, int fd, long long ms); int anetSendTimeout(char *err, int fd, long long ms);
int anetRecvTimeout(char *err, int fd, long long ms);
int anetPeerToString(int fd, char *ip, size_t ip_len, int *port); int anetPeerToString(int fd, char *ip, size_t ip_len, int *port);
int anetKeepAlive(char *err, int fd, int interval); int anetKeepAlive(char *err, int fd, int interval);
int anetSockName(int fd, char *ip, size_t ip_len, int *port); int anetSockName(int fd, char *ip, size_t ip_len, int *port);
......
...@@ -197,6 +197,12 @@ ssize_t aofRewriteBufferWrite(int fd) { ...@@ -197,6 +197,12 @@ ssize_t aofRewriteBufferWrite(int fd) {
* AOF file implementation * AOF file implementation
* ------------------------------------------------------------------------- */ * ------------------------------------------------------------------------- */
/* Return true if an AOf fsync is currently already in progress in a
* BIO thread. */
int aofFsyncInProgress(void) {
return bioPendingJobsOfType(BIO_AOF_FSYNC) != 0;
}
/* Starts a background task that performs fsync() against the specified /* Starts a background task that performs fsync() against the specified
* file descriptor (the one of the AOF file) in another thread. */ * file descriptor (the one of the AOF file) in another thread. */
void aof_background_fsync(int fd) { void aof_background_fsync(int fd) {
...@@ -258,9 +264,9 @@ int startAppendOnly(void) { ...@@ -258,9 +264,9 @@ int startAppendOnly(void) {
strerror(errno)); strerror(errno));
return C_ERR; return C_ERR;
} }
if (server.rdb_child_pid != -1) { if (hasActiveChildProcess() && server.aof_child_pid == -1) {
server.aof_rewrite_scheduled = 1; server.aof_rewrite_scheduled = 1;
serverLog(LL_WARNING,"AOF was enabled but there is already a child process saving an RDB file on disk. An AOF background was scheduled to start when possible."); serverLog(LL_WARNING,"AOF was enabled but there is already another background operation. An AOF background was scheduled to start when possible.");
} else { } else {
/* If there is a pending AOF rewrite, we need to switch it off and /* If there is a pending AOF rewrite, we need to switch it off and
* start a new one: the old one cannot be reused because it is not * start a new one: the old one cannot be reused because it is not
...@@ -297,9 +303,7 @@ ssize_t aofWrite(int fd, const char *buf, size_t len) { ...@@ -297,9 +303,7 @@ ssize_t aofWrite(int fd, const char *buf, size_t len) {
nwritten = write(fd, buf, len); nwritten = write(fd, buf, len);
if (nwritten < 0) { if (nwritten < 0) {
if (errno == EINTR) { if (errno == EINTR) continue;
continue;
}
return totwritten ? totwritten : -1; return totwritten ? totwritten : -1;
} }
...@@ -335,10 +339,24 @@ void flushAppendOnlyFile(int force) { ...@@ -335,10 +339,24 @@ void flushAppendOnlyFile(int force) {
int sync_in_progress = 0; int sync_in_progress = 0;
mstime_t latency; mstime_t latency;
if (sdslen(server.aof_buf) == 0) return; if (sdslen(server.aof_buf) == 0) {
/* Check if we need to do fsync even the aof buffer is empty,
* because previously in AOF_FSYNC_EVERYSEC mode, fsync is
* called only when aof buffer is not empty, so if users
* stop write commands before fsync called in one second,
* the data in page cache cannot be flushed in time. */
if (server.aof_fsync == AOF_FSYNC_EVERYSEC &&
server.aof_fsync_offset != server.aof_current_size &&
server.unixtime > server.aof_last_fsync &&
!(sync_in_progress = aofFsyncInProgress())) {
goto try_fsync;
} else {
return;
}
}
if (server.aof_fsync == AOF_FSYNC_EVERYSEC) if (server.aof_fsync == AOF_FSYNC_EVERYSEC)
sync_in_progress = bioPendingJobsOfType(BIO_AOF_FSYNC) != 0; sync_in_progress = aofFsyncInProgress();
if (server.aof_fsync == AOF_FSYNC_EVERYSEC && !force) { if (server.aof_fsync == AOF_FSYNC_EVERYSEC && !force) {
/* With this append fsync policy we do background fsyncing. /* With this append fsync policy we do background fsyncing.
...@@ -367,6 +385,10 @@ void flushAppendOnlyFile(int force) { ...@@ -367,6 +385,10 @@ void flushAppendOnlyFile(int force) {
* there is much to do about the whole server stopping for power problems * there is much to do about the whole server stopping for power problems
* or alike */ * or alike */
if (server.aof_flush_sleep && sdslen(server.aof_buf)) {
usleep(server.aof_flush_sleep);
}
latencyStartMonitor(latency); latencyStartMonitor(latency);
nwritten = aofWrite(server.aof_fd,server.aof_buf,sdslen(server.aof_buf)); nwritten = aofWrite(server.aof_fd,server.aof_buf,sdslen(server.aof_buf));
latencyEndMonitor(latency); latencyEndMonitor(latency);
...@@ -377,7 +399,7 @@ void flushAppendOnlyFile(int force) { ...@@ -377,7 +399,7 @@ void flushAppendOnlyFile(int force) {
* useful for graphing / monitoring purposes. */ * useful for graphing / monitoring purposes. */
if (sync_in_progress) { if (sync_in_progress) {
latencyAddSampleIfNeeded("aof-write-pending-fsync",latency); latencyAddSampleIfNeeded("aof-write-pending-fsync",latency);
} else if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) { } else if (hasActiveChildProcess()) {
latencyAddSampleIfNeeded("aof-write-active-child",latency); latencyAddSampleIfNeeded("aof-write-active-child",latency);
} else { } else {
latencyAddSampleIfNeeded("aof-write-alone",latency); latencyAddSampleIfNeeded("aof-write-alone",latency);
...@@ -470,11 +492,11 @@ void flushAppendOnlyFile(int force) { ...@@ -470,11 +492,11 @@ void flushAppendOnlyFile(int force) {
server.aof_buf = sdsempty(); server.aof_buf = sdsempty();
} }
try_fsync:
/* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are /* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are
* children doing I/O in the background. */ * children doing I/O in the background. */
if (server.aof_no_fsync_on_rewrite && if (server.aof_no_fsync_on_rewrite && hasActiveChildProcess())
(server.aof_child_pid != -1 || server.rdb_child_pid != -1)) return;
return;
/* Perform the fsync if needed. */ /* Perform the fsync if needed. */
if (server.aof_fsync == AOF_FSYNC_ALWAYS) { if (server.aof_fsync == AOF_FSYNC_ALWAYS) {
...@@ -484,10 +506,14 @@ void flushAppendOnlyFile(int force) { ...@@ -484,10 +506,14 @@ void flushAppendOnlyFile(int force) {
redis_fsync(server.aof_fd); /* Let's try to get this data on the disk */ redis_fsync(server.aof_fd); /* Let's try to get this data on the disk */
latencyEndMonitor(latency); latencyEndMonitor(latency);
latencyAddSampleIfNeeded("aof-fsync-always",latency); latencyAddSampleIfNeeded("aof-fsync-always",latency);
server.aof_fsync_offset = server.aof_current_size;
server.aof_last_fsync = server.unixtime; server.aof_last_fsync = server.unixtime;
} else if ((server.aof_fsync == AOF_FSYNC_EVERYSEC && } else if ((server.aof_fsync == AOF_FSYNC_EVERYSEC &&
server.unixtime > server.aof_last_fsync)) { server.unixtime > server.aof_last_fsync)) {
if (!sync_in_progress) aof_background_fsync(server.aof_fd); if (!sync_in_progress) {
aof_background_fsync(server.aof_fd);
server.aof_fsync_offset = server.aof_current_size;
}
server.aof_last_fsync = server.unixtime; server.aof_last_fsync = server.unixtime;
} }
} }
...@@ -626,11 +652,12 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a ...@@ -626,11 +652,12 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a
/* In Redis commands are always executed in the context of a client, so in /* In Redis commands are always executed in the context of a client, so in
* order to load the append only file we need to create a fake client. */ * order to load the append only file we need to create a fake client. */
struct client *createFakeClient(void) { struct client *createAOFClient(void) {
struct client *c = zmalloc(sizeof(*c)); struct client *c = zmalloc(sizeof(*c));
selectDb(c,0); selectDb(c,0);
c->fd = -1; c->id = CLIENT_ID_AOF; /* So modules can identify it's the AOF client. */
c->conn = NULL;
c->name = NULL; c->name = NULL;
c->querybuf = sdsempty(); c->querybuf = sdsempty();
c->querybuf_peak = 0; c->querybuf_peak = 0;
...@@ -694,6 +721,7 @@ int loadAppendOnlyFile(char *filename) { ...@@ -694,6 +721,7 @@ int loadAppendOnlyFile(char *filename) {
* operation is received. */ * operation is received. */
if (fp && redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0) { if (fp && redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0) {
server.aof_current_size = 0; server.aof_current_size = 0;
server.aof_fsync_offset = server.aof_current_size;
fclose(fp); fclose(fp);
return C_ERR; return C_ERR;
} }
...@@ -702,8 +730,8 @@ int loadAppendOnlyFile(char *filename) { ...@@ -702,8 +730,8 @@ int loadAppendOnlyFile(char *filename) {
* to the same file we're about to read. */ * to the same file we're about to read. */
server.aof_state = AOF_OFF; server.aof_state = AOF_OFF;
fakeClient = createFakeClient(); fakeClient = createAOFClient();
startLoading(fp); startLoadingFile(fp, filename);
/* Check if this AOF file has an RDB preamble. In that case we need to /* Check if this AOF file has an RDB preamble. In that case we need to
* load the RDB file and later continue loading the AOF tail. */ * load the RDB file and later continue loading the AOF tail. */
...@@ -812,6 +840,8 @@ int loadAppendOnlyFile(char *filename) { ...@@ -812,6 +840,8 @@ int loadAppendOnlyFile(char *filename) {
freeFakeClientArgv(fakeClient); freeFakeClientArgv(fakeClient);
fakeClient->cmd = NULL; fakeClient->cmd = NULL;
if (server.aof_load_truncated) valid_up_to = ftello(fp); if (server.aof_load_truncated) valid_up_to = ftello(fp);
if (server.key_load_delay)
usleep(server.key_load_delay);
} }
/* This point can only be reached when EOF is reached without errors. /* This point can only be reached when EOF is reached without errors.
...@@ -832,11 +862,13 @@ loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */ ...@@ -832,11 +862,13 @@ loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */
stopLoading(); stopLoading();
aofUpdateCurrentSize(); aofUpdateCurrentSize();
server.aof_rewrite_base_size = server.aof_current_size; server.aof_rewrite_base_size = server.aof_current_size;
server.aof_fsync_offset = server.aof_current_size;
return C_OK; return C_OK;
readerr: /* Read error. If feof(fp) is true, fall through to unexpected EOF. */ readerr: /* Read error. If feof(fp) is true, fall through to unexpected EOF. */
if (!feof(fp)) { if (!feof(fp)) {
if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */
fclose(fp);
serverLog(LL_WARNING,"Unrecoverable error reading the append only file: %s", strerror(errno)); serverLog(LL_WARNING,"Unrecoverable error reading the append only file: %s", strerror(errno));
exit(1); exit(1);
} }
...@@ -867,11 +899,13 @@ uxeof: /* Unexpected AOF end of file. */ ...@@ -867,11 +899,13 @@ uxeof: /* Unexpected AOF end of file. */
} }
} }
if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */
fclose(fp);
serverLog(LL_WARNING,"Unexpected end of file reading the append only file. You can: 1) Make a backup of your AOF file, then use ./redis-check-aof --fix <filename>. 2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server."); serverLog(LL_WARNING,"Unexpected end of file reading the append only file. You can: 1) Make a backup of your AOF file, then use ./redis-check-aof --fix <filename>. 2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server.");
exit(1); exit(1);
fmterr: /* Format error. */ fmterr: /* Format error. */
if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */
fclose(fp);
serverLog(LL_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix <filename>"); serverLog(LL_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix <filename>");
exit(1); exit(1);
} }
...@@ -1239,7 +1273,7 @@ int rewriteModuleObject(rio *r, robj *key, robj *o) { ...@@ -1239,7 +1273,7 @@ int rewriteModuleObject(rio *r, robj *key, robj *o) {
RedisModuleIO io; RedisModuleIO io;
moduleValue *mv = o->ptr; moduleValue *mv = o->ptr;
moduleType *mt = mv->type; moduleType *mt = mv->type;
moduleInitIOContext(io,mt,r); moduleInitIOContext(io,mt,r,key);
mt->aof_rewrite(&io,key,mv->value); mt->aof_rewrite(&io,key,mv->value);
if (io.ctx) { if (io.ctx) {
moduleFreeContext(io.ctx); moduleFreeContext(io.ctx);
...@@ -1535,39 +1569,24 @@ void aofClosePipes(void) { ...@@ -1535,39 +1569,24 @@ void aofClosePipes(void) {
*/ */
int rewriteAppendOnlyFileBackground(void) { int rewriteAppendOnlyFileBackground(void) {
pid_t childpid; pid_t childpid;
long long start;
if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) return C_ERR; if (hasActiveChildProcess()) return C_ERR;
if (aofCreatePipes() != C_OK) return C_ERR; if (aofCreatePipes() != C_OK) return C_ERR;
openChildInfoPipe(); openChildInfoPipe();
start = ustime(); if ((childpid = redisFork()) == 0) {
if ((childpid = fork()) == 0) {
char tmpfile[256]; char tmpfile[256];
/* Child */ /* Child */
closeListeningSockets(0);
redisSetProcTitle("redis-aof-rewrite"); redisSetProcTitle("redis-aof-rewrite");
snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid()); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
if (rewriteAppendOnlyFile(tmpfile) == C_OK) { if (rewriteAppendOnlyFile(tmpfile) == C_OK) {
size_t private_dirty = zmalloc_get_private_dirty(-1); sendChildCOWInfo(CHILD_INFO_TYPE_AOF, "AOF rewrite");
if (private_dirty) {
serverLog(LL_NOTICE,
"AOF rewrite: %zu MB of memory used by copy-on-write",
private_dirty/(1024*1024));
}
server.child_info_data.cow_size = private_dirty;
sendChildInfo(CHILD_INFO_TYPE_AOF);
exitFromChild(0); exitFromChild(0);
} else { } else {
exitFromChild(1); exitFromChild(1);
} }
} else { } else {
/* Parent */ /* Parent */
server.stat_fork_time = ustime()-start;
server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */
latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000);
if (childpid == -1) { if (childpid == -1) {
closeChildInfoPipe(); closeChildInfoPipe();
serverLog(LL_WARNING, serverLog(LL_WARNING,
...@@ -1581,7 +1600,6 @@ int rewriteAppendOnlyFileBackground(void) { ...@@ -1581,7 +1600,6 @@ int rewriteAppendOnlyFileBackground(void) {
server.aof_rewrite_scheduled = 0; server.aof_rewrite_scheduled = 0;
server.aof_rewrite_time_start = time(NULL); server.aof_rewrite_time_start = time(NULL);
server.aof_child_pid = childpid; server.aof_child_pid = childpid;
updateDictResizePolicy();
/* We set appendseldb to -1 in order to force the next call to the /* We set appendseldb to -1 in order to force the next call to the
* feedAppendOnlyFile() to issue a SELECT command, so the differences * feedAppendOnlyFile() to issue a SELECT command, so the differences
* accumulated by the parent into server.aof_rewrite_buf will start * accumulated by the parent into server.aof_rewrite_buf will start
...@@ -1596,13 +1614,14 @@ int rewriteAppendOnlyFileBackground(void) { ...@@ -1596,13 +1614,14 @@ int rewriteAppendOnlyFileBackground(void) {
void bgrewriteaofCommand(client *c) { void bgrewriteaofCommand(client *c) {
if (server.aof_child_pid != -1) { if (server.aof_child_pid != -1) {
addReplyError(c,"Background append only file rewriting already in progress"); addReplyError(c,"Background append only file rewriting already in progress");
} else if (server.rdb_child_pid != -1) { } else if (hasActiveChildProcess()) {
server.aof_rewrite_scheduled = 1; server.aof_rewrite_scheduled = 1;
addReplyStatus(c,"Background append only file rewriting scheduled"); addReplyStatus(c,"Background append only file rewriting scheduled");
} else if (rewriteAppendOnlyFileBackground() == C_OK) { } else if (rewriteAppendOnlyFileBackground() == C_OK) {
addReplyStatus(c,"Background append only file rewriting started"); addReplyStatus(c,"Background append only file rewriting started");
} else { } else {
addReply(c,shared.err); addReplyError(c,"Can't execute an AOF background rewriting. "
"Please check the server logs for more information.");
} }
} }
...@@ -1741,6 +1760,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { ...@@ -1741,6 +1760,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
server.aof_selected_db = -1; /* Make sure SELECT is re-issued */ server.aof_selected_db = -1; /* Make sure SELECT is re-issued */
aofUpdateCurrentSize(); aofUpdateCurrentSize();
server.aof_rewrite_base_size = server.aof_current_size; server.aof_rewrite_base_size = server.aof_current_size;
server.aof_current_size = server.aof_current_size;
/* Clear regular AOF buffer since its contents was just written to /* Clear regular AOF buffer since its contents was just written to
* the new AOF from the background rewrite buffer. */ * the new AOF from the background rewrite buffer. */
......
...@@ -994,12 +994,18 @@ void bitfieldCommand(client *c) { ...@@ -994,12 +994,18 @@ void bitfieldCommand(client *c) {
/* Lookup for read is ok if key doesn't exit, but errors /* Lookup for read is ok if key doesn't exit, but errors
* if it's not a string. */ * if it's not a string. */
o = lookupKeyRead(c->db,c->argv[1]); o = lookupKeyRead(c->db,c->argv[1]);
if (o != NULL && checkType(c,o,OBJ_STRING)) return; if (o != NULL && checkType(c,o,OBJ_STRING)) {
zfree(ops);
return;
}
} else { } else {
/* Lookup by making room up to the farest bit reached by /* Lookup by making room up to the farest bit reached by
* this operation. */ * this operation. */
if ((o = lookupStringForBitCommand(c, if ((o = lookupStringForBitCommand(c,
highest_write_offset)) == NULL) return; highest_write_offset)) == NULL) {
zfree(ops);
return;
}
} }
addReplyArrayLen(c,numops); addReplyArrayLen(c,numops);
......
This diff is collapsed.
...@@ -80,6 +80,8 @@ void receiveChildInfo(void) { ...@@ -80,6 +80,8 @@ void receiveChildInfo(void) {
server.stat_rdb_cow_bytes = server.child_info_data.cow_size; server.stat_rdb_cow_bytes = server.child_info_data.cow_size;
} else if (server.child_info_data.process_type == CHILD_INFO_TYPE_AOF) { } else if (server.child_info_data.process_type == CHILD_INFO_TYPE_AOF) {
server.stat_aof_cow_bytes = server.child_info_data.cow_size; server.stat_aof_cow_bytes = server.child_info_data.cow_size;
} else if (server.child_info_data.process_type == CHILD_INFO_TYPE_MODULE) {
server.stat_module_cow_bytes = server.child_info_data.cow_size;
} }
} }
} }
This diff is collapsed.
...@@ -40,7 +40,7 @@ struct clusterNode; ...@@ -40,7 +40,7 @@ struct clusterNode;
/* clusterLink encapsulates everything needed to talk with a remote node. */ /* clusterLink encapsulates everything needed to talk with a remote node. */
typedef struct clusterLink { typedef struct clusterLink {
mstime_t ctime; /* Link creation time */ mstime_t ctime; /* Link creation time */
int fd; /* TCP socket file descriptor */ connection *conn; /* Connection to remote node */
sds sndbuf; /* Packet send buffer */ sds sndbuf; /* Packet send buffer */
sds rcvbuf; /* Packet reception buffer */ sds rcvbuf; /* Packet reception buffer */
struct clusterNode *node; /* Node related to this link if any, or NULL */ struct clusterNode *node; /* Node related to this link if any, or NULL */
......
This diff is collapsed.
This diff is collapsed.
/*
* Copyright (c) 2019, Redis Labs
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __REDIS_CONNECTION_H
#define __REDIS_CONNECTION_H
#define CONN_INFO_LEN 32
struct aeEventLoop;
typedef struct connection connection;
typedef enum {
CONN_STATE_NONE = 0,
CONN_STATE_CONNECTING,
CONN_STATE_ACCEPTING,
CONN_STATE_CONNECTED,
CONN_STATE_CLOSED,
CONN_STATE_ERROR
} ConnectionState;
#define CONN_FLAG_IN_HANDLER (1<<0) /* A handler execution is in progress */
#define CONN_FLAG_CLOSE_SCHEDULED (1<<1) /* Closed scheduled by a handler */
#define CONN_FLAG_WRITE_BARRIER (1<<2) /* Write barrier requested */
typedef void (*ConnectionCallbackFunc)(struct connection *conn);
typedef struct ConnectionType {
void (*ae_handler)(struct aeEventLoop *el, int fd, void *clientData, int mask);
int (*connect)(struct connection *conn, const char *addr, int port, const char *source_addr, ConnectionCallbackFunc connect_handler);
int (*write)(struct connection *conn, const void *data, size_t data_len);
int (*read)(struct connection *conn, void *buf, size_t buf_len);
void (*close)(struct connection *conn);
int (*accept)(struct connection *conn, ConnectionCallbackFunc accept_handler);
int (*set_write_handler)(struct connection *conn, ConnectionCallbackFunc handler, int barrier);
int (*set_read_handler)(struct connection *conn, ConnectionCallbackFunc handler);
const char *(*get_last_error)(struct connection *conn);
int (*blocking_connect)(struct connection *conn, const char *addr, int port, long long timeout);
ssize_t (*sync_write)(struct connection *conn, char *ptr, ssize_t size, long long timeout);
ssize_t (*sync_read)(struct connection *conn, char *ptr, ssize_t size, long long timeout);
ssize_t (*sync_readline)(struct connection *conn, char *ptr, ssize_t size, long long timeout);
} ConnectionType;
struct connection {
ConnectionType *type;
ConnectionState state;
int flags;
int last_errno;
void *private_data;
ConnectionCallbackFunc conn_handler;
ConnectionCallbackFunc write_handler;
ConnectionCallbackFunc read_handler;
int fd;
};
/* The connection module does not deal with listening and accepting sockets,
* so we assume we have a socket when an incoming connection is created.
*
* The fd supplied should therefore be associated with an already accept()ed
* socket.
*
* connAccept() may directly call accept_handler(), or return and call it
* at a later time. This behavior is a bit awkward but aims to reduce the need
* to wait for the next event loop, if no additional handshake is required.
*/
static inline int connAccept(connection *conn, ConnectionCallbackFunc accept_handler) {
return conn->type->accept(conn, accept_handler);
}
/* Establish a connection. The connect_handler will be called when the connection
* is established, or if an error has occured.
*
* The connection handler will be responsible to set up any read/write handlers
* as needed.
*
* If C_ERR is returned, the operation failed and the connection handler shall
* not be expected.
*/
static inline int connConnect(connection *conn, const char *addr, int port, const char *src_addr,
ConnectionCallbackFunc connect_handler) {
return conn->type->connect(conn, addr, port, src_addr, connect_handler);
}
/* Blocking connect.
*
* NOTE: This is implemented in order to simplify the transition to the abstract
* connections, but should probably be refactored out of cluster.c and replication.c,
* in favor of a pure async implementation.
*/
static inline int connBlockingConnect(connection *conn, const char *addr, int port, long long timeout) {
return conn->type->blocking_connect(conn, addr, port, timeout);
}
/* Write to connection, behaves the same as write(2).
*
* Like write(2), a short write is possible. A -1 return indicates an error.
*
* The caller should NOT rely on errno. Testing for an EAGAIN-like condition, use
* connGetState() to see if the connection state is still CONN_STATE_CONNECTED.
*/
static inline int connWrite(connection *conn, const void *data, size_t data_len) {
return conn->type->write(conn, data, data_len);
}
/* Read from the connection, behaves the same as read(2).
*
* Like read(2), a short read is possible. A return value of 0 will indicate the
* connection was closed, and -1 will indicate an error.
*
* The caller should NOT rely on errno. Testing for an EAGAIN-like condition, use
* connGetState() to see if the connection state is still CONN_STATE_CONNECTED.
*/
static inline int connRead(connection *conn, void *buf, size_t buf_len) {
return conn->type->read(conn, buf, buf_len);
}
/* Register a write handler, to be called when the connection is writable.
* If NULL, the existing handler is removed.
*/
static inline int connSetWriteHandler(connection *conn, ConnectionCallbackFunc func) {
return conn->type->set_write_handler(conn, func, 0);
}
/* Register a read handler, to be called when the connection is readable.
* If NULL, the existing handler is removed.
*/
static inline int connSetReadHandler(connection *conn, ConnectionCallbackFunc func) {
return conn->type->set_read_handler(conn, func);
}
/* Set a write handler, and possibly enable a write barrier, this flag is
* cleared when write handler is changed or removed.
* With barroer enabled, we never fire the event if the read handler already
* fired in the same event loop iteration. Useful when you want to persist
* things to disk before sending replies, and want to do that in a group fashion. */
static inline int connSetWriteHandlerWithBarrier(connection *conn, ConnectionCallbackFunc func, int barrier) {
return conn->type->set_write_handler(conn, func, barrier);
}
static inline void connClose(connection *conn) {
conn->type->close(conn);
}
/* Returns the last error encountered by the connection, as a string. If no error,
* a NULL is returned.
*/
static inline const char *connGetLastError(connection *conn) {
return conn->type->get_last_error(conn);
}
static inline ssize_t connSyncWrite(connection *conn, char *ptr, ssize_t size, long long timeout) {
return conn->type->sync_write(conn, ptr, size, timeout);
}
static inline ssize_t connSyncRead(connection *conn, char *ptr, ssize_t size, long long timeout) {
return conn->type->sync_read(conn, ptr, size, timeout);
}
static inline ssize_t connSyncReadLine(connection *conn, char *ptr, ssize_t size, long long timeout) {
return conn->type->sync_readline(conn, ptr, size, timeout);
}
connection *connCreateSocket();
connection *connCreateAcceptedSocket(int fd);
connection *connCreateTLS();
connection *connCreateAcceptedTLS(int fd, int require_auth);
void connSetPrivateData(connection *conn, void *data);
void *connGetPrivateData(connection *conn);
int connGetState(connection *conn);
int connHasWriteHandler(connection *conn);
int connHasReadHandler(connection *conn);
int connGetSocketError(connection *conn);
/* anet-style wrappers to conns */
int connBlock(connection *conn);
int connNonBlock(connection *conn);
int connEnableTcpNoDelay(connection *conn);
int connDisableTcpNoDelay(connection *conn);
int connKeepAlive(connection *conn, int interval);
int connSendTimeout(connection *conn, long long ms);
int connRecvTimeout(connection *conn, long long ms);
int connPeerToString(connection *conn, char *ip, size_t ip_len, int *port);
int connFormatPeer(connection *conn, char *buf, size_t buf_len);
int connSockName(connection *conn, char *ip, size_t ip_len, int *port);
const char *connGetInfo(connection *conn, char *buf, size_t buf_len);
/* Helpers for tls special considerations */
int tlsHasPendingData();
void tlsProcessPendingData();
#endif /* __REDIS_CONNECTION_H */
/*
* Copyright (c) 2019, Redis Labs
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __REDIS_CONNHELPERS_H
#define __REDIS_CONNHELPERS_H
#include "connection.h"
/* These are helper functions that are common to different connection
* implementations (currently sockets in connection.c and TLS in tls.c).
*
* Currently helpers implement the mechanisms for invoking connection
* handlers, tracking in-handler states and dealing with deferred
* destruction (if invoked by a handler).
*/
/* Called whenever a handler is invoked on a connection and sets the
* CONN_FLAG_IN_HANDLER flag to indicate we're in a handler context.
*
* An attempt to close a connection while CONN_FLAG_IN_HANDLER is
* set will result with deferred close, i.e. setting the CONN_FLAG_CLOSE_SCHEDULED
* instead of destructing it.
*/
static inline void enterHandler(connection *conn) {
conn->flags |= CONN_FLAG_IN_HANDLER;
}
/* Called whenever a handler returns. This unsets the CONN_FLAG_IN_HANDLER
* flag and performs actual close/destruction if a deferred close was
* scheduled by the handler.
*/
static inline int exitHandler(connection *conn) {
conn->flags &= ~CONN_FLAG_IN_HANDLER;
if (conn->flags & CONN_FLAG_CLOSE_SCHEDULED) {
connClose(conn);
return 0;
}
return 1;
}
/* Helper for connection implementations to call handlers:
* 1. Mark the handler in use.
* 2. Execute the handler (if set).
* 3. Mark the handler as NOT in use and perform deferred close if was
* requested by the handler at any time.
*/
static inline int callHandler(connection *conn, ConnectionCallbackFunc handler) {
conn->flags |= CONN_FLAG_IN_HANDLER;
if (handler) handler(conn);
conn->flags &= ~CONN_FLAG_IN_HANDLER;
if (conn->flags & CONN_FLAG_CLOSE_SCHEDULED) {
connClose(conn);
return 0;
}
return 1;
}
#endif /* __REDIS_CONNHELPERS_H */
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment