Unverified Commit 5cbc73c6 authored by jared's avatar jared Committed by GitHub
Browse files

Merge pull request #1 from antirez/unstable

merge redis
parents a16aa03a d5536e04
......@@ -3,7 +3,9 @@
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <netdb.h>
#include <assert.h>
#include <unistd.h>
#include <signal.h>
......@@ -91,7 +93,7 @@ static int disconnect(redisContext *c, int keep_fd) {
return -1;
}
static redisContext *connect(struct config config) {
static redisContext *do_connect(struct config config) {
redisContext *c = NULL;
if (config.type == CONN_TCP) {
......@@ -248,7 +250,7 @@ static void test_append_formatted_commands(struct config config) {
char *cmd;
int len;
c = connect(config);
c = do_connect(config);
test("Append format command: ");
......@@ -302,6 +304,82 @@ static void test_reply_reader(void) {
strncasecmp(reader->errstr,"No support for",14) == 0);
redisReaderFree(reader);
test("Correctly parses LLONG_MAX: ");
reader = redisReaderCreate();
redisReaderFeed(reader, ":9223372036854775807\r\n",22);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_OK &&
((redisReply*)reply)->type == REDIS_REPLY_INTEGER &&
((redisReply*)reply)->integer == LLONG_MAX);
freeReplyObject(reply);
redisReaderFree(reader);
test("Set error when > LLONG_MAX: ");
reader = redisReaderCreate();
redisReaderFeed(reader, ":9223372036854775808\r\n",22);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_ERR &&
strcasecmp(reader->errstr,"Bad integer value") == 0);
freeReplyObject(reply);
redisReaderFree(reader);
test("Correctly parses LLONG_MIN: ");
reader = redisReaderCreate();
redisReaderFeed(reader, ":-9223372036854775808\r\n",23);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_OK &&
((redisReply*)reply)->type == REDIS_REPLY_INTEGER &&
((redisReply*)reply)->integer == LLONG_MIN);
freeReplyObject(reply);
redisReaderFree(reader);
test("Set error when < LLONG_MIN: ");
reader = redisReaderCreate();
redisReaderFeed(reader, ":-9223372036854775809\r\n",23);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_ERR &&
strcasecmp(reader->errstr,"Bad integer value") == 0);
freeReplyObject(reply);
redisReaderFree(reader);
test("Set error when array < -1: ");
reader = redisReaderCreate();
redisReaderFeed(reader, "*-2\r\n+asdf\r\n",12);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_ERR &&
strcasecmp(reader->errstr,"Multi-bulk length out of range") == 0);
freeReplyObject(reply);
redisReaderFree(reader);
test("Set error when bulk < -1: ");
reader = redisReaderCreate();
redisReaderFeed(reader, "$-2\r\nasdf\r\n",11);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_ERR &&
strcasecmp(reader->errstr,"Bulk string length out of range") == 0);
freeReplyObject(reply);
redisReaderFree(reader);
test("Set error when array > INT_MAX: ");
reader = redisReaderCreate();
redisReaderFeed(reader, "*9223372036854775807\r\n+asdf\r\n",29);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_ERR &&
strcasecmp(reader->errstr,"Multi-bulk length out of range") == 0);
freeReplyObject(reply);
redisReaderFree(reader);
#if LLONG_MAX > SIZE_MAX
test("Set error when bulk > SIZE_MAX: ");
reader = redisReaderCreate();
redisReaderFeed(reader, "$9223372036854775807\r\nasdf\r\n",28);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_ERR &&
strcasecmp(reader->errstr,"Bulk string length out of range") == 0);
freeReplyObject(reply);
redisReaderFree(reader);
#endif
test("Works with NULL functions for reply: ");
reader = redisReaderCreate();
reader->fn = NULL;
......@@ -358,18 +436,32 @@ static void test_free_null(void) {
static void test_blocking_connection_errors(void) {
redisContext *c;
struct addrinfo hints = {.ai_family = AF_INET};
struct addrinfo *ai_tmp = NULL;
const char *bad_domain = "idontexist.com";
int rv = getaddrinfo(bad_domain, "6379", &hints, &ai_tmp);
if (rv != 0) {
// Address does *not* exist
test("Returns error when host cannot be resolved: ");
c = redisConnect((char*)"idontexist.test", 6379);
test_cond(c->err == REDIS_ERR_OTHER &&
(strcmp(c->errstr,"Name or service not known") == 0 ||
strcmp(c->errstr,"Can't resolve: idontexist.test") == 0 ||
strcmp(c->errstr,"nodename nor servname provided, or not known") == 0 ||
strcmp(c->errstr,"No address associated with hostname") == 0 ||
strcmp(c->errstr,"Temporary failure in name resolution") == 0 ||
strcmp(c->errstr,"hostname nor servname provided, or not known") == 0 ||
strcmp(c->errstr,"no address associated with name") == 0));
// First see if this domain name *actually* resolves to NXDOMAIN
c = redisConnect("dontexist.com", 6379);
test_cond(
c->err == REDIS_ERR_OTHER &&
(strcmp(c->errstr, "Name or service not known") == 0 ||
strcmp(c->errstr, "Can't resolve: sadkfjaskfjsa.com") == 0 ||
strcmp(c->errstr,
"nodename nor servname provided, or not known") == 0 ||
strcmp(c->errstr, "No address associated with hostname") == 0 ||
strcmp(c->errstr, "Temporary failure in name resolution") == 0 ||
strcmp(c->errstr,
"hostname nor servname provided, or not known") == 0 ||
strcmp(c->errstr, "no address associated with name") == 0));
redisFree(c);
} else {
printf("Skipping NXDOMAIN test. Found evil ISP!\n");
freeaddrinfo(ai_tmp);
}
test("Returns error when the port is not open: ");
c = redisConnect((char*)"localhost", 1);
......@@ -387,7 +479,7 @@ static void test_blocking_connection(struct config config) {
redisContext *c;
redisReply *reply;
c = connect(config);
c = do_connect(config);
test("Is able to deliver commands: ");
reply = redisCommand(c,"PING");
......@@ -468,7 +560,7 @@ static void test_blocking_connection_timeouts(struct config config) {
const char *cmd = "DEBUG SLEEP 3\r\n";
struct timeval tv;
c = connect(config);
c = do_connect(config);
test("Successfully completes a command when the timeout is not exceeded: ");
reply = redisCommand(c,"SET foo fast");
freeReplyObject(reply);
......@@ -480,7 +572,7 @@ static void test_blocking_connection_timeouts(struct config config) {
freeReplyObject(reply);
disconnect(c, 0);
c = connect(config);
c = do_connect(config);
test("Does not return a reply when the command times out: ");
s = write(c->fd, cmd, strlen(cmd));
tv.tv_sec = 0;
......@@ -514,7 +606,7 @@ static void test_blocking_io_errors(struct config config) {
int major, minor;
/* Connect to target given by config. */
c = connect(config);
c = do_connect(config);
{
/* Find out Redis version to determine the path for the next test */
const char *field = "redis_version:";
......@@ -549,7 +641,7 @@ static void test_blocking_io_errors(struct config config) {
strcmp(c->errstr,"Server closed the connection") == 0);
redisFree(c);
c = connect(config);
c = do_connect(config);
test("Returns I/O error on socket timeout: ");
struct timeval tv = { 0, 1000 };
assert(redisSetTimeout(c,tv) == REDIS_OK);
......@@ -583,7 +675,7 @@ static void test_invalid_timeout_errors(struct config config) {
}
static void test_throughput(struct config config) {
redisContext *c = connect(config);
redisContext *c = do_connect(config);
redisReply **replies;
int i, num;
long long t1, t2;
......@@ -616,6 +708,17 @@ static void test_throughput(struct config config) {
free(replies);
printf("\t(%dx LRANGE with 500 elements: %.3fs)\n", num, (t2-t1)/1000000.0);
replies = malloc(sizeof(redisReply*)*num);
t1 = usec();
for (i = 0; i < num; i++) {
replies[i] = redisCommand(c, "INCRBY incrkey %d", 1000000);
assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_INTEGER);
}
t2 = usec();
for (i = 0; i < num; i++) freeReplyObject(replies[i]);
free(replies);
printf("\t(%dx INCRBY: %.3fs)\n", num, (t2-t1)/1000000.0);
num = 10000;
replies = malloc(sizeof(redisReply*)*num);
for (i = 0; i < num; i++)
......@@ -644,6 +747,19 @@ static void test_throughput(struct config config) {
free(replies);
printf("\t(%dx LRANGE with 500 elements (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0);
replies = malloc(sizeof(redisReply*)*num);
for (i = 0; i < num; i++)
redisAppendCommand(c,"INCRBY incrkey %d", 1000000);
t1 = usec();
for (i = 0; i < num; i++) {
assert(redisGetReply(c, (void*)&replies[i]) == REDIS_OK);
assert(replies[i] != NULL && replies[i]->type == REDIS_REPLY_INTEGER);
}
t2 = usec();
for (i = 0; i < num; i++) freeReplyObject(replies[i]);
free(replies);
printf("\t(%dx INCRBY (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0);
disconnect(c, 0);
}
......
......@@ -215,4 +215,32 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra,
return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero);
}
JEMALLOC_ALWAYS_INLINE int
iget_defrag_hint(tsdn_t *tsdn, void* ptr, int *bin_util, int *run_util) {
int defrag = 0;
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
szind_t szind;
bool is_slab;
rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &szind, &is_slab);
if (likely(is_slab)) {
/* Small allocation. */
extent_t *slab = iealloc(tsdn, ptr);
arena_t *arena = extent_arena_get(slab);
szind_t binind = extent_szind_get(slab);
bin_t *bin = &arena->bins[binind];
malloc_mutex_lock(tsdn, &bin->lock);
/* don't bother moving allocations from the slab currently used for new allocations */
if (slab != bin->slabcur) {
const bin_info_t *bin_info = &bin_infos[binind];
size_t availregs = bin_info->nregs * bin->stats.curslabs;
*bin_util = ((long long)bin->stats.curregs<<16) / availregs;
*run_util = ((long long)(bin_info->nregs - extent_nfree_get(slab))<<16) / bin_info->nregs;
defrag = 1;
}
malloc_mutex_unlock(tsdn, &bin->lock);
}
return defrag;
}
#endif /* JEMALLOC_INTERNAL_INLINES_C_H */
......@@ -120,3 +120,7 @@
# define JEMALLOC_RESTRICT_RETURN
# define JEMALLOC_ALLOCATOR
#endif
/* This version of Jemalloc, modified for Redis, has the je_get_defrag_hint()
* function. */
#define JEMALLOC_FRAG_HINT
......@@ -3324,3 +3324,14 @@ jemalloc_postfork_child(void) {
}
/******************************************************************************/
/* Helps the application decide if a pointer is worth re-allocating in order to reduce fragmentation.
* returns 0 if the allocation is in the currently active run,
* or when it is not causing any frag issue (large or huge bin)
* returns the bin utilization and run utilization both in fixed point 16:16.
* If the application decides to re-allocate it should use MALLOCX_TCACHE_NONE when doing so. */
JEMALLOC_EXPORT int JEMALLOC_NOTHROW
get_defrag_hint(void* ptr, int *bin_util, int *run_util) {
assert(ptr != NULL);
return iget_defrag_hint(TSDN_NULL, ptr, bin_util, run_util);
}
......@@ -264,86 +264,100 @@ dir ./
################################# REPLICATION #################################
# Master-Slave replication. Use slaveof to make a Redis instance a copy of
# Master-Replica replication. Use replicaof to make a Redis instance a copy of
# another Redis server. A few things to understand ASAP about Redis replication.
#
# +------------------+ +---------------+
# | Master | ---> | Replica |
# | (receive writes) | | (exact copy) |
# +------------------+ +---------------+
#
# 1) Redis replication is asynchronous, but you can configure a master to
# stop accepting writes if it appears to be not connected with at least
# a given number of slaves.
# 2) Redis slaves are able to perform a partial resynchronization with the
# a given number of replicas.
# 2) Redis replicas are able to perform a partial resynchronization with the
# master if the replication link is lost for a relatively small amount of
# time. You may want to configure the replication backlog size (see the next
# sections of this file) with a sensible value depending on your needs.
# 3) Replication is automatic and does not need user intervention. After a
# network partition slaves automatically try to reconnect to masters
# network partition replicas automatically try to reconnect to masters
# and resynchronize with them.
#
# slaveof <masterip> <masterport>
# replicaof <masterip> <masterport>
# If the master is password protected (using the "requirepass" configuration
# directive below) it is possible to tell the slave to authenticate before
# directive below) it is possible to tell the replica to authenticate before
# starting the replication synchronization process, otherwise the master will
# refuse the slave request.
# refuse the replica request.
#
# masterauth <master-password>
#
# However this is not enough if you are using Redis ACLs (for Redis version
# 6 or greater), and the default user is not capable of running the PSYNC
# command and/or other commands needed for replication. In this case it's
# better to configure a special user to use with replication, and specify the
# masteruser configuration as such:
#
# masteruser <username>
#
# When masteruser is specified, the replica will authenticate against its
# master using the new AUTH form: AUTH <username> <password>.
# When a slave loses its connection with the master, or when the replication
# is still in progress, the slave can act in two different ways:
# When a replica loses its connection with the master, or when the replication
# is still in progress, the replica can act in two different ways:
#
# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will
# 1) if replica-serve-stale-data is set to 'yes' (the default) the replica will
# still reply to client requests, possibly with out of date data, or the
# data set may just be empty if this is the first synchronization.
#
# 2) if slave-serve-stale-data is set to 'no' the slave will reply with
# 2) if replica-serve-stale-data is set to 'no' the replica will reply with
# an error "SYNC with master in progress" to all the kind of commands
# but to INFO, SLAVEOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG,
# but to INFO, replicaOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG,
# SUBSCRIBE, UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB,
# COMMAND, POST, HOST: and LATENCY.
#
slave-serve-stale-data yes
replica-serve-stale-data yes
# You can configure a slave instance to accept writes or not. Writing against
# a slave instance may be useful to store some ephemeral data (because data
# written on a slave will be easily deleted after resync with the master) but
# You can configure a replica instance to accept writes or not. Writing against
# a replica instance may be useful to store some ephemeral data (because data
# written on a replica will be easily deleted after resync with the master) but
# may also cause problems if clients are writing to it because of a
# misconfiguration.
#
# Since Redis 2.6 by default slaves are read-only.
# Since Redis 2.6 by default replicas are read-only.
#
# Note: read only slaves are not designed to be exposed to untrusted clients
# Note: read only replicas are not designed to be exposed to untrusted clients
# on the internet. It's just a protection layer against misuse of the instance.
# Still a read only slave exports by default all the administrative commands
# Still a read only replica exports by default all the administrative commands
# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve
# security of read only slaves using 'rename-command' to shadow all the
# security of read only replicas using 'rename-command' to shadow all the
# administrative / dangerous commands.
slave-read-only yes
replica-read-only yes
# Replication SYNC strategy: disk or socket.
#
# -------------------------------------------------------
# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY
# -------------------------------------------------------
# New replicas and reconnecting replicas that are not able to continue the
# replication process just receiving differences, need to do what is called a
# "full synchronization". An RDB file is transmitted from the master to the
# replicas.
#
# New slaves and reconnecting slaves that are not able to continue the replication
# process just receiving differences, need to do what is called a "full
# synchronization". An RDB file is transmitted from the master to the slaves.
# The transmission can happen in two different ways:
#
# 1) Disk-backed: The Redis master creates a new process that writes the RDB
# file on disk. Later the file is transferred by the parent
# process to the slaves incrementally.
# process to the replicas incrementally.
# 2) Diskless: The Redis master creates a new process that directly writes the
# RDB file to slave sockets, without touching the disk at all.
# RDB file to replica sockets, without touching the disk at all.
#
# With disk-backed replication, while the RDB file is generated, more slaves
# can be queued and served with the RDB file as soon as the current child producing
# the RDB file finishes its work. With diskless replication instead once
# the transfer starts, new slaves arriving will be queued and a new transfer
# will start when the current one terminates.
# With disk-backed replication, while the RDB file is generated, more replicas
# can be queued and served with the RDB file as soon as the current child
# producing the RDB file finishes its work. With diskless replication instead
# once the transfer starts, new replicas arriving will be queued and a new
# transfer will start when the current one terminates.
#
# When diskless replication is used, the master waits a configurable amount of
# time (in seconds) before starting the transfer in the hope that multiple slaves
# will arrive and the transfer can be parallelized.
# time (in seconds) before starting the transfer in the hope that multiple
# replicas will arrive and the transfer can be parallelized.
#
# With slow disks and fast (large bandwidth) networks, diskless replication
# works better.
......@@ -351,157 +365,323 @@ repl-diskless-sync no
# When diskless replication is enabled, it is possible to configure the delay
# the server waits in order to spawn the child that transfers the RDB via socket
# to the slaves.
# to the replicas.
#
# This is important since once the transfer starts, it is not possible to serve
# new slaves arriving, that will be queued for the next RDB transfer, so the server
# waits a delay in order to let more slaves arrive.
# new replicas arriving, that will be queued for the next RDB transfer, so the
# server waits a delay in order to let more replicas arrive.
#
# The delay is specified in seconds, and by default is 5 seconds. To disable
# it entirely just set it to 0 seconds and the transfer will start ASAP.
repl-diskless-sync-delay 5
# Slaves send PINGs to server in a predefined interval. It's possible to change
# this interval with the repl_ping_slave_period option. The default value is 10
# seconds.
#
# repl-ping-slave-period 10
# -----------------------------------------------------------------------------
# WARNING: RDB diskless load is experimental. Since in this setup the replica
# does not immediately store an RDB on disk, it may cause data loss during
# failovers. RDB diskless load + Redis modules not handling I/O reads may also
# cause Redis to abort in case of I/O errors during the initial synchronization
# stage with the master. Use only if your do what you are doing.
# -----------------------------------------------------------------------------
#
# Replica can load the RDB it reads from the replication link directly from the
# socket, or store the RDB to a file and read that file after it was completely
# recived from the master.
#
# In many cases the disk is slower than the network, and storing and loading
# the RDB file may increase replication time (and even increase the master's
# Copy on Write memory and salve buffers).
# However, parsing the RDB file directly from the socket may mean that we have
# to flush the contents of the current database before the full rdb was
# received. For this reason we have the following options:
#
# "disabled" - Don't use diskless load (store the rdb file to the disk first)
# "on-empty-db" - Use diskless load only when it is completely safe.
# "swapdb" - Keep a copy of the current db contents in RAM while parsing
# the data directly from the socket. note that this requires
# sufficient memory, if you don't have it, you risk an OOM kill.
repl-diskless-load disabled
# Replicas send PINGs to server in a predefined interval. It's possible to
# change this interval with the repl_ping_replica_period option. The default
# value is 10 seconds.
#
# repl-ping-replica-period 10
# The following option sets the replication timeout for:
#
# 1) Bulk transfer I/O during SYNC, from the point of view of slave.
# 2) Master timeout from the point of view of slaves (data, pings).
# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings).
# 1) Bulk transfer I/O during SYNC, from the point of view of replica.
# 2) Master timeout from the point of view of replicas (data, pings).
# 3) Replica timeout from the point of view of masters (REPLCONF ACK pings).
#
# It is important to make sure that this value is greater than the value
# specified for repl-ping-slave-period otherwise a timeout will be detected
# every time there is low traffic between the master and the slave.
# specified for repl-ping-replica-period otherwise a timeout will be detected
# every time there is low traffic between the master and the replica.
#
# repl-timeout 60
# Disable TCP_NODELAY on the slave socket after SYNC?
# Disable TCP_NODELAY on the replica socket after SYNC?
#
# If you select "yes" Redis will use a smaller number of TCP packets and
# less bandwidth to send data to slaves. But this can add a delay for
# the data to appear on the slave side, up to 40 milliseconds with
# less bandwidth to send data to replicas. But this can add a delay for
# the data to appear on the replica side, up to 40 milliseconds with
# Linux kernels using a default configuration.
#
# If you select "no" the delay for data to appear on the slave side will
# If you select "no" the delay for data to appear on the replica side will
# be reduced but more bandwidth will be used for replication.
#
# By default we optimize for low latency, but in very high traffic conditions
# or when the master and slaves are many hops away, turning this to "yes" may
# or when the master and replicas are many hops away, turning this to "yes" may
# be a good idea.
repl-disable-tcp-nodelay no
# Set the replication backlog size. The backlog is a buffer that accumulates
# slave data when slaves are disconnected for some time, so that when a slave
# wants to reconnect again, often a full resync is not needed, but a partial
# resync is enough, just passing the portion of data the slave missed while
# disconnected.
# replica data when replicas are disconnected for some time, so that when a
# replica wants to reconnect again, often a full resync is not needed, but a
# partial resync is enough, just passing the portion of data the replica
# missed while disconnected.
#
# The bigger the replication backlog, the longer the time the slave can be
# The bigger the replication backlog, the longer the time the replica can be
# disconnected and later be able to perform a partial resynchronization.
#
# The backlog is only allocated once there is at least a slave connected.
# The backlog is only allocated once there is at least a replica connected.
#
# repl-backlog-size 1mb
# After a master has no longer connected slaves for some time, the backlog
# After a master has no longer connected replicas for some time, the backlog
# will be freed. The following option configures the amount of seconds that
# need to elapse, starting from the time the last slave disconnected, for
# need to elapse, starting from the time the last replica disconnected, for
# the backlog buffer to be freed.
#
# Note that slaves never free the backlog for timeout, since they may be
# Note that replicas never free the backlog for timeout, since they may be
# promoted to masters later, and should be able to correctly "partially
# resynchronize" with the slaves: hence they should always accumulate backlog.
# resynchronize" with the replicas: hence they should always accumulate backlog.
#
# A value of 0 means to never release the backlog.
#
# repl-backlog-ttl 3600
# The slave priority is an integer number published by Redis in the INFO output.
# It is used by Redis Sentinel in order to select a slave to promote into a
# master if the master is no longer working correctly.
# The replica priority is an integer number published by Redis in the INFO
# output. It is used by Redis Sentinel in order to select a replica to promote
# into a master if the master is no longer working correctly.
#
# A slave with a low priority number is considered better for promotion, so
# for instance if there are three slaves with priority 10, 100, 25 Sentinel will
# pick the one with priority 10, that is the lowest.
# A replica with a low priority number is considered better for promotion, so
# for instance if there are three replicas with priority 10, 100, 25 Sentinel
# will pick the one with priority 10, that is the lowest.
#
# However a special priority of 0 marks the slave as not able to perform the
# role of master, so a slave with priority of 0 will never be selected by
# However a special priority of 0 marks the replica as not able to perform the
# role of master, so a replica with priority of 0 will never be selected by
# Redis Sentinel for promotion.
#
# By default the priority is 100.
slave-priority 100
replica-priority 100
# It is possible for a master to stop accepting writes if there are less than
# N slaves connected, having a lag less or equal than M seconds.
# N replicas connected, having a lag less or equal than M seconds.
#
# The N slaves need to be in "online" state.
# The N replicas need to be in "online" state.
#
# The lag in seconds, that must be <= the specified value, is calculated from
# the last ping received from the slave, that is usually sent every second.
# the last ping received from the replica, that is usually sent every second.
#
# This option does not GUARANTEE that N replicas will accept the write, but
# will limit the window of exposure for lost writes in case not enough slaves
# will limit the window of exposure for lost writes in case not enough replicas
# are available, to the specified number of seconds.
#
# For example to require at least 3 slaves with a lag <= 10 seconds use:
# For example to require at least 3 replicas with a lag <= 10 seconds use:
#
# min-slaves-to-write 3
# min-slaves-max-lag 10
# min-replicas-to-write 3
# min-replicas-max-lag 10
#
# Setting one or the other to 0 disables the feature.
#
# By default min-slaves-to-write is set to 0 (feature disabled) and
# min-slaves-max-lag is set to 10.
# By default min-replicas-to-write is set to 0 (feature disabled) and
# min-replicas-max-lag is set to 10.
# A Redis master is able to list the address and port of the attached
# slaves in different ways. For example the "INFO replication" section
# replicas in different ways. For example the "INFO replication" section
# offers this information, which is used, among other tools, by
# Redis Sentinel in order to discover slave instances.
# Redis Sentinel in order to discover replica instances.
# Another place where this info is available is in the output of the
# "ROLE" command of a master.
#
# The listed IP and address normally reported by a slave is obtained
# The listed IP and address normally reported by a replica is obtained
# in the following way:
#
# IP: The address is auto detected by checking the peer address
# of the socket used by the slave to connect with the master.
# of the socket used by the replica to connect with the master.
#
# Port: The port is communicated by the slave during the replication
# handshake, and is normally the port that the slave is using to
# list for connections.
# Port: The port is communicated by the replica during the replication
# handshake, and is normally the port that the replica is using to
# listen for connections.
#
# However when port forwarding or Network Address Translation (NAT) is
# used, the slave may be actually reachable via different IP and port
# pairs. The following two options can be used by a slave in order to
# used, the replica may be actually reachable via different IP and port
# pairs. The following two options can be used by a replica in order to
# report to its master a specific set of IP and port, so that both INFO
# and ROLE will report those values.
#
# There is no need to use both the options if you need to override just
# the port or the IP address.
#
# slave-announce-ip 5.5.5.5
# slave-announce-port 1234
# replica-announce-ip 5.5.5.5
# replica-announce-port 1234
################################## SECURITY ###################################
############################### KEYS TRACKING #################################
# Require clients to issue AUTH <PASSWORD> before processing any other
# commands. This might be useful in environments in which you do not trust
# others with access to the host running redis-server.
# Redis implements server assisted support for client side caching of values.
# This is implemented using an invalidation table that remembers, using
# 16 millions of slots, what clients may have certain subsets of keys. In turn
# this is used in order to send invalidation messages to clients. Please
# to understand more about the feature check this page:
#
# https://redis.io/topics/client-side-caching
#
# When tracking is enabled for a client, all the read only queries are assumed
# to be cached: this will force Redis to store information in the invalidation
# table. When keys are modified, such information is flushed away, and
# invalidation messages are sent to the clients. However if the workload is
# heavily dominated by reads, Redis could use more and more memory in order
# to track the keys fetched by many clients.
#
# This should stay commented out for backward compatibility and because most
# people do not need auth (e.g. they run their own servers).
# For this reason it is possible to configure a maximum fill value for the
# invalidation table. By default it is set to 10%, and once this limit is
# reached, Redis will start to evict caching slots in the invalidation table
# even if keys are not modified, just to reclaim memory: this will in turn
# force the clients to invalidate the cached values. Basically the table
# maximum fill rate is a trade off between the memory you want to spend server
# side to track information about who cached what, and the ability of clients
# to retain cached objects in memory.
#
# If you set the value to 0, it means there are no limits, and all the 16
# millions of caching slots can be used at the same time. In the "stats"
# INFO section, you can find information about the amount of caching slots
# used at every given moment.
#
# tracking-table-max-fill 10
################################## SECURITY ###################################
# Warning: since Redis is pretty fast an outside user can try up to
# 150k passwords per second against a good box. This means that you should
# use a very strong password otherwise it will be very easy to break.
# 1 million passwords per second against a modern box. This means that you
# should use very strong passwords, otherwise they will be very easy to break.
# Note that because the password is really a shared secret between the client
# and the server, and should not be memorized by any human, the password
# can be easily a long string from /dev/urandom or whatever, so by using a
# long and unguessable password no brute force attack will be possible.
# Redis ACL users are defined in the following format:
#
# user <username> ... acl rules ...
#
# For example:
#
# user worker +@list +@connection ~jobs:* on >ffa9203c493aa99
#
# The special username "default" is used for new connections. If this user
# has the "nopass" rule, then new connections will be immediately authenticated
# as the "default" user without the need of any password provided via the
# AUTH command. Otherwise if the "default" user is not flagged with "nopass"
# the connections will start in not authenticated state, and will require
# AUTH (or the HELLO command AUTH option) in order to be authenticated and
# start to work.
#
# The ACL rules that describe what an user can do are the following:
#
# on Enable the user: it is possible to authenticate as this user.
# off Disable the user: it's no longer possible to authenticate
# with this user, however the already authenticated connections
# will still work.
# +<command> Allow the execution of that command
# -<command> Disallow the execution of that command
# +@<category> Allow the execution of all the commands in such category
# with valid categories are like @admin, @set, @sortedset, ...
# and so forth, see the full list in the server.c file where
# the Redis command table is described and defined.
# The special category @all means all the commands, but currently
# present in the server, and that will be loaded in the future
# via modules.
# +<command>|subcommand Allow a specific subcommand of an otherwise
# disabled command. Note that this form is not
# allowed as negative like -DEBUG|SEGFAULT, but
# only additive starting with "+".
# allcommands Alias for +@all. Note that it implies the ability to execute
# all the future commands loaded via the modules system.
# nocommands Alias for -@all.
# ~<pattern> Add a pattern of keys that can be mentioned as part of
# commands. For instance ~* allows all the keys. The pattern
# is a glob-style pattern like the one of KEYS.
# It is possible to specify multiple patterns.
# allkeys Alias for ~*
# resetkeys Flush the list of allowed keys patterns.
# ><password> Add this passowrd to the list of valid password for the user.
# For example >mypass will add "mypass" to the list.
# This directive clears the "nopass" flag (see later).
# <<password> Remove this password from the list of valid passwords.
# nopass All the set passwords of the user are removed, and the user
# is flagged as requiring no password: it means that every
# password will work against this user. If this directive is
# used for the default user, every new connection will be
# immediately authenticated with the default user without
# any explicit AUTH command required. Note that the "resetpass"
# directive will clear this condition.
# resetpass Flush the list of allowed passwords. Moreover removes the
# "nopass" status. After "resetpass" the user has no associated
# passwords and there is no way to authenticate without adding
# some password (or setting it as "nopass" later).
# reset Performs the following actions: resetpass, resetkeys, off,
# -@all. The user returns to the same state it has immediately
# after its creation.
#
# ACL rules can be specified in any order: for instance you can start with
# passwords, then flags, or key patterns. However note that the additive
# and subtractive rules will CHANGE MEANING depending on the ordering.
# For instance see the following example:
#
# user alice on +@all -DEBUG ~* >somepassword
#
# This will allow "alice" to use all the commands with the exception of the
# DEBUG command, since +@all added all the commands to the set of the commands
# alice can use, and later DEBUG was removed. However if we invert the order
# of two ACL rules the result will be different:
#
# user alice on -DEBUG +@all ~* >somepassword
#
# Now DEBUG was removed when alice had yet no commands in the set of allowed
# commands, later all the commands are added, so the user will be able to
# execute everything.
#
# Basically ACL rules are processed left-to-right.
#
# For more information about ACL configuration please refer to
# the Redis web site at https://redis.io/topics/acl
# Using an external ACL file
#
# Instead of configuring users here in this file, it is possible to use
# a stand-alone file just listing users. The two methods cannot be mixed:
# if you configure users here and at the same time you activate the exteranl
# ACL file, the server will refuse to start.
#
# The format of the external ACL user file is exactly the same as the
# format that is used inside redis.conf to describe users.
#
# aclfile /etc/redis/users.acl
# IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatiblity
# layer on top of the new ACL system. The option effect will be just setting
# the password for the default user. Clients will still authenticate using
# AUTH <password> as usually, or more explicitly with AUTH default <password>
# if they follow the new protocol: both will work.
#
# requirepass foobared
# Command renaming.
# Command renaming (DEPRECATED).
#
# ------------------------------------------------------------------------
# WARNING: avoid using this option if possible. Instead use ACLs to remove
# commands from the default user, and put them only in some admin user you
# create for administrative purposes.
# ------------------------------------------------------------------------
#
# It is possible to change the name of dangerous commands in a shared
# environment. For instance the CONFIG command may be renamed into something
......@@ -518,7 +698,7 @@ slave-priority 100
# rename-command CONFIG ""
#
# Please note that changing the name of commands that are logged into the
# AOF file or transmitted to slaves may cause problems.
# AOF file or transmitted to replicas may cause problems.
################################### CLIENTS ####################################
......@@ -547,15 +727,15 @@ slave-priority 100
# This option is usually useful when using Redis as an LRU or LFU cache, or to
# set a hard memory limit for an instance (using the 'noeviction' policy).
#
# WARNING: If you have slaves attached to an instance with maxmemory on,
# the size of the output buffers needed to feed the slaves are subtracted
# WARNING: If you have replicas attached to an instance with maxmemory on,
# the size of the output buffers needed to feed the replicas are subtracted
# from the used memory count, so that network problems / resyncs will
# not trigger a loop where keys are evicted, and in turn the output
# buffer of slaves is full with DELs of keys evicted triggering the deletion
# buffer of replicas is full with DELs of keys evicted triggering the deletion
# of more keys, and so forth until the database is completely emptied.
#
# In short... if you have slaves attached it is suggested that you set a lower
# limit for maxmemory so that there is some free RAM on the system for slave
# In short... if you have replicas attached it is suggested that you set a lower
# limit for maxmemory so that there is some free RAM on the system for replica
# output buffers (but this is not needed if the policy is 'noeviction').
#
# maxmemory <bytes>
......@@ -602,6 +782,26 @@ slave-priority 100
#
# maxmemory-samples 5
# Starting from Redis 5, by default a replica will ignore its maxmemory setting
# (unless it is promoted to master after a failover or manually). It means
# that the eviction of keys will be just handled by the master, sending the
# DEL commands to the replica as keys evict in the master side.
#
# This behavior ensures that masters and replicas stay consistent, and is usually
# what you want, however if your replica is writable, or you want the replica
# to have a different memory setting, and you are sure all the writes performed
# to the replica are idempotent, then you may change this default (but be sure
# to understand what you are doing).
#
# Note that since the replica by default does not evict, it may end using more
# memory than the one set via maxmemory (there are certain buffers that may
# be larger on the replica, or data structures may sometimes take more memory
# and so forth). So make sure you monitor your replicas and make sure they
# have enough memory to never hit a real out-of-memory condition before the
# master hits the configured maxmemory setting.
#
# replica-ignore-maxmemory yes
############################# LAZY FREEING ####################################
# Redis has two primitives to delete keys. One is called DEL and is a blocking
......@@ -637,9 +837,9 @@ slave-priority 100
# or SORT with STORE option may delete existing keys. The SET command
# itself removes any old content of the specified key in order to replace
# it with the specified string.
# 4) During replication, when a slave performs a full resynchronization with
# 4) During replication, when a replica performs a full resynchronization with
# its master, the content of the whole database is removed in order to
# load the RDB file just transfered.
# load the RDB file just transferred.
#
# In all the above cases the default is to delete objects in a blocking way,
# like if DEL was called. However you can configure each case specifically
......@@ -649,7 +849,7 @@ slave-priority 100
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
slave-lazy-flush no
replica-lazy-flush no
############################## APPEND ONLY MODE ###############################
......@@ -799,13 +999,7 @@ aof-use-rdb-preamble yes
lua-time-limit 5000
################################ REDIS CLUSTER ###############################
#
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# WARNING EXPERIMENTAL: Redis Cluster is considered to be stable code, however
# in order to mark it as "mature" we need to wait for a non trivial percentage
# of users to deploy it in production.
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
#
# Normal Redis instances can't be part of a Redis Cluster; only nodes that are
# started as cluster nodes can. In order to start a Redis instance as a
# cluster node enable the cluster support uncommenting the following:
......@@ -826,42 +1020,42 @@ lua-time-limit 5000
#
# cluster-node-timeout 15000
# A slave of a failing master will avoid to start a failover if its data
# A replica of a failing master will avoid to start a failover if its data
# looks too old.
#
# There is no simple way for a slave to actually have an exact measure of
# There is no simple way for a replica to actually have an exact measure of
# its "data age", so the following two checks are performed:
#
# 1) If there are multiple slaves able to failover, they exchange messages
# in order to try to give an advantage to the slave with the best
# 1) If there are multiple replicas able to failover, they exchange messages
# in order to try to give an advantage to the replica with the best
# replication offset (more data from the master processed).
# Slaves will try to get their rank by offset, and apply to the start
# Replicas will try to get their rank by offset, and apply to the start
# of the failover a delay proportional to their rank.
#
# 2) Every single slave computes the time of the last interaction with
# 2) Every single replica computes the time of the last interaction with
# its master. This can be the last ping or command received (if the master
# is still in the "connected" state), or the time that elapsed since the
# disconnection with the master (if the replication link is currently down).
# If the last interaction is too old, the slave will not try to failover
# If the last interaction is too old, the replica will not try to failover
# at all.
#
# The point "2" can be tuned by user. Specifically a slave will not perform
# The point "2" can be tuned by user. Specifically a replica will not perform
# the failover if, since the last interaction with the master, the time
# elapsed is greater than:
#
# (node-timeout * slave-validity-factor) + repl-ping-slave-period
# (node-timeout * replica-validity-factor) + repl-ping-replica-period
#
# So for example if node-timeout is 30 seconds, and the slave-validity-factor
# is 10, and assuming a default repl-ping-slave-period of 10 seconds, the
# slave will not try to failover if it was not able to talk with the master
# So for example if node-timeout is 30 seconds, and the replica-validity-factor
# is 10, and assuming a default repl-ping-replica-period of 10 seconds, the
# replica will not try to failover if it was not able to talk with the master
# for longer than 310 seconds.
#
# A large slave-validity-factor may allow slaves with too old data to failover
# A large replica-validity-factor may allow replicas with too old data to failover
# a master, while a too small value may prevent the cluster from being able to
# elect a slave at all.
# elect a replica at all.
#
# For maximum availability, it is possible to set the slave-validity-factor
# to a value of 0, which means, that slaves will always try to failover the
# For maximum availability, it is possible to set the replica-validity-factor
# to a value of 0, which means, that replicas will always try to failover the
# master regardless of the last time they interacted with the master.
# (However they'll always try to apply a delay proportional to their
# offset rank).
......@@ -869,22 +1063,22 @@ lua-time-limit 5000
# Zero is the only value able to guarantee that when all the partitions heal
# the cluster will always be able to continue.
#
# cluster-slave-validity-factor 10
# cluster-replica-validity-factor 10
# Cluster slaves are able to migrate to orphaned masters, that are masters
# that are left without working slaves. This improves the cluster ability
# Cluster replicas are able to migrate to orphaned masters, that are masters
# that are left without working replicas. This improves the cluster ability
# to resist to failures as otherwise an orphaned master can't be failed over
# in case of failure if it has no working slaves.
# in case of failure if it has no working replicas.
#
# Slaves migrate to orphaned masters only if there are still at least a
# given number of other working slaves for their old master. This number
# is the "migration barrier". A migration barrier of 1 means that a slave
# will migrate only if there is at least 1 other working slave for its master
# and so forth. It usually reflects the number of slaves you want for every
# Replicas migrate to orphaned masters only if there are still at least a
# given number of other working replicas for their old master. This number
# is the "migration barrier". A migration barrier of 1 means that a replica
# will migrate only if there is at least 1 other working replica for its master
# and so forth. It usually reflects the number of replicas you want for every
# master in your cluster.
#
# Default is 1 (slaves migrate only if their masters remain with at least
# one slave). To disable migration just set it to a very large value.
# Default is 1 (replicas migrate only if their masters remain with at least
# one replica). To disable migration just set it to a very large value.
# A value of 0 can be set but is useful only for debugging and dangerous
# in production.
#
......@@ -903,7 +1097,7 @@ lua-time-limit 5000
#
# cluster-require-full-coverage yes
# This option, when set to yes, prevents slaves from trying to failover its
# This option, when set to yes, prevents replicas from trying to failover its
# master during master failures. However the master can still perform a
# manual failover, if forced to do so.
#
......@@ -911,7 +1105,7 @@ lua-time-limit 5000
# data center operations, where we want one side to never be promoted if not
# in the case of a total DC failure.
#
# cluster-slave-no-failover no
# cluster-replica-no-failover no
# In order to setup your cluster make sure to read the documentation
# available at http://redis.io web site.
......@@ -1040,6 +1234,61 @@ latency-monitor-threshold 0
# specify at least one of K or E, no events will be delivered.
notify-keyspace-events ""
############################### GOPHER SERVER #################################
# Redis contains an implementation of the Gopher protocol, as specified in
# the RFC 1436 (https://www.ietf.org/rfc/rfc1436.txt).
#
# The Gopher protocol was very popular in the late '90s. It is an alternative
# to the web, and the implementation both server and client side is so simple
# that the Redis server has just 100 lines of code in order to implement this
# support.
#
# What do you do with Gopher nowadays? Well Gopher never *really* died, and
# lately there is a movement in order for the Gopher more hierarchical content
# composed of just plain text documents to be resurrected. Some want a simpler
# internet, others believe that the mainstream internet became too much
# controlled, and it's cool to create an alternative space for people that
# want a bit of fresh air.
#
# Anyway for the 10nth birthday of the Redis, we gave it the Gopher protocol
# as a gift.
#
# --- HOW IT WORKS? ---
#
# The Redis Gopher support uses the inline protocol of Redis, and specifically
# two kind of inline requests that were anyway illegal: an empty request
# or any request that starts with "/" (there are no Redis commands starting
# with such a slash). Normal RESP2/RESP3 requests are completely out of the
# path of the Gopher protocol implementation and are served as usually as well.
#
# If you open a connection to Redis when Gopher is enabled and send it
# a string like "/foo", if there is a key named "/foo" it is served via the
# Gopher protocol.
#
# In order to create a real Gopher "hole" (the name of a Gopher site in Gopher
# talking), you likely need a script like the following:
#
# https://github.com/antirez/gopher2redis
#
# --- SECURITY WARNING ---
#
# If you plan to put Redis on the internet in a publicly accessible address
# to server Gopher pages MAKE SURE TO SET A PASSWORD to the instance.
# Once a password is set:
#
# 1. The Gopher server (when enabled, not by default) will kill serve
# content via Gopher.
# 2. However other commands cannot be called before the client will
# authenticate.
#
# So use the 'requirepass' option to protect your instance.
#
# To enable Gopher support uncomment the following line and set
# the option from no (the default) to yes.
#
# gopher-enabled no
############################### ADVANCED CONFIG ###############################
# Hashes are encoded using a memory efficient data structure when they have a
......@@ -1145,7 +1394,7 @@ activerehashing yes
# The limit can be set differently for the three different classes of clients:
#
# normal -> normal clients including MONITOR clients
# slave -> slave clients
# replica -> replica clients
# pubsub -> clients subscribed to at least one pubsub channel or pattern
#
# The syntax of every client-output-buffer-limit directive is the following:
......@@ -1166,12 +1415,12 @@ activerehashing yes
# asynchronous clients may create a scenario where data is requested faster
# than it can read.
#
# Instead there is a default limit for pubsub and slave clients, since
# subscribers and slaves receive data in a push fashion.
# Instead there is a default limit for pubsub and replica clients, since
# subscribers and replicas receive data in a push fashion.
#
# Both the hard or the soft limit can be disabled by setting them to zero.
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
# Client query buffers accumulate new commands. They are limited to a fixed
......@@ -1205,6 +1454,22 @@ client-output-buffer-limit pubsub 32mb 8mb 60
# 100 only in environments where very low latency is required.
hz 10
# Normally it is useful to have an HZ value which is proportional to the
# number of clients connected. This is useful in order, for instance, to
# avoid too many clients are processed for each background task invocation
# in order to avoid latency spikes.
#
# Since the default HZ value by default is conservatively set to 10, Redis
# offers, and enables by default, the ability to use an adaptive HZ value
# which will temporary raise when there are many connected clients.
#
# When dynamic HZ is enabled, the actual configured HZ will be used as
# as a baseline, but multiples of the configured HZ value will be actually
# used as needed once more clients are connected. In this way an idle
# instance will use very little CPU time while a busy instance will be
# more responsive.
dynamic-hz yes
# When a child rewrites the AOF file, if the following option is enabled
# the file will be fsync-ed every 32 MB of data generated. This is useful
# in order to commit the file to the disk more incrementally and avoid
......
......@@ -11,4 +11,4 @@ then
echo "You need tcl 8.5 or newer in order to run the Redis test"
exit 1
fi
$TCLSH tests/test_helper.tcl $*
$TCLSH tests/test_helper.tcl "${@}"
#!/bin/sh
TCL_VERSIONS="8.5 8.6"
TCLSH=""
for VERSION in $TCL_VERSIONS; do
TCL=`which tclsh$VERSION 2>/dev/null` && TCLSH=$TCL
done
if [ -z $TCLSH ]
then
echo "You need tcl 8.5 or newer in order to run the Redis test"
exit 1
fi
make -C tests/modules && \
$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/testrdb "${@}"
......@@ -20,6 +20,21 @@
# The port that this sentinel instance will run on
port 26379
# By default Redis Sentinel does not run as a daemon. Use 'yes' if you need it.
# Note that Redis will write a pid file in /var/run/redis-sentinel.pid when
# daemonized.
daemonize no
# When running daemonized, Redis Sentinel writes a pid file in
# /var/run/redis-sentinel.pid by default. You can specify a custom pid file
# location here.
pidfile /var/run/redis-sentinel.pid
# Specify the log file name. Also the empty string can be used to force
# Sentinel to log on the standard output. Note that if you use standard
# output for logging but daemonize, logs will be sent to /dev/null
logfile ""
# sentinel announce-ip <ip>
# sentinel announce-port <port>
#
......@@ -58,11 +73,11 @@ dir /tmp
# be elected by the majority of the known Sentinels in order to
# start a failover, so no failover can be performed in minority.
#
# Slaves are auto-discovered, so you don't need to specify slaves in
# Replicas are auto-discovered, so you don't need to specify replicas in
# any way. Sentinel itself will rewrite this configuration file adding
# the slaves using additional configuration options.
# the replicas using additional configuration options.
# Also note that the configuration file is rewritten when a
# slave is promoted to master.
# replica is promoted to master.
#
# Note: master name should not include special characters or spaces.
# The valid charset is A-z 0-9 and the three characters ".-_".
......@@ -70,11 +85,11 @@ sentinel monitor mymaster 127.0.0.1 6379 2
# sentinel auth-pass <master-name> <password>
#
# Set the password to use to authenticate with the master and slaves.
# Set the password to use to authenticate with the master and replicas.
# Useful if there is a password set in the Redis instances to monitor.
#
# Note that the master password is also used for slaves, so it is not
# possible to set a different password in masters and slaves instances
# Note that the master password is also used for replicas, so it is not
# possible to set a different password in masters and replicas instances
# if you want to be able to monitor these instances with Sentinel.
#
# However you can have Redis instances without the authentication enabled
......@@ -89,7 +104,7 @@ sentinel monitor mymaster 127.0.0.1 6379 2
# sentinel down-after-milliseconds <master-name> <milliseconds>
#
# Number of milliseconds the master (or any attached slave or sentinel) should
# Number of milliseconds the master (or any attached replica or sentinel) should
# be unreachable (as in, not acceptable reply to PING, continuously, for the
# specified period) in order to consider it in S_DOWN state (Subjectively
# Down).
......@@ -97,11 +112,11 @@ sentinel monitor mymaster 127.0.0.1 6379 2
# Default is 30 seconds.
sentinel down-after-milliseconds mymaster 30000
# sentinel parallel-syncs <master-name> <numslaves>
# sentinel parallel-syncs <master-name> <numreplicas>
#
# How many slaves we can reconfigure to point to the new slave simultaneously
# during the failover. Use a low number if you use the slaves to serve query
# to avoid that all the slaves will be unreachable at about the same
# How many replicas we can reconfigure to point to the new replica simultaneously
# during the failover. Use a low number if you use the replicas to serve query
# to avoid that all the replicas will be unreachable at about the same
# time while performing the synchronization with the master.
sentinel parallel-syncs mymaster 1
......@@ -113,18 +128,18 @@ sentinel parallel-syncs mymaster 1
# already tried against the same master by a given Sentinel, is two
# times the failover timeout.
#
# - The time needed for a slave replicating to a wrong master according
# - The time needed for a replica replicating to a wrong master according
# to a Sentinel current configuration, to be forced to replicate
# with the right master, is exactly the failover timeout (counting since
# the moment a Sentinel detected the misconfiguration).
#
# - The time needed to cancel a failover that is already in progress but
# did not produced any configuration change (SLAVEOF NO ONE yet not
# acknowledged by the promoted slave).
# acknowledged by the promoted replica).
#
# - The maximum time a failover in progress waits for all the slaves to be
# reconfigured as slaves of the new master. However even after this time
# the slaves will be reconfigured by the Sentinels anyway, but not with
# - The maximum time a failover in progress waits for all the replicas to be
# reconfigured as replicas of the new master. However even after this time
# the replicas will be reconfigured by the Sentinels anyway, but not with
# the exact parallel-syncs progression as specified.
#
# Default is 3 minutes.
......@@ -185,7 +200,7 @@ sentinel failover-timeout mymaster 180000
# <role> is either "leader" or "observer"
#
# The arguments from-ip, from-port, to-ip, to-port are used to communicate
# the old address of the master and the new address of the elected slave
# the old address of the master and the new address of the elected replica
# (now a master).
#
# This script should be resistant to multiple invocations.
......@@ -203,3 +218,27 @@ sentinel failover-timeout mymaster 180000
sentinel deny-scripts-reconfig yes
# REDIS COMMANDS RENAMING
#
# Sometimes the Redis server has certain commands, that are needed for Sentinel
# to work correctly, renamed to unguessable strings. This is often the case
# of CONFIG and SLAVEOF in the context of providers that provide Redis as
# a service, and don't want the customers to reconfigure the instances outside
# of the administration console.
#
# In such case it is possible to tell Sentinel to use different command names
# instead of the normal ones. For example if the master "mymaster", and the
# associated replicas, have "CONFIG" all renamed to "GUESSME", I could use:
#
# SENTINEL rename-command mymaster CONFIG GUESSME
#
# After such configuration is set, every time Sentinel would use CONFIG it will
# use GUESSME instead. Note that there is no actual need to respect the command
# case, so writing "config guessme" is the same in the example above.
#
# SENTINEL SET can also be used in order to perform this configuration at runtime.
#
# In order to set a command back to its original name (undo the renaming), it
# is possible to just rename a command to itsef:
#
# SENTINEL rename-command mymaster CONFIG CONFIG
......@@ -20,7 +20,12 @@ DEPENDENCY_TARGETS=hiredis linenoise lua
NODEPS:=clean distclean
# Default settings
STD=-std=c99 -pedantic -DREDIS_STATIC=''
STD=-std=c11 -pedantic -DREDIS_STATIC=''
ifneq (,$(findstring clang,$(CC)))
ifneq (,$(findstring FreeBSD,$(uname_S)))
STD+=-Wno-c11-extensions
endif
endif
WARN=-Wall -W -Wno-missing-field-initializers
OPT=$(OPTIMIZATION)
......@@ -41,6 +46,10 @@ endif
# To get ARM stack traces if Redis crashes we need a special C flag.
ifneq (,$(filter aarch64 armv,$(uname_M)))
CFLAGS+=-funwind-tables
else
ifneq (,$(findstring armv,$(uname_M)))
CFLAGS+=-funwind-tables
endif
endif
# Backwards compatibility for selecting an allocator
......@@ -93,10 +102,20 @@ else
ifeq ($(uname_S),OpenBSD)
# OpenBSD
FINAL_LIBS+= -lpthread
ifeq ($(USE_BACKTRACE),yes)
FINAL_CFLAGS+= -DUSE_BACKTRACE -I/usr/local/include
FINAL_LDFLAGS+= -L/usr/local/lib
FINAL_LIBS+= -lexecinfo
endif
else
ifeq ($(uname_S),FreeBSD)
# FreeBSD
FINAL_LIBS+= -lpthread
FINAL_LIBS+= -lpthread -lexecinfo
else
ifeq ($(uname_S),DragonFly)
# FreeBSD
FINAL_LIBS+= -lpthread -lexecinfo
else
# All the other OSes (notably Linux)
FINAL_LDFLAGS+= -rdynamic
......@@ -106,6 +125,7 @@ endif
endif
endif
endif
endif
# Include paths to dependencies
FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src
......@@ -144,11 +164,11 @@ endif
REDIS_SERVER_NAME=redis-server
REDIS_SENTINEL_NAME=redis-sentinel
REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o
REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o gopher.o tracking.o
REDIS_CLI_NAME=redis-cli
REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o
REDIS_BENCHMARK_NAME=redis-benchmark
REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o zmalloc.o redis-benchmark.o
REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o redis-benchmark.o
REDIS_CHECK_RDB_NAME=redis-check-rdb
REDIS_CHECK_AOF_NAME=redis-check-aof
......@@ -290,3 +310,6 @@ install: all
$(REDIS_INSTALL) $(REDIS_CHECK_RDB_NAME) $(INSTALL_BIN)
$(REDIS_INSTALL) $(REDIS_CHECK_AOF_NAME) $(INSTALL_BIN)
@ln -sf $(REDIS_SERVER_NAME) $(INSTALL_BIN)/$(REDIS_SENTINEL_NAME)
uninstall:
rm -f $(INSTALL_BIN)/{$(REDIS_SERVER_NAME),$(REDIS_BENCHMARK_NAME),$(REDIS_CLI_NAME),$(REDIS_CHECK_RDB_NAME),$(REDIS_CHECK_AOF_NAME),$(REDIS_SENTINEL_NAME)}
/*
* Copyright (c) 2018, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "server.h"
#include <fcntl.h>
/* =============================================================================
* Global state for ACLs
* ==========================================================================*/
rax *Users; /* Table mapping usernames to user structures. */
user *DefaultUser; /* Global reference to the default user.
Every new connection is associated to it, if no
AUTH or HELLO is used to authenticate with a
different user. */
list *UsersToLoad; /* This is a list of users found in the configuration file
that we'll need to load in the final stage of Redis
initialization, after all the modules are already
loaded. Every list element is a NULL terminated
array of SDS pointers: the first is the user name,
all the remaining pointers are ACL rules in the same
format as ACLSetUser(). */
struct ACLCategoryItem {
const char *name;
uint64_t flag;
} ACLCommandCategories[] = {
{"keyspace", CMD_CATEGORY_KEYSPACE},
{"read", CMD_CATEGORY_READ},
{"write", CMD_CATEGORY_WRITE},
{"set", CMD_CATEGORY_SET},
{"sortedset", CMD_CATEGORY_SORTEDSET},
{"list", CMD_CATEGORY_LIST},
{"hash", CMD_CATEGORY_HASH},
{"string", CMD_CATEGORY_STRING},
{"bitmap", CMD_CATEGORY_BITMAP},
{"hyperloglog", CMD_CATEGORY_HYPERLOGLOG},
{"geo", CMD_CATEGORY_GEO},
{"stream", CMD_CATEGORY_STREAM},
{"pubsub", CMD_CATEGORY_PUBSUB},
{"admin", CMD_CATEGORY_ADMIN},
{"fast", CMD_CATEGORY_FAST},
{"slow", CMD_CATEGORY_SLOW},
{"blocking", CMD_CATEGORY_BLOCKING},
{"dangerous", CMD_CATEGORY_DANGEROUS},
{"connection", CMD_CATEGORY_CONNECTION},
{"transaction", CMD_CATEGORY_TRANSACTION},
{"scripting", CMD_CATEGORY_SCRIPTING},
{NULL,0} /* Terminator. */
};
struct ACLUserFlag {
const char *name;
uint64_t flag;
} ACLUserFlags[] = {
{"on", USER_FLAG_ENABLED},
{"off", USER_FLAG_DISABLED},
{"allkeys", USER_FLAG_ALLKEYS},
{"allcommands", USER_FLAG_ALLCOMMANDS},
{"nopass", USER_FLAG_NOPASS},
{NULL,0} /* Terminator. */
};
void ACLResetSubcommandsForCommand(user *u, unsigned long id);
void ACLResetSubcommands(user *u);
void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub);
/* =============================================================================
* Helper functions for the rest of the ACL implementation
* ==========================================================================*/
/* Return zero if strings are the same, non-zero if they are not.
* The comparison is performed in a way that prevents an attacker to obtain
* information about the nature of the strings just monitoring the execution
* time of the function.
*
* Note that limiting the comparison length to strings up to 512 bytes we
* can avoid leaking any information about the password length and any
* possible branch misprediction related leak.
*/
int time_independent_strcmp(char *a, char *b) {
char bufa[CONFIG_AUTHPASS_MAX_LEN], bufb[CONFIG_AUTHPASS_MAX_LEN];
/* The above two strlen perform len(a) + len(b) operations where either
* a or b are fixed (our password) length, and the difference is only
* relative to the length of the user provided string, so no information
* leak is possible in the following two lines of code. */
unsigned int alen = strlen(a);
unsigned int blen = strlen(b);
unsigned int j;
int diff = 0;
/* We can't compare strings longer than our static buffers.
* Note that this will never pass the first test in practical circumstances
* so there is no info leak. */
if (alen > sizeof(bufa) || blen > sizeof(bufb)) return 1;
memset(bufa,0,sizeof(bufa)); /* Constant time. */
memset(bufb,0,sizeof(bufb)); /* Constant time. */
/* Again the time of the following two copies is proportional to
* len(a) + len(b) so no info is leaked. */
memcpy(bufa,a,alen);
memcpy(bufb,b,blen);
/* Always compare all the chars in the two buffers without
* conditional expressions. */
for (j = 0; j < sizeof(bufa); j++) {
diff |= (bufa[j] ^ bufb[j]);
}
/* Length must be equal as well. */
diff |= alen ^ blen;
return diff; /* If zero strings are the same. */
}
/* =============================================================================
* Low level ACL API
* ==========================================================================*/
/* Given the category name the command returns the corresponding flag, or
* zero if there is no match. */
uint64_t ACLGetCommandCategoryFlagByName(const char *name) {
for (int j = 0; ACLCommandCategories[j].flag != 0; j++) {
if (!strcasecmp(name,ACLCommandCategories[j].name)) {
return ACLCommandCategories[j].flag;
}
}
return 0; /* No match. */
}
/* Method for passwords/pattern comparison used for the user->passwords list
* so that we can search for items with listSearchKey(). */
int ACLListMatchSds(void *a, void *b) {
return sdscmp(a,b) == 0;
}
/* Method to free list elements from ACL users password/ptterns lists. */
void ACLListFreeSds(void *item) {
sdsfree(item);
}
/* Method to duplicate list elements from ACL users password/ptterns lists. */
void *ACLListDupSds(void *item) {
return sdsdup(item);
}
/* Create a new user with the specified name, store it in the list
* of users (the Users global radix tree), and returns a reference to
* the structure representing the user.
*
* If the user with such name already exists NULL is returned. */
user *ACLCreateUser(const char *name, size_t namelen) {
if (raxFind(Users,(unsigned char*)name,namelen) != raxNotFound) return NULL;
user *u = zmalloc(sizeof(*u));
u->name = sdsnewlen(name,namelen);
u->flags = USER_FLAG_DISABLED;
u->allowed_subcommands = NULL;
u->passwords = listCreate();
u->patterns = listCreate();
listSetMatchMethod(u->passwords,ACLListMatchSds);
listSetFreeMethod(u->passwords,ACLListFreeSds);
listSetDupMethod(u->passwords,ACLListDupSds);
listSetMatchMethod(u->patterns,ACLListMatchSds);
listSetFreeMethod(u->patterns,ACLListFreeSds);
listSetDupMethod(u->patterns,ACLListDupSds);
memset(u->allowed_commands,0,sizeof(u->allowed_commands));
raxInsert(Users,(unsigned char*)name,namelen,u,NULL);
return u;
}
/* This function should be called when we need an unlinked "fake" user
* we can use in order to validate ACL rules or for other similar reasons.
* The user will not get linked to the Users radix tree. The returned
* user should be released with ACLFreeUser() as usually. */
user *ACLCreateUnlinkedUser(void) {
char username[64];
for (int j = 0; ; j++) {
snprintf(username,sizeof(username),"__fakeuser:%d__",j);
user *fakeuser = ACLCreateUser(username,strlen(username));
if (fakeuser == NULL) continue;
int retval = raxRemove(Users,(unsigned char*) username,
strlen(username),NULL);
serverAssert(retval != 0);
return fakeuser;
}
}
/* Release the memory used by the user structure. Note that this function
* will not remove the user from the Users global radix tree. */
void ACLFreeUser(user *u) {
sdsfree(u->name);
listRelease(u->passwords);
listRelease(u->patterns);
ACLResetSubcommands(u);
zfree(u);
}
/* When a user is deleted we need to cycle the active
* connections in order to kill all the pending ones that
* are authenticated with such user. */
void ACLFreeUserAndKillClients(user *u) {
listIter li;
listNode *ln;
listRewind(server.clients,&li);
while ((ln = listNext(&li)) != NULL) {
client *c = listNodeValue(ln);
if (c->user == u) {
/* We'll free the conenction asynchronously, so
* in theory to set a different user is not needed.
* However if there are bugs in Redis, soon or later
* this may result in some security hole: it's much
* more defensive to set the default user and put
* it in non authenticated mode. */
c->user = DefaultUser;
c->authenticated = 0;
freeClientAsync(c);
}
}
ACLFreeUser(u);
}
/* Copy the user ACL rules from the source user 'src' to the destination
* user 'dst' so that at the end of the process they'll have exactly the
* same rules (but the names will continue to be the original ones). */
void ACLCopyUser(user *dst, user *src) {
listRelease(dst->passwords);
listRelease(dst->patterns);
dst->passwords = listDup(src->passwords);
dst->patterns = listDup(src->patterns);
memcpy(dst->allowed_commands,src->allowed_commands,
sizeof(dst->allowed_commands));
dst->flags = src->flags;
ACLResetSubcommands(dst);
/* Copy the allowed subcommands array of array of SDS strings. */
if (src->allowed_subcommands) {
for (int j = 0; j < USER_COMMAND_BITS_COUNT; j++) {
if (src->allowed_subcommands[j]) {
for (int i = 0; src->allowed_subcommands[j][i]; i++)
{
ACLAddAllowedSubcommand(dst, j,
src->allowed_subcommands[j][i]);
}
}
}
}
}
/* Free all the users registered in the radix tree 'users' and free the
* radix tree itself. */
void ACLFreeUsersSet(rax *users) {
raxFreeWithCallback(users,(void(*)(void*))ACLFreeUserAndKillClients);
}
/* Given a command ID, this function set by reference 'word' and 'bit'
* so that user->allowed_commands[word] will address the right word
* where the corresponding bit for the provided ID is stored, and
* so that user->allowed_commands[word]&bit will identify that specific
* bit. The function returns C_ERR in case the specified ID overflows
* the bitmap in the user representation. */
int ACLGetCommandBitCoordinates(uint64_t id, uint64_t *word, uint64_t *bit) {
if (id >= USER_COMMAND_BITS_COUNT) return C_ERR;
*word = id / sizeof(uint64_t) / 8;
*bit = 1ULL << (id % (sizeof(uint64_t) * 8));
return C_OK;
}
/* Check if the specified command bit is set for the specified user.
* The function returns 1 is the bit is set or 0 if it is not.
* Note that this function does not check the ALLCOMMANDS flag of the user
* but just the lowlevel bitmask.
*
* If the bit overflows the user internal representation, zero is returned
* in order to disallow the execution of the command in such edge case. */
int ACLGetUserCommandBit(user *u, unsigned long id) {
uint64_t word, bit;
if (ACLGetCommandBitCoordinates(id,&word,&bit) == C_ERR) return 0;
return (u->allowed_commands[word] & bit) != 0;
}
/* When +@all or allcommands is given, we set a reserved bit as well that we
* can later test, to see if the user has the right to execute "future commands",
* that is, commands loaded later via modules. */
int ACLUserCanExecuteFutureCommands(user *u) {
return ACLGetUserCommandBit(u,USER_COMMAND_BITS_COUNT-1);
}
/* Set the specified command bit for the specified user to 'value' (0 or 1).
* If the bit overflows the user internal representation, no operation
* is performed. As a side effect of calling this function with a value of
* zero, the user flag ALLCOMMANDS is cleared since it is no longer possible
* to skip the command bit explicit test. */
void ACLSetUserCommandBit(user *u, unsigned long id, int value) {
uint64_t word, bit;
if (value == 0) u->flags &= ~USER_FLAG_ALLCOMMANDS;
if (ACLGetCommandBitCoordinates(id,&word,&bit) == C_ERR) return;
if (value)
u->allowed_commands[word] |= bit;
else
u->allowed_commands[word] &= ~bit;
}
/* This is like ACLSetUserCommandBit(), but instead of setting the specified
* ID, it will check all the commands in the category specified as argument,
* and will set all the bits corresponding to such commands to the specified
* value. Since the category passed by the user may be non existing, the
* function returns C_ERR if the category was not found, or C_OK if it was
* found and the operation was performed. */
int ACLSetUserCommandBitsForCategory(user *u, const char *category, int value) {
uint64_t cflag = ACLGetCommandCategoryFlagByName(category);
if (!cflag) return C_ERR;
dictIterator *di = dictGetIterator(server.orig_commands);
dictEntry *de;
while ((de = dictNext(di)) != NULL) {
struct redisCommand *cmd = dictGetVal(de);
if (cmd->flags & CMD_MODULE) continue; /* Ignore modules commands. */
if (cmd->flags & cflag) {
ACLSetUserCommandBit(u,cmd->id,value);
ACLResetSubcommandsForCommand(u,cmd->id);
}
}
dictReleaseIterator(di);
return C_OK;
}
/* Return the number of commands allowed (on) and denied (off) for the user 'u'
* in the subset of commands flagged with the specified category name.
* If the category name is not valid, C_ERR is returned, otherwise C_OK is
* returned and on and off are populated by reference. */
int ACLCountCategoryBitsForUser(user *u, unsigned long *on, unsigned long *off,
const char *category)
{
uint64_t cflag = ACLGetCommandCategoryFlagByName(category);
if (!cflag) return C_ERR;
*on = *off = 0;
dictIterator *di = dictGetIterator(server.orig_commands);
dictEntry *de;
while ((de = dictNext(di)) != NULL) {
struct redisCommand *cmd = dictGetVal(de);
if (cmd->flags & cflag) {
if (ACLGetUserCommandBit(u,cmd->id))
(*on)++;
else
(*off)++;
}
}
dictReleaseIterator(di);
return C_OK;
}
/* This function returns an SDS string representing the specified user ACL
* rules related to command execution, in the same format you could set them
* back using ACL SETUSER. The function will return just the set of rules needed
* to recreate the user commands bitmap, without including other user flags such
* as on/off, passwords and so forth. The returned string always starts with
* the +@all or -@all rule, depending on the user bitmap, and is followed, if
* needed, by the other rules needed to narrow or extend what the user can do. */
sds ACLDescribeUserCommandRules(user *u) {
sds rules = sdsempty();
int additive; /* If true we start from -@all and add, otherwise if
false we start from +@all and remove. */
/* This code is based on a trick: as we generate the rules, we apply
* them to a fake user, so that as we go we still know what are the
* bit differences we should try to address by emitting more rules. */
user fu = {0};
user *fakeuser = &fu;
/* Here we want to understand if we should start with +@all and remove
* the commands corresponding to the bits that are not set in the user
* commands bitmap, or the contrary. Note that semantically the two are
* different. For instance starting with +@all and subtracting, the user
* will be able to execute future commands, while -@all and adding will just
* allow the user the run the selected commands and/or categories.
* How do we test for that? We use the trick of a reserved command ID bit
* that is set only by +@all (and its alias "allcommands"). */
if (ACLUserCanExecuteFutureCommands(u)) {
additive = 0;
rules = sdscat(rules,"+@all ");
ACLSetUser(fakeuser,"+@all",-1);
} else {
additive = 1;
rules = sdscat(rules,"-@all ");
ACLSetUser(fakeuser,"-@all",-1);
}
/* Try to add or subtract each category one after the other. Often a
* single category will not perfectly match the set of commands into
* it, so at the end we do a final pass adding/removing the single commands
* needed to make the bitmap exactly match. */
for (int j = 0; ACLCommandCategories[j].flag != 0; j++) {
unsigned long on, off;
ACLCountCategoryBitsForUser(u,&on,&off,ACLCommandCategories[j].name);
if ((additive && on > off) || (!additive && off > on)) {
sds op = sdsnewlen(additive ? "+@" : "-@", 2);
op = sdscat(op,ACLCommandCategories[j].name);
ACLSetUser(fakeuser,op,-1);
rules = sdscatsds(rules,op);
rules = sdscatlen(rules," ",1);
sdsfree(op);
}
}
/* Fix the final ACLs with single commands differences. */
dictIterator *di = dictGetIterator(server.orig_commands);
dictEntry *de;
while ((de = dictNext(di)) != NULL) {
struct redisCommand *cmd = dictGetVal(de);
int userbit = ACLGetUserCommandBit(u,cmd->id);
int fakebit = ACLGetUserCommandBit(fakeuser,cmd->id);
if (userbit != fakebit) {
rules = sdscatlen(rules, userbit ? "+" : "-", 1);
rules = sdscat(rules,cmd->name);
rules = sdscatlen(rules," ",1);
ACLSetUserCommandBit(fakeuser,cmd->id,userbit);
}
/* Emit the subcommands if there are any. */
if (userbit == 0 && u->allowed_subcommands &&
u->allowed_subcommands[cmd->id])
{
for (int j = 0; u->allowed_subcommands[cmd->id][j]; j++) {
rules = sdscatlen(rules,"+",1);
rules = sdscat(rules,cmd->name);
rules = sdscatlen(rules,"|",1);
rules = sdscatsds(rules,u->allowed_subcommands[cmd->id][j]);
rules = sdscatlen(rules," ",1);
}
}
}
dictReleaseIterator(di);
/* Trim the final useless space. */
sdsrange(rules,0,-2);
/* This is technically not needed, but we want to verify that now the
* predicted bitmap is exactly the same as the user bitmap, and abort
* otherwise, because aborting is better than a security risk in this
* code path. */
if (memcmp(fakeuser->allowed_commands,
u->allowed_commands,
sizeof(u->allowed_commands)) != 0)
{
serverLog(LL_WARNING,
"CRITICAL ERROR: User ACLs don't match final bitmap: '%s'",
rules);
serverPanic("No bitmap match in ACLDescribeUserCommandRules()");
}
return rules;
}
/* This is similar to ACLDescribeUserCommandRules(), however instead of
* describing just the user command rules, everything is described: user
* flags, keys, passwords and finally the command rules obtained via
* the ACLDescribeUserCommandRules() function. This is the function we call
* when we want to rewrite the configuration files describing ACLs and
* in order to show users with ACL LIST. */
sds ACLDescribeUser(user *u) {
sds res = sdsempty();
/* Flags. */
for (int j = 0; ACLUserFlags[j].flag; j++) {
/* Skip the allcommands and allkeys flags because they'll be emitted
* later as ~* and +@all. */
if (ACLUserFlags[j].flag == USER_FLAG_ALLKEYS ||
ACLUserFlags[j].flag == USER_FLAG_ALLCOMMANDS) continue;
if (u->flags & ACLUserFlags[j].flag) {
res = sdscat(res,ACLUserFlags[j].name);
res = sdscatlen(res," ",1);
}
}
/* Passwords. */
listIter li;
listNode *ln;
listRewind(u->passwords,&li);
while((ln = listNext(&li))) {
sds thispass = listNodeValue(ln);
res = sdscatlen(res,">",1);
res = sdscatsds(res,thispass);
res = sdscatlen(res," ",1);
}
/* Key patterns. */
if (u->flags & USER_FLAG_ALLKEYS) {
res = sdscatlen(res,"~* ",3);
} else {
listRewind(u->patterns,&li);
while((ln = listNext(&li))) {
sds thispat = listNodeValue(ln);
res = sdscatlen(res,"~",1);
res = sdscatsds(res,thispat);
res = sdscatlen(res," ",1);
}
}
/* Command rules. */
sds rules = ACLDescribeUserCommandRules(u);
res = sdscatsds(res,rules);
sdsfree(rules);
return res;
}
/* Get a command from the original command table, that is not affected
* by the command renaming operations: we base all the ACL work from that
* table, so that ACLs are valid regardless of command renaming. */
struct redisCommand *ACLLookupCommand(const char *name) {
struct redisCommand *cmd;
sds sdsname = sdsnew(name);
cmd = dictFetchValue(server.orig_commands, sdsname);
sdsfree(sdsname);
return cmd;
}
/* Flush the array of allowed subcommands for the specified user
* and command ID. */
void ACLResetSubcommandsForCommand(user *u, unsigned long id) {
if (u->allowed_subcommands && u->allowed_subcommands[id]) {
for (int i = 0; u->allowed_subcommands[id][i]; i++)
sdsfree(u->allowed_subcommands[id][i]);
zfree(u->allowed_subcommands[id]);
u->allowed_subcommands[id] = NULL;
}
}
/* Flush the entire table of subcommands. This is useful on +@all, -@all
* or similar to return back to the minimal memory usage (and checks to do)
* for the user. */
void ACLResetSubcommands(user *u) {
if (u->allowed_subcommands == NULL) return;
for (int j = 0; j < USER_COMMAND_BITS_COUNT; j++) {
if (u->allowed_subcommands[j]) {
for (int i = 0; u->allowed_subcommands[j][i]; i++)
sdsfree(u->allowed_subcommands[j][i]);
zfree(u->allowed_subcommands[j]);
}
}
zfree(u->allowed_subcommands);
u->allowed_subcommands = NULL;
}
/* Add a subcommand to the list of subcommands for the user 'u' and
* the command id specified. */
void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) {
/* If this is the first subcommand to be configured for
* this user, we have to allocate the subcommands array. */
if (u->allowed_subcommands == NULL) {
u->allowed_subcommands = zcalloc(USER_COMMAND_BITS_COUNT *
sizeof(sds*));
}
/* We also need to enlarge the allocation pointing to the
* null terminated SDS array, to make space for this one.
* To start check the current size, and while we are here
* make sure the subcommand is not already specified inside. */
long items = 0;
if (u->allowed_subcommands[id]) {
while(u->allowed_subcommands[id][items]) {
/* If it's already here do not add it again. */
if (!strcasecmp(u->allowed_subcommands[id][items],sub)) return;
items++;
}
}
/* Now we can make space for the new item (and the null term). */
items += 2;
u->allowed_subcommands[id] = zrealloc(u->allowed_subcommands[id],
sizeof(sds)*items);
u->allowed_subcommands[id][items-2] = sdsnew(sub);
u->allowed_subcommands[id][items-1] = NULL;
}
/* Set user properties according to the string "op". The following
* is a description of what different strings will do:
*
* on Enable the user: it is possible to authenticate as this user.
* off Disable the user: it's no longer possible to authenticate
* with this user, however the already authenticated connections
* will still work.
* +<command> Allow the execution of that command
* -<command> Disallow the execution of that command
* +@<category> Allow the execution of all the commands in such category
* with valid categories are like @admin, @set, @sortedset, ...
* and so forth, see the full list in the server.c file where
* the Redis command table is described and defined.
* The special category @all means all the commands, but currently
* present in the server, and that will be loaded in the future
* via modules.
* +<command>|subcommand Allow a specific subcommand of an otherwise
* disabled command. Note that this form is not
* allowed as negative like -DEBUG|SEGFAULT, but
* only additive starting with "+".
* allcommands Alias for +@all. Note that it implies the ability to execute
* all the future commands loaded via the modules system.
* nocommands Alias for -@all.
* ~<pattern> Add a pattern of keys that can be mentioned as part of
* commands. For instance ~* allows all the keys. The pattern
* is a glob-style pattern like the one of KEYS.
* It is possible to specify multiple patterns.
* allkeys Alias for ~*
* resetkeys Flush the list of allowed keys patterns.
* ><password> Add this password to the list of valid password for the user.
* For example >mypass will add "mypass" to the list.
* This directive clears the "nopass" flag (see later).
* <<password> Remove this password from the list of valid passwords.
* nopass All the set passwords of the user are removed, and the user
* is flagged as requiring no password: it means that every
* password will work against this user. If this directive is
* used for the default user, every new connection will be
* immediately authenticated with the default user without
* any explicit AUTH command required. Note that the "resetpass"
* directive will clear this condition.
* resetpass Flush the list of allowed passwords. Moreover removes the
* "nopass" status. After "resetpass" the user has no associated
* passwords and there is no way to authenticate without adding
* some password (or setting it as "nopass" later).
* reset Performs the following actions: resetpass, resetkeys, off,
* -@all. The user returns to the same state it has immediately
* after its creation.
*
* The 'op' string must be null terminated. The 'oplen' argument should
* specify the length of the 'op' string in case the caller requires to pass
* binary data (for instance the >password form may use a binary password).
* Otherwise the field can be set to -1 and the function will use strlen()
* to determine the length.
*
* The function returns C_OK if the action to perform was understood because
* the 'op' string made sense. Otherwise C_ERR is returned if the operation
* is unknown or has some syntax error.
*
* When an error is returned, errno is set to the following values:
*
* EINVAL: The specified opcode is not understood.
* ENOENT: The command name or command category provided with + or - is not
* known.
* EBUSY: The subcommand you want to add is about a command that is currently
* fully added.
* EEXIST: You are adding a key pattern after "*" was already added. This is
* almost surely an error on the user side.
* ENODEV: The password you are trying to remove from the user does not exist.
*/
int ACLSetUser(user *u, const char *op, ssize_t oplen) {
if (oplen == -1) oplen = strlen(op);
if (!strcasecmp(op,"on")) {
u->flags |= USER_FLAG_ENABLED;
u->flags &= ~USER_FLAG_DISABLED;
} else if (!strcasecmp(op,"off")) {
u->flags |= USER_FLAG_DISABLED;
u->flags &= ~USER_FLAG_ENABLED;
} else if (!strcasecmp(op,"allkeys") ||
!strcasecmp(op,"~*"))
{
u->flags |= USER_FLAG_ALLKEYS;
listEmpty(u->patterns);
} else if (!strcasecmp(op,"resetkeys")) {
u->flags &= ~USER_FLAG_ALLKEYS;
listEmpty(u->patterns);
} else if (!strcasecmp(op,"allcommands") ||
!strcasecmp(op,"+@all"))
{
memset(u->allowed_commands,255,sizeof(u->allowed_commands));
u->flags |= USER_FLAG_ALLCOMMANDS;
ACLResetSubcommands(u);
} else if (!strcasecmp(op,"nocommands") ||
!strcasecmp(op,"-@all"))
{
memset(u->allowed_commands,0,sizeof(u->allowed_commands));
u->flags &= ~USER_FLAG_ALLCOMMANDS;
ACLResetSubcommands(u);
} else if (!strcasecmp(op,"nopass")) {
u->flags |= USER_FLAG_NOPASS;
listEmpty(u->passwords);
} else if (!strcasecmp(op,"resetpass")) {
u->flags &= ~USER_FLAG_NOPASS;
listEmpty(u->passwords);
} else if (op[0] == '>') {
sds newpass = sdsnewlen(op+1,oplen-1);
listNode *ln = listSearchKey(u->passwords,newpass);
/* Avoid re-adding the same password multiple times. */
if (ln == NULL) listAddNodeTail(u->passwords,newpass);
u->flags &= ~USER_FLAG_NOPASS;
} else if (op[0] == '<') {
sds delpass = sdsnewlen(op+1,oplen-1);
listNode *ln = listSearchKey(u->passwords,delpass);
sdsfree(delpass);
if (ln) {
listDelNode(u->passwords,ln);
} else {
errno = ENODEV;
return C_ERR;
}
} else if (op[0] == '~') {
if (u->flags & USER_FLAG_ALLKEYS) {
errno = EEXIST;
return C_ERR;
}
sds newpat = sdsnewlen(op+1,oplen-1);
listNode *ln = listSearchKey(u->patterns,newpat);
/* Avoid re-adding the same pattern multiple times. */
if (ln == NULL) listAddNodeTail(u->patterns,newpat);
u->flags &= ~USER_FLAG_ALLKEYS;
} else if (op[0] == '+' && op[1] != '@') {
if (strchr(op,'|') == NULL) {
if (ACLLookupCommand(op+1) == NULL) {
errno = ENOENT;
return C_ERR;
}
unsigned long id = ACLGetCommandID(op+1);
ACLSetUserCommandBit(u,id,1);
ACLResetSubcommandsForCommand(u,id);
} else {
/* Split the command and subcommand parts. */
char *copy = zstrdup(op+1);
char *sub = strchr(copy,'|');
sub[0] = '\0';
sub++;
/* Check if the command exists. We can't check the
* subcommand to see if it is valid. */
if (ACLLookupCommand(copy) == NULL) {
zfree(copy);
errno = ENOENT;
return C_ERR;
}
unsigned long id = ACLGetCommandID(copy);
/* The subcommand cannot be empty, so things like DEBUG|
* are syntax errors of course. */
if (strlen(sub) == 0) {
zfree(copy);
errno = EINVAL;
return C_ERR;
}
/* The command should not be set right now in the command
* bitmap, because adding a subcommand of a fully added
* command is probably an error on the user side. */
if (ACLGetUserCommandBit(u,id) == 1) {
zfree(copy);
errno = EBUSY;
return C_ERR;
}
/* Add the subcommand to the list of valid ones. */
ACLAddAllowedSubcommand(u,id,sub);
/* We have to clear the command bit so that we force the
* subcommand check. */
ACLSetUserCommandBit(u,id,0);
zfree(copy);
}
} else if (op[0] == '-' && op[1] != '@') {
if (ACLLookupCommand(op+1) == NULL) {
errno = ENOENT;
return C_ERR;
}
unsigned long id = ACLGetCommandID(op+1);
ACLSetUserCommandBit(u,id,0);
ACLResetSubcommandsForCommand(u,id);
} else if ((op[0] == '+' || op[0] == '-') && op[1] == '@') {
int bitval = op[0] == '+' ? 1 : 0;
if (ACLSetUserCommandBitsForCategory(u,op+2,bitval) == C_ERR) {
errno = ENOENT;
return C_ERR;
}
} else if (!strcasecmp(op,"reset")) {
serverAssert(ACLSetUser(u,"resetpass",-1) == C_OK);
serverAssert(ACLSetUser(u,"resetkeys",-1) == C_OK);
serverAssert(ACLSetUser(u,"off",-1) == C_OK);
serverAssert(ACLSetUser(u,"-@all",-1) == C_OK);
} else {
errno = EINVAL;
return C_ERR;
}
return C_OK;
}
/* Return a description of the error that occurred in ACLSetUser() according to
* the errno value set by the function on error. */
char *ACLSetUserStringError(void) {
char *errmsg = "Wrong format";
if (errno == ENOENT)
errmsg = "Unknown command or category name in ACL";
else if (errno == EINVAL)
errmsg = "Syntax error";
else if (errno == EBUSY)
errmsg = "Adding a subcommand of a command already fully "
"added is not allowed. Remove the command to start. "
"Example: -DEBUG +DEBUG|DIGEST";
else if (errno == EEXIST)
errmsg = "Adding a pattern after the * pattern (or the "
"'allkeys' flag) is not valid and does not have any "
"effect. Try 'resetkeys' to start with an empty "
"list of patterns";
else if (errno == ENODEV)
errmsg = "The password you are trying to remove from the user does "
"not exist";
return errmsg;
}
/* Return the first password of the default user or NULL.
* This function is needed for backward compatibility with the old
* directive "requirepass" when Redis supported a single global
* password. */
sds ACLDefaultUserFirstPassword(void) {
if (listLength(DefaultUser->passwords) == 0) return NULL;
listNode *first = listFirst(DefaultUser->passwords);
return listNodeValue(first);
}
/* Initialize the default user, that will always exist for all the process
* lifetime. */
void ACLInitDefaultUser(void) {
DefaultUser = ACLCreateUser("default",7);
ACLSetUser(DefaultUser,"+@all",-1);
ACLSetUser(DefaultUser,"~*",-1);
ACLSetUser(DefaultUser,"on",-1);
ACLSetUser(DefaultUser,"nopass",-1);
}
/* Initialization of the ACL subsystem. */
void ACLInit(void) {
Users = raxNew();
UsersToLoad = listCreate();
ACLInitDefaultUser();
}
/* Check the username and password pair and return C_OK if they are valid,
* otherwise C_ERR is returned and errno is set to:
*
* EINVAL: if the username-password do not match.
* ENONENT: if the specified user does not exist at all.
*/
int ACLCheckUserCredentials(robj *username, robj *password) {
user *u = ACLGetUserByName(username->ptr,sdslen(username->ptr));
if (u == NULL) {
errno = ENOENT;
return C_ERR;
}
/* Disabled users can't login. */
if (u->flags & USER_FLAG_DISABLED) {
errno = EINVAL;
return C_ERR;
}
/* If the user is configured to don't require any password, we
* are already fine here. */
if (u->flags & USER_FLAG_NOPASS) return C_OK;
/* Check all the user passwords for at least one to match. */
listIter li;
listNode *ln;
listRewind(u->passwords,&li);
while((ln = listNext(&li))) {
sds thispass = listNodeValue(ln);
if (!time_independent_strcmp(password->ptr, thispass))
return C_OK;
}
/* If we reached this point, no password matched. */
errno = EINVAL;
return C_ERR;
}
/* This is like ACLCheckUserCredentials(), however if the user/pass
* are correct, the connection is put in authenticated state and the
* connection user reference is populated.
*
* The return value is C_OK or C_ERR with the same meaning as
* ACLCheckUserCredentials(). */
int ACLAuthenticateUser(client *c, robj *username, robj *password) {
if (ACLCheckUserCredentials(username,password) == C_OK) {
c->authenticated = 1;
c->user = ACLGetUserByName(username->ptr,sdslen(username->ptr));
return C_OK;
} else {
return C_ERR;
}
}
/* For ACL purposes, every user has a bitmap with the commands that such
* user is allowed to execute. In order to populate the bitmap, every command
* should have an assigned ID (that is used to index the bitmap). This function
* creates such an ID: it uses sequential IDs, reusing the same ID for the same
* command name, so that a command retains the same ID in case of modules that
* are unloaded and later reloaded. */
unsigned long ACLGetCommandID(const char *cmdname) {
static rax *map = NULL;
static unsigned long nextid = 0;
sds lowername = sdsnew(cmdname);
sdstolower(lowername);
if (map == NULL) map = raxNew();
void *id = raxFind(map,(unsigned char*)lowername,sdslen(lowername));
if (id != raxNotFound) {
sdsfree(lowername);
return (unsigned long)id;
}
raxInsert(map,(unsigned char*)lowername,strlen(lowername),
(void*)nextid,NULL);
sdsfree(lowername);
unsigned long thisid = nextid;
nextid++;
/* We never assign the last bit in the user commands bitmap structure,
* this way we can later check if this bit is set, understanding if the
* current ACL for the user was created starting with a +@all to add all
* the possible commands and just subtracting other single commands or
* categories, or if, instead, the ACL was created just adding commands
* and command categories from scratch, not allowing future commands by
* default (loaded via modules). This is useful when rewriting the ACLs
* with ACL SAVE. */
if (nextid == USER_COMMAND_BITS_COUNT-1) nextid++;
return thisid;
}
/* Return an username by its name, or NULL if the user does not exist. */
user *ACLGetUserByName(const char *name, size_t namelen) {
void *myuser = raxFind(Users,(unsigned char*)name,namelen);
if (myuser == raxNotFound) return NULL;
return myuser;
}
/* Check if the command is ready to be executed in the client 'c', already
* referenced by c->cmd, and can be executed by this client according to the
* ACLs associated to the client user c->user.
*
* If the user can execute the command ACL_OK is returned, otherwise
* ACL_DENIED_CMD or ACL_DENIED_KEY is returned: the first in case the
* command cannot be executed because the user is not allowed to run such
* command, the second if the command is denied because the user is trying
* to access keys that are not among the specified patterns. */
int ACLCheckCommandPerm(client *c) {
user *u = c->user;
uint64_t id = c->cmd->id;
/* If there is no associated user, the connection can run anything. */
if (u == NULL) return ACL_OK;
/* Check if the user can execute this command. */
if (!(u->flags & USER_FLAG_ALLCOMMANDS) &&
c->cmd->proc != authCommand)
{
/* If the bit is not set we have to check further, in case the
* command is allowed just with that specific subcommand. */
if (ACLGetUserCommandBit(u,id) == 0) {
/* Check if the subcommand matches. */
if (c->argc < 2 ||
u->allowed_subcommands == NULL ||
u->allowed_subcommands[id] == NULL)
{
return ACL_DENIED_CMD;
}
long subid = 0;
while (1) {
if (u->allowed_subcommands[id][subid] == NULL)
return ACL_DENIED_CMD;
if (!strcasecmp(c->argv[1]->ptr,
u->allowed_subcommands[id][subid]))
break; /* Subcommand match found. Stop here. */
subid++;
}
}
}
/* Check if the user can execute commands explicitly touching the keys
* mentioned in the command arguments. */
if (!(c->user->flags & USER_FLAG_ALLKEYS) &&
(c->cmd->getkeys_proc || c->cmd->firstkey))
{
int numkeys;
int *keyidx = getKeysFromCommand(c->cmd,c->argv,c->argc,&numkeys);
for (int j = 0; j < numkeys; j++) {
listIter li;
listNode *ln;
listRewind(u->patterns,&li);
/* Test this key against every pattern. */
int match = 0;
while((ln = listNext(&li))) {
sds pattern = listNodeValue(ln);
size_t plen = sdslen(pattern);
int idx = keyidx[j];
if (stringmatchlen(pattern,plen,c->argv[idx]->ptr,
sdslen(c->argv[idx]->ptr),0))
{
match = 1;
break;
}
}
if (!match) {
getKeysFreeResult(keyidx);
return ACL_DENIED_KEY;
}
}
getKeysFreeResult(keyidx);
}
/* If we survived all the above checks, the user can execute the
* command. */
return ACL_OK;
}
/* =============================================================================
* ACL loading / saving functions
* ==========================================================================*/
/* Given an argument vector describing a user in the form:
*
* user <username> ... ACL rules and flags ...
*
* this function validates, and if the syntax is valid, appends
* the user definition to a list for later loading.
*
* The rules are tested for validity and if there obvious syntax errors
* the function returns C_ERR and does nothing, otherwise C_OK is returned
* and the user is appended to the list.
*
* Note that this function cannot stop in case of commands that are not found
* and, in that case, the error will be emitted later, because certain
* commands may be defined later once modules are loaded.
*
* When an error is detected and C_ERR is returned, the function populates
* by reference (if not set to NULL) the argc_err argument with the index
* of the argv vector that caused the error. */
int ACLAppendUserForLoading(sds *argv, int argc, int *argc_err) {
if (argc < 2 || strcasecmp(argv[0],"user")) {
if (argc_err) *argc_err = 0;
return C_ERR;
}
/* Try to apply the user rules in a fake user to see if they
* are actually valid. */
user *fakeuser = ACLCreateUnlinkedUser();
for (int j = 2; j < argc; j++) {
if (ACLSetUser(fakeuser,argv[j],sdslen(argv[j])) == C_ERR) {
if (errno != ENOENT) {
ACLFreeUser(fakeuser);
if (argc_err) *argc_err = j;
return C_ERR;
}
}
}
/* Rules look valid, let's append the user to the list. */
sds *copy = zmalloc(sizeof(sds)*argc);
for (int j = 1; j < argc; j++) copy[j-1] = sdsdup(argv[j]);
copy[argc-1] = NULL;
listAddNodeTail(UsersToLoad,copy);
ACLFreeUser(fakeuser);
return C_OK;
}
/* This function will load the configured users appended to the server
* configuration via ACLAppendUserForLoading(). On loading errors it will
* log an error and return C_ERR, otherwise C_OK will be returned. */
int ACLLoadConfiguredUsers(void) {
listIter li;
listNode *ln;
listRewind(UsersToLoad,&li);
while ((ln = listNext(&li)) != NULL) {
sds *aclrules = listNodeValue(ln);
sds username = aclrules[0];
user *u = ACLCreateUser(username,sdslen(username));
if (!u) {
u = ACLGetUserByName(username,sdslen(username));
serverAssert(u != NULL);
ACLSetUser(u,"reset",-1);
}
/* Load every rule defined for this user. */
for (int j = 1; aclrules[j]; j++) {
if (ACLSetUser(u,aclrules[j],sdslen(aclrules[j])) != C_OK) {
char *errmsg = ACLSetUserStringError();
serverLog(LL_WARNING,"Error loading ACL rule '%s' for "
"the user named '%s': %s",
aclrules[j],aclrules[0],errmsg);
return C_ERR;
}
}
/* Having a disabled user in the configuration may be an error,
* warn about it without returning any error to the caller. */
if (u->flags & USER_FLAG_DISABLED) {
serverLog(LL_NOTICE, "The user '%s' is disabled (there is no "
"'on' modifier in the user description). Make "
"sure this is not a configuration error.",
aclrules[0]);
}
}
return C_OK;
}
/* This function loads the ACL from the specified filename: every line
* is validated and should be either empty or in the format used to specify
* users in the redis.conf configuration or in the ACL file, that is:
*
* user <username> ... rules ...
*
* Note that this function considers comments starting with '#' as errors
* because the ACL file is meant to be rewritten, and comments would be
* lost after the rewrite. Yet empty lines are allowed to avoid being too
* strict.
*
* One important part of implementing ACL LOAD, that uses this function, is
* to avoid ending with broken rules if the ACL file is invalid for some
* reason, so the function will attempt to validate the rules before loading
* each user. For every line that will be found broken the function will
* collect an error message.
*
* IMPORTANT: If there is at least a single error, nothing will be loaded
* and the rules will remain exactly as they were.
*
* At the end of the process, if no errors were found in the whole file then
* NULL is returned. Otherwise an SDS string describing in a single line
* a description of all the issues found is returned. */
sds ACLLoadFromFile(const char *filename) {
FILE *fp;
char buf[1024];
/* Open the ACL file. */
if ((fp = fopen(filename,"r")) == NULL) {
sds errors = sdscatprintf(sdsempty(),
"Error loading ACLs, opening file '%s': %s",
filename, strerror(errno));
return errors;
}
/* Load the whole file as a single string in memory. */
sds acls = sdsempty();
while(fgets(buf,sizeof(buf),fp) != NULL)
acls = sdscat(acls,buf);
fclose(fp);
/* Split the file into lines and attempt to load each line. */
int totlines;
sds *lines, errors = sdsempty();
lines = sdssplitlen(acls,strlen(acls),"\n",1,&totlines);
sdsfree(acls);
/* We need a fake user to validate the rules before making changes
* to the real user mentioned in the ACL line. */
user *fakeuser = ACLCreateUnlinkedUser();
/* We do all the loading in a fresh instance of the Users radix tree,
* so if there are errors loading the ACL file we can rollback to the
* old version. */
rax *old_users = Users;
user *old_default_user = DefaultUser;
Users = raxNew();
ACLInitDefaultUser();
/* Load each line of the file. */
for (int i = 0; i < totlines; i++) {
sds *argv;
int argc;
int linenum = i+1;
lines[i] = sdstrim(lines[i]," \t\r\n");
/* Skip blank lines */
if (lines[i][0] == '\0') continue;
/* Split into arguments */
argv = sdssplitargs(lines[i],&argc);
if (argv == NULL) {
errors = sdscatprintf(errors,
"%s:%d: unbalanced quotes in acl line. ",
server.acl_filename, linenum);
continue;
}
/* Skip this line if the resulting command vector is empty. */
if (argc == 0) {
sdsfreesplitres(argv,argc);
continue;
}
/* The line should start with the "user" keyword. */
if (strcmp(argv[0],"user") || argc < 2) {
errors = sdscatprintf(errors,
"%s:%d should start with user keyword followed "
"by the username. ", server.acl_filename,
linenum);
sdsfreesplitres(argv,argc);
continue;
}
/* Try to process the line using the fake user to validate iif
* the rules are able to apply cleanly. */
ACLSetUser(fakeuser,"reset",-1);
int j;
for (j = 2; j < argc; j++) {
if (ACLSetUser(fakeuser,argv[j],sdslen(argv[j])) != C_OK) {
char *errmsg = ACLSetUserStringError();
errors = sdscatprintf(errors,
"%s:%d: %s. ",
server.acl_filename, linenum, errmsg);
continue;
}
}
/* Apply the rule to the new users set only if so far there
* are no errors, otherwise it's useless since we are going
* to discard the new users set anyway. */
if (sdslen(errors) != 0) {
sdsfreesplitres(argv,argc);
continue;
}
/* We can finally lookup the user and apply the rule. If the
* user already exists we always reset it to start. */
user *u = ACLCreateUser(argv[1],sdslen(argv[1]));
if (!u) {
u = ACLGetUserByName(argv[1],sdslen(argv[1]));
serverAssert(u != NULL);
ACLSetUser(u,"reset",-1);
}
/* Note that the same rules already applied to the fake user, so
* we just assert that everything goes well: it should. */
for (j = 2; j < argc; j++)
serverAssert(ACLSetUser(u,argv[j],sdslen(argv[j])) == C_OK);
sdsfreesplitres(argv,argc);
}
ACLFreeUser(fakeuser);
sdsfreesplitres(lines,totlines);
DefaultUser = old_default_user; /* This pointer must never change. */
/* Check if we found errors and react accordingly. */
if (sdslen(errors) == 0) {
/* The default user pointer is referenced in different places: instead
* of replacing such occurrences it is much simpler to copy the new
* default user configuration in the old one. */
user *new = ACLGetUserByName("default",7);
serverAssert(new != NULL);
ACLCopyUser(DefaultUser,new);
ACLFreeUser(new);
raxInsert(Users,(unsigned char*)"default",7,DefaultUser,NULL);
raxRemove(old_users,(unsigned char*)"default",7,NULL);
ACLFreeUsersSet(old_users);
sdsfree(errors);
return NULL;
} else {
ACLFreeUsersSet(Users);
Users = old_users;
errors = sdscat(errors,"WARNING: ACL errors detected, no change to the previously active ACL rules was performed");
return errors;
}
}
/* Generate a copy of the ACLs currently in memory in the specified filename.
* Returns C_OK on success or C_ERR if there was an error during the I/O.
* When C_ERR is returned a log is produced with hints about the issue. */
int ACLSaveToFile(const char *filename) {
sds acl = sdsempty();
int fd = -1;
sds tmpfilename = NULL;
int retval = C_ERR;
/* Let's generate an SDS string containing the new version of the
* ACL file. */
raxIterator ri;
raxStart(&ri,Users);
raxSeek(&ri,"^",NULL,0);
while(raxNext(&ri)) {
user *u = ri.data;
/* Return information in the configuration file format. */
sds user = sdsnew("user ");
user = sdscatsds(user,u->name);
user = sdscatlen(user," ",1);
sds descr = ACLDescribeUser(u);
user = sdscatsds(user,descr);
sdsfree(descr);
acl = sdscatsds(acl,user);
acl = sdscatlen(acl,"\n",1);
sdsfree(user);
}
raxStop(&ri);
/* Create a temp file with the new content. */
tmpfilename = sdsnew(filename);
tmpfilename = sdscatfmt(tmpfilename,".tmp-%i-%I",
(int)getpid(),(int)mstime());
if ((fd = open(tmpfilename,O_WRONLY|O_CREAT,0644)) == -1) {
serverLog(LL_WARNING,"Opening temp ACL file for ACL SAVE: %s",
strerror(errno));
goto cleanup;
}
/* Write it. */
if (write(fd,acl,sdslen(acl)) != (ssize_t)sdslen(acl)) {
serverLog(LL_WARNING,"Writing ACL file for ACL SAVE: %s",
strerror(errno));
goto cleanup;
}
close(fd); fd = -1;
/* Let's replace the new file with the old one. */
if (rename(tmpfilename,filename) == -1) {
serverLog(LL_WARNING,"Renaming ACL file for ACL SAVE: %s",
strerror(errno));
goto cleanup;
}
sdsfree(tmpfilename); tmpfilename = NULL;
retval = C_OK; /* If we reached this point, everything is fine. */
cleanup:
if (fd != -1) close(fd);
if (tmpfilename) unlink(tmpfilename);
sdsfree(tmpfilename);
sdsfree(acl);
return retval;
}
/* This function is called once the server is already running, modules are
* loaded, and we are ready to start, in order to load the ACLs either from
* the pending list of users defined in redis.conf, or from the ACL file.
* The function will just exit with an error if the user is trying to mix
* both the loading methods. */
void ACLLoadUsersAtStartup(void) {
if (server.acl_filename[0] != '\0' && listLength(UsersToLoad) != 0) {
serverLog(LL_WARNING,
"Configuring Redis with users defined in redis.conf and at "
"the same setting an ACL file path is invalid. This setup "
"is very likely to lead to configuration errors and security "
"holes, please define either an ACL file or declare users "
"directly in your redis.conf, but not both.");
exit(1);
}
if (ACLLoadConfiguredUsers() == C_ERR) {
serverLog(LL_WARNING,
"Critical error while loading ACLs. Exiting.");
exit(1);
}
if (server.acl_filename[0] != '\0') {
sds errors = ACLLoadFromFile(server.acl_filename);
if (errors) {
serverLog(LL_WARNING,
"Aborting Redis startup because of ACL errors: %s", errors);
sdsfree(errors);
exit(1);
}
}
}
/* =============================================================================
* ACL related commands
* ==========================================================================*/
/* ACL -- show and modify the configuration of ACL users.
* ACL HELP
* ACL LOAD
* ACL LIST
* ACL USERS
* ACL CAT [<category>]
* ACL SETUSER <username> ... acl rules ...
* ACL DELUSER <username> [...]
* ACL GETUSER <username>
* ACL GENPASS
* ACL WHOAMI
*/
void aclCommand(client *c) {
char *sub = c->argv[1]->ptr;
if (!strcasecmp(sub,"setuser") && c->argc >= 3) {
sds username = c->argv[2]->ptr;
/* Create a temporary user to validate and stage all changes against
* before applying to an existing user or creating a new user. If all
* arguments are valid the user parameters will all be applied together.
* If there are any errors then none of the changes will be applied. */
user *tempu = ACLCreateUnlinkedUser();
user *u = ACLGetUserByName(username,sdslen(username));
if (u) ACLCopyUser(tempu, u);
for (int j = 3; j < c->argc; j++) {
if (ACLSetUser(tempu,c->argv[j]->ptr,sdslen(c->argv[j]->ptr)) != C_OK) {
char *errmsg = ACLSetUserStringError();
addReplyErrorFormat(c,
"Error in ACL SETUSER modifier '%s': %s",
(char*)c->argv[j]->ptr, errmsg);
ACLFreeUser(tempu);
return;
}
}
/* Overwrite the user with the temporary user we modified above. */
if (!u) u = ACLCreateUser(username,sdslen(username));
serverAssert(u != NULL);
ACLCopyUser(u, tempu);
ACLFreeUser(tempu);
addReply(c,shared.ok);
} else if (!strcasecmp(sub,"deluser") && c->argc >= 3) {
int deleted = 0;
for (int j = 2; j < c->argc; j++) {
sds username = c->argv[j]->ptr;
if (!strcmp(username,"default")) {
addReplyError(c,"The 'default' user cannot be removed");
return;
}
}
for (int j = 2; j < c->argc; j++) {
sds username = c->argv[j]->ptr;
user *u;
if (raxRemove(Users,(unsigned char*)username,
sdslen(username),
(void**)&u))
{
ACLFreeUserAndKillClients(u);
deleted++;
}
}
addReplyLongLong(c,deleted);
} else if (!strcasecmp(sub,"getuser") && c->argc == 3) {
user *u = ACLGetUserByName(c->argv[2]->ptr,sdslen(c->argv[2]->ptr));
if (u == NULL) {
addReplyNull(c);
return;
}
addReplyMapLen(c,4);
/* Flags */
addReplyBulkCString(c,"flags");
void *deflen = addReplyDeferredLen(c);
int numflags = 0;
for (int j = 0; ACLUserFlags[j].flag; j++) {
if (u->flags & ACLUserFlags[j].flag) {
addReplyBulkCString(c,ACLUserFlags[j].name);
numflags++;
}
}
setDeferredSetLen(c,deflen,numflags);
/* Passwords */
addReplyBulkCString(c,"passwords");
addReplyArrayLen(c,listLength(u->passwords));
listIter li;
listNode *ln;
listRewind(u->passwords,&li);
while((ln = listNext(&li))) {
sds thispass = listNodeValue(ln);
addReplyBulkCBuffer(c,thispass,sdslen(thispass));
}
/* Commands */
addReplyBulkCString(c,"commands");
sds cmddescr = ACLDescribeUserCommandRules(u);
addReplyBulkSds(c,cmddescr);
/* Key patterns */
addReplyBulkCString(c,"keys");
if (u->flags & USER_FLAG_ALLKEYS) {
addReplyArrayLen(c,1);
addReplyBulkCBuffer(c,"*",1);
} else {
addReplyArrayLen(c,listLength(u->patterns));
listIter li;
listNode *ln;
listRewind(u->patterns,&li);
while((ln = listNext(&li))) {
sds thispat = listNodeValue(ln);
addReplyBulkCBuffer(c,thispat,sdslen(thispat));
}
}
} else if ((!strcasecmp(sub,"list") || !strcasecmp(sub,"users")) &&
c->argc == 2)
{
int justnames = !strcasecmp(sub,"users");
addReplyArrayLen(c,raxSize(Users));
raxIterator ri;
raxStart(&ri,Users);
raxSeek(&ri,"^",NULL,0);
while(raxNext(&ri)) {
user *u = ri.data;
if (justnames) {
addReplyBulkCBuffer(c,u->name,sdslen(u->name));
} else {
/* Return information in the configuration file format. */
sds config = sdsnew("user ");
config = sdscatsds(config,u->name);
config = sdscatlen(config," ",1);
sds descr = ACLDescribeUser(u);
config = sdscatsds(config,descr);
sdsfree(descr);
addReplyBulkSds(c,config);
}
}
raxStop(&ri);
} else if (!strcasecmp(sub,"whoami") && c->argc == 2) {
if (c->user != NULL) {
addReplyBulkCBuffer(c,c->user->name,sdslen(c->user->name));
} else {
addReplyNull(c);
}
} else if (server.acl_filename[0] == '\0' &&
(!strcasecmp(sub,"load") || !strcasecmp(sub,"save")))
{
addReplyError(c,"This Redis instance is not configured to use an ACL file. You may want to specify users via the ACL SETUSER command and then issue a CONFIG REWRITE (assuming you have a Redis configuration file set) in order to store users in the Redis configuration.");
return;
} else if (!strcasecmp(sub,"load") && c->argc == 2) {
sds errors = ACLLoadFromFile(server.acl_filename);
if (errors == NULL) {
addReply(c,shared.ok);
} else {
addReplyError(c,errors);
sdsfree(errors);
}
} else if (!strcasecmp(sub,"save") && c->argc == 2) {
if (ACLSaveToFile(server.acl_filename) == C_OK) {
addReply(c,shared.ok);
} else {
addReplyError(c,"There was an error trying to save the ACLs. "
"Please check the server logs for more "
"information");
}
} else if (!strcasecmp(sub,"cat") && c->argc == 2) {
void *dl = addReplyDeferredLen(c);
int j;
for (j = 0; ACLCommandCategories[j].flag != 0; j++)
addReplyBulkCString(c,ACLCommandCategories[j].name);
setDeferredArrayLen(c,dl,j);
} else if (!strcasecmp(sub,"cat") && c->argc == 3) {
uint64_t cflag = ACLGetCommandCategoryFlagByName(c->argv[2]->ptr);
if (cflag == 0) {
addReplyErrorFormat(c, "Unknown category '%s'", (char*)c->argv[2]->ptr);
return;
}
int arraylen = 0;
void *dl = addReplyDeferredLen(c);
dictIterator *di = dictGetIterator(server.orig_commands);
dictEntry *de;
while ((de = dictNext(di)) != NULL) {
struct redisCommand *cmd = dictGetVal(de);
if (cmd->flags & CMD_MODULE) continue;
if (cmd->flags & cflag) {
addReplyBulkCString(c,cmd->name);
arraylen++;
}
}
dictReleaseIterator(di);
setDeferredArrayLen(c,dl,arraylen);
} else if (!strcasecmp(sub,"genpass") && c->argc == 2) {
char pass[32]; /* 128 bits of actual pseudo random data. */
getRandomHexChars(pass,sizeof(pass));
addReplyBulkCBuffer(c,pass,sizeof(pass));
} else if (!strcasecmp(sub,"help")) {
const char *help[] = {
"LOAD -- Reload users from the ACL file.",
"LIST -- Show user details in config file format.",
"USERS -- List all the registered usernames.",
"SETUSER <username> [attribs ...] -- Create or modify a user.",
"GETUSER <username> -- Get the user details.",
"DELUSER <username> [...] -- Delete a list of users.",
"CAT -- List available categories.",
"CAT <category> -- List commands inside category.",
"GENPASS -- Generate a secure user password.",
"WHOAMI -- Return the current connection username.",
NULL
};
addReplyHelp(c,help);
} else {
addReplySubcommandSyntaxError(c);
}
}
void addReplyCommandCategories(client *c, struct redisCommand *cmd) {
int flagcount = 0;
void *flaglen = addReplyDeferredLen(c);
for (int j = 0; ACLCommandCategories[j].flag != 0; j++) {
if (cmd->flags & ACLCommandCategories[j].flag) {
addReplyStatusFormat(c, "@%s", ACLCommandCategories[j].name);
flagcount++;
}
}
setDeferredSetLen(c, flaglen, flagcount);
}
/* AUTH <password>
* AUTH <username> <password> (Redis >= 6.0 form)
*
* When the user is omitted it means that we are trying to authenticate
* against the default user. */
void authCommand(client *c) {
/* Only two or three argument forms are allowed. */
if (c->argc > 3) {
addReply(c,shared.syntaxerr);
return;
}
/* Handle the two different forms here. The form with two arguments
* will just use "default" as username. */
robj *username, *password;
if (c->argc == 2) {
/* Mimic the old behavior of giving an error for the two commands
* from if no password is configured. */
if (DefaultUser->flags & USER_FLAG_NOPASS) {
addReplyError(c,"AUTH <password> called without any password "
"configured for the default user. Are you sure "
"your configuration is correct?");
return;
}
username = createStringObject("default",7);
password = c->argv[1];
} else {
username = c->argv[1];
password = c->argv[2];
}
if (ACLAuthenticateUser(c,username,password) == C_OK) {
addReply(c,shared.ok);
} else {
addReplyError(c,"-WRONGPASS invalid username-password pair");
}
/* Free the "default" string object we created for the two
* arguments form. */
if (c->argc == 2) decrRefCount(username);
}
......@@ -351,8 +351,8 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
* if flags has AE_FILE_EVENTS set, file events are processed.
* if flags has AE_TIME_EVENTS set, time events are processed.
* if flags has AE_DONT_WAIT set the function returns ASAP until all
* if flags has AE_CALL_AFTER_SLEEP set, the aftersleep callback is called.
* the events that's possible to process without to wait are processed.
* if flags has AE_CALL_AFTER_SLEEP set, the aftersleep callback is called.
*
* The function returns the number of events processed. */
int aeProcessEvents(aeEventLoop *eventLoop, int flags)
......
......@@ -193,6 +193,20 @@ int anetSendTimeout(char *err, int fd, long long ms) {
return ANET_OK;
}
/* Set the socket receive timeout (SO_RCVTIMEO socket option) to the specified
* number of milliseconds, or disable it if the 'ms' argument is zero. */
int anetRecvTimeout(char *err, int fd, long long ms) {
struct timeval tv;
tv.tv_sec = ms/1000;
tv.tv_usec = (ms%1000)*1000;
if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) == -1) {
anetSetError(err, "setsockopt SO_RCVTIMEO: %s", strerror(errno));
return ANET_ERR;
}
return ANET_OK;
}
/* anetGenericResolve() is called by anetResolve() and anetResolveIP() to
* do the actual work. It resolves the hostname "host" and set the string
* representation of the IP address into the buffer pointed by "ipbuf".
......
......@@ -70,6 +70,7 @@ int anetEnableTcpNoDelay(char *err, int fd);
int anetDisableTcpNoDelay(char *err, int fd);
int anetTcpKeepAlive(char *err, int fd);
int anetSendTimeout(char *err, int fd, long long ms);
int anetRecvTimeout(char *err, int fd, long long ms);
int anetPeerToString(int fd, char *ip, size_t ip_len, int *port);
int anetKeepAlive(char *err, int fd, int interval);
int anetSockName(int fd, char *ip, size_t ip_len, int *port);
......
......@@ -197,6 +197,12 @@ ssize_t aofRewriteBufferWrite(int fd) {
* AOF file implementation
* ------------------------------------------------------------------------- */
/* Return true if an AOf fsync is currently already in progress in a
* BIO thread. */
int aofFsyncInProgress(void) {
return bioPendingJobsOfType(BIO_AOF_FSYNC) != 0;
}
/* Starts a background task that performs fsync() against the specified
* file descriptor (the one of the AOF file) in another thread. */
void aof_background_fsync(int fd) {
......@@ -204,7 +210,7 @@ void aof_background_fsync(int fd) {
}
/* Kills an AOFRW child process if exists */
static void killAppendOnlyChild(void) {
void killAppendOnlyChild(void) {
int statloc;
/* No AOFRW child? return. */
if (server.aof_child_pid == -1) return;
......@@ -221,6 +227,8 @@ static void killAppendOnlyChild(void) {
server.aof_rewrite_time_start = -1;
/* Close pipes used for IPC between the two processes. */
aofClosePipes();
closeChildInfoPipe();
updateDictResizePolicy();
}
/* Called when the user switches from "appendonly yes" to "appendonly no"
......@@ -261,7 +269,7 @@ int startAppendOnly(void) {
serverLog(LL_WARNING,"AOF was enabled but there is already a child process saving an RDB file on disk. An AOF background was scheduled to start when possible.");
} else {
/* If there is a pending AOF rewrite, we need to switch it off and
* start a new one: the old one cannot be reused becuase it is not
* start a new one: the old one cannot be reused because it is not
* accumulating the AOF buffer. */
if (server.aof_child_pid != -1) {
serverLog(LL_WARNING,"AOF was enabled but there is already an AOF rewriting in background. Stopping background AOF and starting a rewrite now.");
......@@ -295,9 +303,7 @@ ssize_t aofWrite(int fd, const char *buf, size_t len) {
nwritten = write(fd, buf, len);
if (nwritten < 0) {
if (errno == EINTR) {
continue;
}
if (errno == EINTR) continue;
return totwritten ? totwritten : -1;
}
......@@ -333,10 +339,24 @@ void flushAppendOnlyFile(int force) {
int sync_in_progress = 0;
mstime_t latency;
if (sdslen(server.aof_buf) == 0) return;
if (sdslen(server.aof_buf) == 0) {
/* Check if we need to do fsync even the aof buffer is empty,
* because previously in AOF_FSYNC_EVERYSEC mode, fsync is
* called only when aof buffer is not empty, so if users
* stop write commands before fsync called in one second,
* the data in page cache cannot be flushed in time. */
if (server.aof_fsync == AOF_FSYNC_EVERYSEC &&
server.aof_fsync_offset != server.aof_current_size &&
server.unixtime > server.aof_last_fsync &&
!(sync_in_progress = aofFsyncInProgress())) {
goto try_fsync;
} else {
return;
}
}
if (server.aof_fsync == AOF_FSYNC_EVERYSEC)
sync_in_progress = bioPendingJobsOfType(BIO_AOF_FSYNC) != 0;
sync_in_progress = aofFsyncInProgress();
if (server.aof_fsync == AOF_FSYNC_EVERYSEC && !force) {
/* With this append fsync policy we do background fsyncing.
......@@ -468,6 +488,7 @@ void flushAppendOnlyFile(int force) {
server.aof_buf = sdsempty();
}
try_fsync:
/* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are
* children doing I/O in the background. */
if (server.aof_no_fsync_on_rewrite &&
......@@ -482,10 +503,14 @@ void flushAppendOnlyFile(int force) {
redis_fsync(server.aof_fd); /* Let's try to get this data on the disk */
latencyEndMonitor(latency);
latencyAddSampleIfNeeded("aof-fsync-always",latency);
server.aof_fsync_offset = server.aof_current_size;
server.aof_last_fsync = server.unixtime;
} else if ((server.aof_fsync == AOF_FSYNC_EVERYSEC &&
server.unixtime > server.aof_last_fsync)) {
if (!sync_in_progress) aof_background_fsync(server.aof_fd);
if (!sync_in_progress) {
aof_background_fsync(server.aof_fd);
server.aof_fsync_offset = server.aof_current_size;
}
server.aof_last_fsync = server.unixtime;
}
}
......@@ -645,7 +670,9 @@ struct client *createFakeClient(void) {
c->obuf_soft_limit_reached_time = 0;
c->watched_keys = listCreate();
c->peerid = NULL;
listSetFreeMethod(c->reply,decrRefCountVoid);
c->resp = 2;
c->user = NULL;
listSetFreeMethod(c->reply,freeClientReplyValue);
listSetDupMethod(c->reply,dupClientReplyValue);
initClientMultiState(c);
return c;
......@@ -677,18 +704,20 @@ int loadAppendOnlyFile(char *filename) {
int old_aof_state = server.aof_state;
long loops = 0;
off_t valid_up_to = 0; /* Offset of latest well-formed command loaded. */
off_t valid_before_multi = 0; /* Offset before MULTI command loaded. */
if (fp == NULL) {
serverLog(LL_WARNING,"Fatal error: can't open the append log file for reading: %s",strerror(errno));
exit(1);
}
/* Handle a zero-length AOF file as a special case. An emtpy AOF file
/* Handle a zero-length AOF file as a special case. An empty AOF file
* is a valid AOF because an empty server with AOF enabled will create
* a zero length file at startup, that will remain like that if no write
* operation is received. */
if (fp && redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0) {
server.aof_current_size = 0;
server.aof_fsync_offset = server.aof_current_size;
fclose(fp);
return C_ERR;
}
......@@ -698,7 +727,7 @@ int loadAppendOnlyFile(char *filename) {
server.aof_state = AOF_OFF;
fakeClient = createFakeClient();
startLoading(fp);
startLoadingFile(fp, filename);
/* Check if this AOF file has an RDB preamble. In that case we need to
* load the RDB file and later continue loading the AOF tail. */
......@@ -777,16 +806,28 @@ int loadAppendOnlyFile(char *filename) {
/* Command lookup */
cmd = lookupCommand(argv[0]->ptr);
if (!cmd) {
serverLog(LL_WARNING,"Unknown command '%s' reading the append only file", (char*)argv[0]->ptr);
serverLog(LL_WARNING,
"Unknown command '%s' reading the append only file",
(char*)argv[0]->ptr);
exit(1);
}
if (cmd == server.multiCommand) valid_before_multi = valid_up_to;
/* Run the command in the context of a fake client */
fakeClient->cmd = cmd;
if (fakeClient->flags & CLIENT_MULTI &&
fakeClient->cmd->proc != execCommand)
{
queueMultiCommand(fakeClient);
} else {
cmd->proc(fakeClient);
}
/* The fake client should not have a reply */
serverAssert(fakeClient->bufpos == 0 && listLength(fakeClient->reply) == 0);
serverAssert(fakeClient->bufpos == 0 &&
listLength(fakeClient->reply) == 0);
/* The fake client should never get blocked */
serverAssert((fakeClient->flags & CLIENT_BLOCKED) == 0);
......@@ -798,8 +839,15 @@ int loadAppendOnlyFile(char *filename) {
}
/* This point can only be reached when EOF is reached without errors.
* If the client is in the middle of a MULTI/EXEC, log error and quit. */
if (fakeClient->flags & CLIENT_MULTI) goto uxeof;
* If the client is in the middle of a MULTI/EXEC, handle it as it was
* a short read, even if technically the protocol is correct: we want
* to remove the unprocessed tail and continue. */
if (fakeClient->flags & CLIENT_MULTI) {
serverLog(LL_WARNING,
"Revert incomplete MULTI/EXEC transaction in AOF file");
valid_up_to = valid_before_multi;
goto uxeof;
}
loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */
fclose(fp);
......@@ -808,6 +856,7 @@ loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */
stopLoading();
aofUpdateCurrentSize();
server.aof_rewrite_base_size = server.aof_current_size;
server.aof_fsync_offset = server.aof_current_size;
return C_OK;
readerr: /* Read error. If feof(fp) is true, fall through to unexpected EOF. */
......@@ -1119,6 +1168,7 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) {
streamID id;
int64_t numfields;
if (s->length) {
/* Reconstruct the stream data using XADD commands. */
while(streamIteratorGetID(&si,&id,&numfields)) {
/* Emit a two elements array for each item. The first is
......@@ -1137,6 +1187,27 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) {
if (rioWriteBulkString(r,(char*)value,value_len) == 0) return 0;
}
}
} else {
/* Use the XADD MAXLEN 0 trick to generate an empty stream if
* the key we are serializing is an empty string, which is possible
* for the Stream type. */
if (rioWriteBulkCount(r,'*',7) == 0) return 0;
if (rioWriteBulkString(r,"XADD",4) == 0) return 0;
if (rioWriteBulkObject(r,key) == 0) return 0;
if (rioWriteBulkString(r,"MAXLEN",6) == 0) return 0;
if (rioWriteBulkString(r,"0",1) == 0) return 0;
if (rioWriteBulkStreamID(r,&s->last_id) == 0) return 0;
if (rioWriteBulkString(r,"x",1) == 0) return 0;
if (rioWriteBulkString(r,"y",1) == 0) return 0;
}
/* Append XSETID after XADD, make sure lastid is correct,
* in case of XDEL lastid. */
if (rioWriteBulkCount(r,'*',3) == 0) return 0;
if (rioWriteBulkString(r,"XSETID",6) == 0) return 0;
if (rioWriteBulkObject(r,key) == 0) return 0;
if (rioWriteBulkStreamID(r,&s->last_id) == 0) return 0;
/* Create all the stream consumer groups. */
if (s->cgroups) {
......@@ -1193,7 +1264,7 @@ int rewriteModuleObject(rio *r, robj *key, robj *o) {
RedisModuleIO io;
moduleValue *mv = o->ptr;
moduleType *mt = mv->type;
moduleInitIOContext(io,mt,r);
moduleInitIOContext(io,mt,r,key);
mt->aof_rewrite(&io,key,mv->value);
if (io.ctx) {
moduleFreeContext(io.ctx);
......@@ -1565,6 +1636,9 @@ void aofRemoveTempFile(pid_t childpid) {
snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) childpid);
unlink(tmpfile);
snprintf(tmpfile,256,"temp-rewriteaof-%d.aof", (int) childpid);
unlink(tmpfile);
}
/* Update the server.aof_current_size field explicitly using stat(2)
......@@ -1692,6 +1766,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
server.aof_selected_db = -1; /* Make sure SELECT is re-issued */
aofUpdateCurrentSize();
server.aof_rewrite_base_size = server.aof_current_size;
server.aof_current_size = server.aof_current_size;
/* Clear regular AOF buffer since its contents was just written to
* the new AOF from the background rewrite buffer. */
......@@ -1713,7 +1788,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
"Background AOF rewrite signal handler took %lldus", ustime()-now);
} else if (!bysignal && exitcode != 0) {
/* SIGUSR1 is whitelisted, so we have a way to kill a child without
* tirggering an error conditon. */
* tirggering an error condition. */
if (bysignal != SIGUSR1)
server.aof_lastbgrewrite_status = C_ERR;
serverLog(LL_WARNING,
......
/* This file implements atomic counters using __atomic or __sync macros if
* available, otherwise synchronizing different threads using a mutex.
*
* The exported interaface is composed of three macros:
* The exported interface is composed of three macros:
*
* atomicIncr(var,count) -- Increment the atomic counter
* atomicGetIncr(var,oldvalue_var,count) -- Get and increment the atomic counter
......@@ -16,7 +16,7 @@
* pthread_mutex_t myvar_mutex;
* atomicSet(myvar,12345);
*
* If atomic primitives are availble (tested in config.h) the mutex
* If atomic primitives are available (tested in config.h) the mutex
* is not used.
*
* Never use return value from the macros, instead use the AtomicGetIncr()
......
......@@ -17,7 +17,7 @@
*
* The design is trivial, we have a structure representing a job to perform
* and a different thread and job queue for every job type.
* Every thread wait for new jobs in its queue, and process every job
* Every thread waits for new jobs in its queue, and process every job
* sequentially.
*
* Jobs of the same type are guaranteed to be processed from the least
......@@ -204,14 +204,14 @@ void *bioProcessBackgroundJobs(void *arg) {
}
zfree(job);
/* Unblock threads blocked on bioWaitStepOfType() if any. */
pthread_cond_broadcast(&bio_step_cond[type]);
/* Lock again before reiterating the loop, if there are no longer
* jobs to process we'll block again in pthread_cond_wait(). */
pthread_mutex_lock(&bio_mutex[type]);
listDelNode(bio_jobs[type],ln);
bio_pending[type]--;
/* Unblock threads blocked on bioWaitStepOfType() if any. */
pthread_cond_broadcast(&bio_step_cond[type]);
}
}
......
......@@ -918,7 +918,7 @@ void bitfieldCommand(client *c) {
struct bitfieldOp *ops = NULL; /* Array of ops to execute at end. */
int owtype = BFOVERFLOW_WRAP; /* Overflow type. */
int readonly = 1;
size_t higest_write_offset = 0;
size_t highest_write_offset = 0;
for (j = 2; j < c->argc; j++) {
int remargs = c->argc-j-1; /* Remaining args other than current. */
......@@ -968,8 +968,8 @@ void bitfieldCommand(client *c) {
if (opcode != BITFIELDOP_GET) {
readonly = 0;
if (higest_write_offset < bitoffset + bits - 1)
higest_write_offset = bitoffset + bits - 1;
if (highest_write_offset < bitoffset + bits - 1)
highest_write_offset = bitoffset + bits - 1;
/* INCRBY and SET require another argument. */
if (getLongLongFromObjectOrReply(c,c->argv[j+3],&i64,NULL) != C_OK){
zfree(ops);
......@@ -994,15 +994,21 @@ void bitfieldCommand(client *c) {
/* Lookup for read is ok if key doesn't exit, but errors
* if it's not a string. */
o = lookupKeyRead(c->db,c->argv[1]);
if (o != NULL && checkType(c,o,OBJ_STRING)) return;
if (o != NULL && checkType(c,o,OBJ_STRING)) {
zfree(ops);
return;
}
} else {
/* Lookup by making room up to the farest bit reached by
* this operation. */
if ((o = lookupStringForBitCommand(c,
higest_write_offset)) == NULL) return;
highest_write_offset)) == NULL) {
zfree(ops);
return;
}
}
addReplyMultiBulkLen(c,numops);
addReplyArrayLen(c,numops);
/* Actually process the operations. */
for (j = 0; j < numops; j++) {
......@@ -1047,7 +1053,7 @@ void bitfieldCommand(client *c) {
setSignedBitfield(o->ptr,thisop->offset,
thisop->bits,newval);
} else {
addReply(c,shared.nullbulk);
addReplyNull(c);
}
} else {
uint64_t oldval, newval, wrapped, retval;
......@@ -1076,7 +1082,7 @@ void bitfieldCommand(client *c) {
setUnsignedBitfield(o->ptr,thisop->offset,
thisop->bits,newval);
} else {
addReply(c,shared.nullbulk);
addReplyNull(c);
}
}
changes++;
......
......@@ -77,10 +77,18 @@ int serveClientBlockedOnList(client *receiver, robj *key, robj *dstkey, redisDb
* is zero. */
int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int unit) {
long long tval;
long double ftval;
if (unit == UNIT_SECONDS) {
if (getLongDoubleFromObjectOrReply(c,object,&ftval,
"timeout is not an float or out of range") != C_OK)
return C_ERR;
tval = (long long) (ftval * 1000.0);
} else {
if (getLongLongFromObjectOrReply(c,object,&tval,
"timeout is not an integer or out of range") != C_OK)
return C_ERR;
}
if (tval < 0) {
addReplyError(c,"timeout is negative");
......@@ -88,7 +96,6 @@ int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int
}
if (tval > 0) {
if (unit == UNIT_SECONDS) tval *= 1000;
tval += mstime();
}
*timeout = tval;
......@@ -126,12 +133,37 @@ void processUnblockedClients(void) {
* the code is conceptually more correct this way. */
if (!(c->flags & CLIENT_BLOCKED)) {
if (c->querybuf && sdslen(c->querybuf) > 0) {
processInputBuffer(c);
processInputBufferAndReplicate(c);
}
}
}
}
/* This function will schedule the client for reprocessing at a safe time.
*
* This is useful when a client was blocked for some reason (blocking opeation,
* CLIENT PAUSE, or whatever), because it may end with some accumulated query
* buffer that needs to be processed ASAP:
*
* 1. When a client is blocked, its readable handler is still active.
* 2. However in this case it only gets data into the query buffer, but the
* query is not parsed or executed once there is enough to proceed as
* usually (because the client is blocked... so we can't execute commands).
* 3. When the client is unblocked, without this function, the client would
* have to write some query in order for the readable handler to finally
* call processQueryBuffer*() on it.
* 4. With this function instead we can put the client in a queue that will
* process it for queries ready to be executed at a safe time.
*/
void queueClientForReprocessing(client *c) {
/* The client may already be into the unblocked list because of a previous
* blocking operation, don't add back it into the list multiple times. */
if (!(c->flags & CLIENT_UNBLOCKED)) {
c->flags |= CLIENT_UNBLOCKED;
listAddNodeTail(server.unblocked_clients,c);
}
}
/* Unblock a client calling the right function depending on the kind
* of operation the client is blocking for. */
void unblockClient(client *c) {
......@@ -152,12 +184,7 @@ void unblockClient(client *c) {
server.blocked_clients_by_type[c->btype]--;
c->flags &= ~CLIENT_BLOCKED;
c->btype = BLOCKED_NONE;
/* The client may already be into the unblocked list because of a previous
* blocking operation, don't add back it into the list multiple times. */
if (!(c->flags & CLIENT_UNBLOCKED)) {
c->flags |= CLIENT_UNBLOCKED;
listAddNodeTail(server.unblocked_clients,c);
}
queueClientForReprocessing(c);
}
/* This function gets called when a blocked client timed out in order to
......@@ -167,7 +194,7 @@ void replyToBlockedClientTimedOut(client *c) {
if (c->btype == BLOCKED_LIST ||
c->btype == BLOCKED_ZSET ||
c->btype == BLOCKED_STREAM) {
addReply(c,shared.nullmultibulk);
addReplyNullArray(c);
} else if (c->btype == BLOCKED_WAIT) {
addReplyLongLong(c,replicationCountAcksByOffset(c->bpop.reploffset));
} else if (c->btype == BLOCKED_MODULE) {
......@@ -195,7 +222,7 @@ void disconnectAllBlockedClients(void) {
if (c->flags & CLIENT_BLOCKED) {
addReplySds(c,sdsnew(
"-UNBLOCKED force unblock from blocking operation, "
"instance state changed (master -> slave?)\r\n"));
"instance state changed (master -> replica?)\r\n"));
unblockClient(c);
c->flags |= CLIENT_CLOSE_AFTER_REPLY;
}
......@@ -370,40 +397,58 @@ void handleClientsBlockedOnKeys(void) {
if (receiver->btype != BLOCKED_STREAM) continue;
streamID *gt = dictFetchValue(receiver->bpop.keys,
rl->key);
if (s->last_id.ms > gt->ms ||
(s->last_id.ms == gt->ms &&
s->last_id.seq > gt->seq))
{
streamID start = *gt;
start.seq++; /* Can't overflow, it's an uint64_t */
/* If we blocked in the context of a consumer
* group, we need to resolve the group and
* consumer here. */
* group, we need to resolve the group and update the
* last ID the client is blocked for: this is needed
* because serving other clients in the same consumer
* group will alter the "last ID" of the consumer
* group, and clients blocked in a consumer group are
* always blocked for the ">" ID: we need to deliver
* only new messages and avoid unblocking the client
* otherwise. */
streamCG *group = NULL;
streamConsumer *consumer = NULL;
if (receiver->bpop.xread_group) {
group = streamLookupCG(s,
receiver->bpop.xread_group->ptr);
/* In theory if the group is not found we
* just perform the read without the group,
* but actually when the group, or the key
* itself is deleted (triggering the removal
* of the group), we check for blocked clients
* and send them an error. */
/* If the group was not found, send an error
* to the consumer. */
if (!group) {
addReplyError(receiver,
"-NOGROUP the consumer group this client "
"was blocked on no longer exists");
unblockClient(receiver);
continue;
} else {
*gt = group->last_id;
}
}
if (streamCompareID(&s->last_id, gt) > 0) {
streamID start = *gt;
start.seq++; /* Can't overflow, it's an uint64_t */
/* Lookup the consumer for the group, if any. */
streamConsumer *consumer = NULL;
int noack = 0;
if (group) {
consumer = streamLookupConsumer(group,
receiver->bpop.xread_consumer->ptr,
1);
noack = receiver->bpop.xread_group_noack;
}
/* Emit the two elements sub-array consisting of
* the name of the stream and the data we
* extracted from it. Wrapped in a single-item
* array, since we have just one key. */
addReplyMultiBulkLen(receiver,1);
addReplyMultiBulkLen(receiver,2);
if (receiver->resp == 2) {
addReplyArrayLen(receiver,1);
addReplyArrayLen(receiver,2);
} else {
addReplyMapLen(receiver,1);
}
addReplyBulk(receiver,rl->key);
streamPropInfo pi = {
......@@ -412,7 +457,7 @@ void handleClientsBlockedOnKeys(void) {
};
streamReplyWithRange(receiver,s,&start,NULL,
receiver->bpop.xread_count,
0, group, consumer, 0, &pi);
0, group, consumer, noack, &pi);
/* Note that after we unblock the client, 'gt'
* and other receiver->bpop stuff are no longer
......
......@@ -1230,7 +1230,7 @@ void clearNodeFailureIfNeeded(clusterNode *node) {
serverLog(LL_NOTICE,
"Clear FAIL state for node %.40s: %s is reachable again.",
node->name,
nodeIsSlave(node) ? "slave" : "master without slots");
nodeIsSlave(node) ? "replica" : "master without slots");
node->flags &= ~CLUSTER_NODE_FAIL;
clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG);
}
......@@ -1589,6 +1589,12 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc
}
}
/* After updating the slots configuration, don't do any actual change
* in the state of the server if a module disabled Redis Cluster
* keys redirections. */
if (server.cluster_module_flags & CLUSTER_MODULE_FLAG_NO_REDIRECTION)
return;
/* If at least one slot was reassigned from a node to another node
* with a greater configEpoch, it is possible that:
* 1) We are a master left without slots. This means that we were
......@@ -2059,7 +2065,7 @@ int clusterProcessPacket(clusterLink *link) {
server.cluster->mf_end = mstime() + CLUSTER_MF_TIMEOUT;
server.cluster->mf_slave = sender;
pauseClients(mstime()+(CLUSTER_MF_TIMEOUT*2));
serverLog(LL_WARNING,"Manual failover requested by slave %.40s.",
serverLog(LL_WARNING,"Manual failover requested by replica %.40s.",
sender->name);
} else if (type == CLUSTERMSG_TYPE_UPDATE) {
clusterNode *n; /* The node the update is about. */
......@@ -2873,7 +2879,7 @@ void clusterLogCantFailover(int reason) {
switch(reason) {
case CLUSTER_CANT_FAILOVER_DATA_AGE:
msg = "Disconnected from master for longer than allowed. "
"Please check the 'cluster-slave-validity-factor' configuration "
"Please check the 'cluster-replica-validity-factor' configuration "
"option.";
break;
case CLUSTER_CANT_FAILOVER_WAITING_DELAY:
......@@ -3025,6 +3031,7 @@ void clusterHandleSlaveFailover(void) {
if (server.cluster->mf_end) {
server.cluster->failover_auth_time = mstime();
server.cluster->failover_auth_rank = 0;
clusterDoBeforeSleep(CLUSTER_TODO_HANDLE_FAILOVER);
}
serverLog(LL_WARNING,
"Start of election delayed for %lld milliseconds "
......@@ -3054,7 +3061,7 @@ void clusterHandleSlaveFailover(void) {
server.cluster->failover_auth_time += added_delay;
server.cluster->failover_auth_rank = newrank;
serverLog(LL_WARNING,
"Slave rank updated to #%d, added %lld milliseconds of delay.",
"Replica rank updated to #%d, added %lld milliseconds of delay.",
newrank, added_delay);
}
}
......@@ -3100,7 +3107,7 @@ void clusterHandleSlaveFailover(void) {
(unsigned long long) myself->configEpoch);
}
/* Take responsability for the cluster slots. */
/* Take responsibility for the cluster slots. */
clusterFailoverReplaceYourMaster();
} else {
clusterLogCantFailover(CLUSTER_CANT_FAILOVER_WAITING_VOTES);
......@@ -3151,11 +3158,11 @@ void clusterHandleSlaveMigration(int max_slaves) {
!nodeTimedOut(mymaster->slaves[j])) okslaves++;
if (okslaves <= server.cluster_migration_barrier) return;
/* Step 3: Idenitfy a candidate for migration, and check if among the
/* Step 3: Identify a candidate for migration, and check if among the
* masters with the greatest number of ok slaves, I'm the one with the
* smallest node ID (the "candidate slave").
*
* Note: this means that eventually a replica migration will occurr
* Note: this means that eventually a replica migration will occur
* since slaves that are reachable again always have their FAIL flag
* cleared, so eventually there must be a candidate. At the same time
* this does not mean that there are no race conditions possible (two
......@@ -3210,7 +3217,8 @@ void clusterHandleSlaveMigration(int max_slaves) {
* the natural slaves of this instance to advertise their switch from
* the old master to the new one. */
if (target && candidate == myself &&
(mstime()-target->orphaned_time) > CLUSTER_SLAVE_MIGRATION_DELAY)
(mstime()-target->orphaned_time) > CLUSTER_SLAVE_MIGRATION_DELAY &&
!(server.cluster_module_flags & CLUSTER_MODULE_FLAG_NO_FAILOVER))
{
serverLog(LL_WARNING,"Migrating to orphaned master %.40s",
target->name);
......@@ -3321,14 +3329,18 @@ void clusterCron(void) {
int changed = 0;
if (prev_ip == NULL && curr_ip != NULL) changed = 1;
if (prev_ip != NULL && curr_ip == NULL) changed = 1;
if (prev_ip && curr_ip && strcmp(prev_ip,curr_ip)) changed = 1;
else if (prev_ip != NULL && curr_ip == NULL) changed = 1;
else if (prev_ip && curr_ip && strcmp(prev_ip,curr_ip)) changed = 1;
if (changed) {
if (prev_ip) zfree(prev_ip);
prev_ip = curr_ip;
if (prev_ip) prev_ip = zstrdup(prev_ip);
if (curr_ip) {
/* We always take a copy of the previous IP address, by
* duplicating the string. This way later we can check if
* the address really changed. */
prev_ip = zstrdup(prev_ip);
strncpy(myself->ip,server.cluster_announce_ip,NET_IP_STR_LEN);
myself->ip[NET_IP_STR_LEN-1] = '\0';
} else {
......@@ -3559,6 +3571,7 @@ void clusterCron(void) {
if (nodeIsSlave(myself)) {
clusterHandleManualFailover();
if (!(server.cluster_module_flags & CLUSTER_MODULE_FLAG_NO_FAILOVER))
clusterHandleSlaveFailover();
/* If there are orphaned slaves, and we are a slave among the masters
* with the max number of non-failing slaves, consider migrating to
......@@ -3736,7 +3749,7 @@ void clusterCloseAllSlots(void) {
* -------------------------------------------------------------------------- */
/* The following are defines that are only used in the evaluation function
* and are based on heuristics. Actaully the main point about the rejoin and
* and are based on heuristics. Actually the main point about the rejoin and
* writable delay is that they should be a few orders of magnitude larger
* than the network latency. */
#define CLUSTER_MAX_REJOIN_DELAY 5000
......@@ -3865,6 +3878,11 @@ int verifyClusterConfigWithData(void) {
int j;
int update_config = 0;
/* Return ASAP if a module disabled cluster redirections. In that case
* every master can store keys about every possible hash slot. */
if (server.cluster_module_flags & CLUSTER_MODULE_FLAG_NO_REDIRECTION)
return C_OK;
/* If this node is a slave, don't perform the check at all as we
* completely depend on the replication stream. */
if (nodeIsSlave(myself)) return C_OK;
......@@ -4109,7 +4127,7 @@ void clusterReplyMultiBulkSlots(client *c) {
*/
int num_masters = 0;
void *slot_replylen = addDeferredMultiBulkLength(c);
void *slot_replylen = addReplyDeferredLen(c);
dictEntry *de;
dictIterator *di = dictGetSafeIterator(server.cluster->nodes);
......@@ -4129,7 +4147,7 @@ void clusterReplyMultiBulkSlots(client *c) {
}
if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) {
int nested_elements = 3; /* slots (2) + master addr (1). */
void *nested_replylen = addDeferredMultiBulkLength(c);
void *nested_replylen = addReplyDeferredLen(c);
if (bit && j == CLUSTER_SLOTS-1) j++;
......@@ -4145,7 +4163,7 @@ void clusterReplyMultiBulkSlots(client *c) {
start = -1;
/* First node reply position is always the master */
addReplyMultiBulkLen(c, 3);
addReplyArrayLen(c, 3);
addReplyBulkCString(c, node->ip);
addReplyLongLong(c, node->port);
addReplyBulkCBuffer(c, node->name, CLUSTER_NAMELEN);
......@@ -4155,19 +4173,19 @@ void clusterReplyMultiBulkSlots(client *c) {
/* This loop is copy/pasted from clusterGenNodeDescription()
* with modifications for per-slot node aggregation */
if (nodeFailed(node->slaves[i])) continue;
addReplyMultiBulkLen(c, 3);
addReplyArrayLen(c, 3);
addReplyBulkCString(c, node->slaves[i]->ip);
addReplyLongLong(c, node->slaves[i]->port);
addReplyBulkCBuffer(c, node->slaves[i]->name, CLUSTER_NAMELEN);
nested_elements++;
}
setDeferredMultiBulkLength(c, nested_replylen, nested_elements);
setDeferredArrayLen(c, nested_replylen, nested_elements);
num_masters++;
}
}
}
dictReleaseIterator(di);
setDeferredMultiBulkLength(c, slot_replylen, num_masters);
setDeferredArrayLen(c, slot_replylen, num_masters);
}
void clusterCommand(client *c) {
......@@ -4178,27 +4196,27 @@ void clusterCommand(client *c) {
if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) {
const char *help[] = {
"addslots <slot> [slot ...] -- Assign slots to current node.",
"bumpepoch -- Advance the cluster config epoch.",
"count-failure-reports <node-id> -- Return number of failure reports for <node-id>.",
"countkeysinslot <slot> - Return the number of keys in <slot>.",
"delslots <slot> [slot ...] -- Delete slots information from current node.",
"failover [force|takeover] -- Promote current slave node to being a master.",
"forget <node-id> -- Remove a node from the cluster.",
"getkeysinslot <slot> <count> -- Return key names stored by current node in a slot.",
"flushslots -- Delete current node own slots information.",
"info - Return onformation about the cluster.",
"keyslot <key> -- Return the hash slot for <key>.",
"meet <ip> <port> [bus-port] -- Connect nodes into a working cluster.",
"myid -- Return the node id.",
"nodes -- Return cluster configuration seen by node. Output format:",
"ADDSLOTS <slot> [slot ...] -- Assign slots to current node.",
"BUMPEPOCH -- Advance the cluster config epoch.",
"COUNT-failure-reports <node-id> -- Return number of failure reports for <node-id>.",
"COUNTKEYSINSLOT <slot> - Return the number of keys in <slot>.",
"DELSLOTS <slot> [slot ...] -- Delete slots information from current node.",
"FAILOVER [force|takeover] -- Promote current replica node to being a master.",
"FORGET <node-id> -- Remove a node from the cluster.",
"GETKEYSINSLOT <slot> <count> -- Return key names stored by current node in a slot.",
"FLUSHSLOTS -- Delete current node own slots information.",
"INFO - Return onformation about the cluster.",
"KEYSLOT <key> -- Return the hash slot for <key>.",
"MEET <ip> <port> [bus-port] -- Connect nodes into a working cluster.",
"MYID -- Return the node id.",
"NODES -- Return cluster configuration seen by node. Output format:",
" <id> <ip:port> <flags> <master> <pings> <pongs> <epoch> <link> <slot> ... <slot>",
"replicate <node-id> -- Configure current node as slave to <node-id>.",
"reset [hard|soft] -- Reset current node (default: soft).",
"set-config-epoch <epoch> - Set config epoch of current node.",
"setslot <slot> (importing|migrating|stable|node <node-id>) -- Set slot state.",
"slaves <node-id> -- Return <node-id> slaves.",
"slots -- Return information about slots range mappings. Each range is made of:",
"REPLICATE <node-id> -- Configure current node as replica to <node-id>.",
"RESET [hard|soft] -- Reset current node (default: soft).",
"SET-config-epoch <epoch> - Set config epoch of current node.",
"SETSLOT <slot> (importing|migrating|stable|node <node-id>) -- Set slot state.",
"REPLICAS <node-id> -- Return <node-id> replicas.",
"SLOTS -- Return information about slots range mappings. Each range is made of:",
" start, end, master and replicas IP addresses, ports and ids",
NULL
};
......@@ -4233,12 +4251,7 @@ NULL
}
} else if (!strcasecmp(c->argv[1]->ptr,"nodes") && c->argc == 2) {
/* CLUSTER NODES */
robj *o;
sds ci = clusterGenNodesDescription(0);
o = createObject(OBJ_STRING,ci);
addReplyBulk(c,o);
decrRefCount(o);
addReplyBulkSds(c,clusterGenNodesDescription(0));
} else if (!strcasecmp(c->argv[1]->ptr,"myid") && c->argc == 2) {
/* CLUSTER MYID */
addReplyBulkCBuffer(c,myself->name, CLUSTER_NAMELEN);
......@@ -4531,7 +4544,7 @@ NULL
keys = zmalloc(sizeof(robj*)*maxkeys);
numkeys = getKeysInSlot(slot, keys, maxkeys);
addReplyMultiBulkLen(c,numkeys);
addReplyArrayLen(c,numkeys);
for (j = 0; j < numkeys; j++) {
addReplyBulk(c,keys[j]);
decrRefCount(keys[j]);
......@@ -4574,7 +4587,7 @@ NULL
/* Can't replicate a slave. */
if (nodeIsSlave(n)) {
addReplyError(c,"I can only replicate a master, not a slave.");
addReplyError(c,"I can only replicate a master, not a replica.");
return;
}
......@@ -4593,7 +4606,8 @@ NULL
clusterSetMaster(n);
clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG);
addReply(c,shared.ok);
} else if (!strcasecmp(c->argv[1]->ptr,"slaves") && c->argc == 3) {
} else if ((!strcasecmp(c->argv[1]->ptr,"slaves") ||
!strcasecmp(c->argv[1]->ptr,"replicas")) && c->argc == 3) {
/* CLUSTER SLAVES <NODE ID> */
clusterNode *n = clusterLookupNode(c->argv[2]->ptr);
int j;
......@@ -4609,7 +4623,7 @@ NULL
return;
}
addReplyMultiBulkLen(c,n->numslaves);
addReplyArrayLen(c,n->numslaves);
for (j = 0; j < n->numslaves; j++) {
sds ni = clusterGenNodeDescription(n->slaves[j]);
addReplyBulkCString(c,ni);
......@@ -4647,10 +4661,10 @@ NULL
/* Check preconditions. */
if (nodeIsMaster(myself)) {
addReplyError(c,"You should send CLUSTER FAILOVER to a slave");
addReplyError(c,"You should send CLUSTER FAILOVER to a replica");
return;
} else if (myself->slaveof == NULL) {
addReplyError(c,"I'm a slave but my master is unknown to me");
addReplyError(c,"I'm a replica but my master is unknown to me");
return;
} else if (!force &&
(nodeFailed(myself->slaveof) ||
......@@ -4746,8 +4760,7 @@ NULL
clusterReset(hard);
addReply(c,shared.ok);
} else {
addReplyErrorFormat(c, "Unknown subcommand or wrong number of arguments for '%s'. Try CLUSTER HELP",
(char*)c->argv[1]->ptr);
addReplySubcommandSyntaxError(c);
return;
}
}
......@@ -4758,7 +4771,7 @@ NULL
/* Generates a DUMP-format representation of the object 'o', adding it to the
* io stream pointed by 'rio'. This function can't fail. */
void createDumpPayload(rio *payload, robj *o) {
void createDumpPayload(rio *payload, robj *o, robj *key) {
unsigned char buf[2];
uint64_t crc;
......@@ -4766,7 +4779,7 @@ void createDumpPayload(rio *payload, robj *o) {
* byte followed by the serialized object. This is understood by RESTORE. */
rioInitWithBuffer(payload,sdsempty());
serverAssert(rdbSaveObjectType(payload,o));
serverAssert(rdbSaveObject(payload,o));
serverAssert(rdbSaveObject(payload,o,key));
/* Write the footer, this is how it looks like:
* ----------------+---------------------+---------------+
......@@ -4814,28 +4827,26 @@ int verifyDumpPayload(unsigned char *p, size_t len) {
* DUMP is actually not used by Redis Cluster but it is the obvious
* complement of RESTORE and can be useful for different applications. */
void dumpCommand(client *c) {
robj *o, *dumpobj;
robj *o;
rio payload;
/* Check if the key is here. */
if ((o = lookupKeyRead(c->db,c->argv[1])) == NULL) {
addReply(c,shared.nullbulk);
addReplyNull(c);
return;
}
/* Create the DUMP encoded representation. */
createDumpPayload(&payload,o);
createDumpPayload(&payload,o,c->argv[1]);
/* Transfer to the client */
dumpobj = createObject(OBJ_STRING,payload.io.buffer.ptr);
addReplyBulk(c,dumpobj);
decrRefCount(dumpobj);
addReplyBulkSds(c,payload.io.buffer.ptr);
return;
}
/* RESTORE key ttl serialized-value [REPLACE] */
void restoreCommand(client *c) {
long long ttl, lfu_freq = -1, lru_idle = -1, lru_clock;
long long ttl, lfu_freq = -1, lru_idle = -1, lru_clock = -1;
rio payload;
int j, type, replace = 0, absttl = 0;
robj *obj;
......@@ -4897,7 +4908,7 @@ void restoreCommand(client *c) {
rioInitWithBuffer(&payload,c->argv[3]->ptr);
if (((type = rdbLoadObjectType(&payload)) == -1) ||
((obj = rdbLoadObject(type,&payload)) == NULL))
((obj = rdbLoadObject(type,&payload,c->argv[1])) == NULL))
{
addReplyError(c,"Bad data format");
return;
......@@ -5147,6 +5158,11 @@ try_again:
serverAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,dbid));
}
int non_expired = 0; /* Number of keys that we'll find non expired.
Note that serializing large keys may take some time
so certain keys that were found non expired by the
lookupKey() function, may be expired later. */
/* Create RESTORE payload and generate the protocol to call the command. */
for (j = 0; j < num_keys; j++) {
long long ttl = 0;
......@@ -5154,8 +5170,17 @@ try_again:
if (expireat != -1) {
ttl = expireat-mstime();
if (ttl < 0) {
continue;
}
if (ttl < 1) ttl = 1;
}
/* Relocate valid (non expired) keys into the array in successive
* positions to remove holes created by the keys that were present
* in the first lookup but are now expired after the second lookup. */
kv[non_expired++] = kv[j];
serverAssertWithInfo(c,NULL,
rioWriteBulkCount(&cmd,'*',replace ? 5 : 4));
......@@ -5171,7 +5196,7 @@ try_again:
/* Emit the payload argument, that is the serialized object using
* the DUMP format. */
createDumpPayload(&payload,ov[j]);
createDumpPayload(&payload,ov[j],kv[j]);
serverAssertWithInfo(c,NULL,
rioWriteBulkString(&cmd,payload.io.buffer.ptr,
sdslen(payload.io.buffer.ptr)));
......@@ -5183,6 +5208,9 @@ try_again:
serverAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"REPLACE",7));
}
/* Fix the actual number of keys we are migrating. */
num_keys = non_expired;
/* Transfer the query to the other node in 64K chunks. */
errno = 0;
{
......@@ -5218,6 +5246,10 @@ try_again:
int socket_error = 0;
int del_idx = 1; /* Index of the key argument for the replicated DEL op. */
/* Allocate the new argument vector that will replace the current command,
* to propagate the MIGRATE as a DEL command (if no COPY option was given).
* We allocate num_keys+1 because the additional argument is for "DEL"
* command name itself. */
if (!copy) newargv = zmalloc(sizeof(robj*)*(num_keys+1));
for (j = 0; j < num_keys; j++) {
......@@ -5418,9 +5450,17 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
multiCmd mc;
int i, slot = 0, migrating_slot = 0, importing_slot = 0, missing_keys = 0;
/* Allow any key to be set if a module disabled cluster redirections. */
if (server.cluster_module_flags & CLUSTER_MODULE_FLAG_NO_REDIRECTION)
return myself;
/* Set error code optimistically for the base case. */
if (error_code) *error_code = CLUSTER_REDIR_NONE;
/* Modules can turn off Redis Cluster redirection: this is useful
* when writing a module that implements a completely different
* distributed system. */
/* We handle all the cases as if they were EXEC commands, so we have
* a common code path for everything */
if (cmd->proc == execCommand) {
......@@ -5585,7 +5625,7 @@ void clusterRedirectClient(client *c, clusterNode *n, int hashslot, int error_co
if (error_code == CLUSTER_REDIR_CROSS_SLOT) {
addReplySds(c,sdsnew("-CROSSSLOT Keys in request don't hash to the same slot\r\n"));
} else if (error_code == CLUSTER_REDIR_UNSTABLE) {
/* The request spawns mutliple keys in the same slot,
/* The request spawns multiple keys in the same slot,
* but the slot is not "stable" currently as there is
* a migration or import in progress. */
addReplySds(c,sdsnew("-TRYAGAIN Multiple keys request during rehashing of slot\r\n"));
......
......@@ -100,6 +100,13 @@ typedef struct clusterLink {
#define CLUSTERMSG_TYPE_MODULE 9 /* Module cluster API message. */
#define CLUSTERMSG_TYPE_COUNT 10 /* Total number of message types. */
/* Flags that a module can set in order to prevent certain Redis Cluster
* features to be enabled. Useful when implementing a different distributed
* system on top of Redis Cluster message bus, using modules. */
#define CLUSTER_MODULE_FLAG_NONE 0
#define CLUSTER_MODULE_FLAG_NO_FAILOVER (1<<1)
#define CLUSTER_MODULE_FLAG_NO_REDIRECTION (1<<2)
/* This structure represent elements of node->fail_reports. */
typedef struct clusterNodeFailReport {
struct clusterNode *node; /* Node reporting the failure condition. */
......@@ -243,7 +250,7 @@ union clusterMsgData {
#define CLUSTER_PROTO_VER 1 /* Cluster bus protocol version. */
typedef struct {
char sig[4]; /* Siganture "RCmb" (Redis Cluster message bus). */
char sig[4]; /* Signature "RCmb" (Redis Cluster message bus). */
uint32_t totlen; /* Total length of this message */
uint16_t ver; /* Protocol version, currently set to 1. */
uint16_t port; /* TCP base port number. */
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment