Commit e5187ad2 authored by Oran Agra's avatar Oran Agra
Browse files

Merge remote-tracking branch 'oss/unstable' into module_rdb_load_errors

parents 4339706e f42846e8
...@@ -406,7 +406,7 @@ replicas, or to continue the replication after a disconnection. ...@@ -406,7 +406,7 @@ replicas, or to continue the replication after a disconnection.
Other C files Other C files
--- ---
* `t_hash.c`, `t_list.c`, `t_set.c`, `t_string.c` and `t_zset.c` contains the implementation of the Redis data types. They implement both an API to access a given data type, and the client commands implementations for these data types. * `t_hash.c`, `t_list.c`, `t_set.c`, `t_string.c`, `t_zset.c` and `t_stream.c` contains the implementation of the Redis data types. They implement both an API to access a given data type, and the client commands implementations for these data types.
* `ae.c` implements the Redis event loop, it's a self contained library which is simple to read and understand. * `ae.c` implements the Redis event loop, it's a self contained library which is simple to read and understand.
* `sds.c` is the Redis string library, check http://github.com/antirez/sds for more information. * `sds.c` is the Redis string library, check http://github.com/antirez/sds for more information.
* `anet.c` is a library to use POSIX networking in a simpler way compared to the raw interface exposed by the kernel. * `anet.c` is a library to use POSIX networking in a simpler way compared to the raw interface exposed by the kernel.
......
...@@ -336,13 +336,11 @@ replica-read-only yes ...@@ -336,13 +336,11 @@ replica-read-only yes
# Replication SYNC strategy: disk or socket. # Replication SYNC strategy: disk or socket.
# #
# ------------------------------------------------------- # New replicas and reconnecting replicas that are not able to continue the
# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY # replication process just receiving differences, need to do what is called a
# ------------------------------------------------------- # "full synchronization". An RDB file is transmitted from the master to the
# replicas.
# #
# New replicas and reconnecting replicas that are not able to continue the replication
# process just receiving differences, need to do what is called a "full
# synchronization". An RDB file is transmitted from the master to the replicas.
# The transmission can happen in two different ways: # The transmission can happen in two different ways:
# #
# 1) Disk-backed: The Redis master creates a new process that writes the RDB # 1) Disk-backed: The Redis master creates a new process that writes the RDB
...@@ -352,14 +350,14 @@ replica-read-only yes ...@@ -352,14 +350,14 @@ replica-read-only yes
# RDB file to replica sockets, without touching the disk at all. # RDB file to replica sockets, without touching the disk at all.
# #
# With disk-backed replication, while the RDB file is generated, more replicas # With disk-backed replication, while the RDB file is generated, more replicas
# can be queued and served with the RDB file as soon as the current child producing # can be queued and served with the RDB file as soon as the current child
# the RDB file finishes its work. With diskless replication instead once # producing the RDB file finishes its work. With diskless replication instead
# the transfer starts, new replicas arriving will be queued and a new transfer # once the transfer starts, new replicas arriving will be queued and a new
# will start when the current one terminates. # transfer will start when the current one terminates.
# #
# When diskless replication is used, the master waits a configurable amount of # When diskless replication is used, the master waits a configurable amount of
# time (in seconds) before starting the transfer in the hope that multiple replicas # time (in seconds) before starting the transfer in the hope that multiple
# will arrive and the transfer can be parallelized. # replicas will arrive and the transfer can be parallelized.
# #
# With slow disks and fast (large bandwidth) networks, diskless replication # With slow disks and fast (large bandwidth) networks, diskless replication
# works better. # works better.
...@@ -370,22 +368,32 @@ repl-diskless-sync no ...@@ -370,22 +368,32 @@ repl-diskless-sync no
# to the replicas. # to the replicas.
# #
# This is important since once the transfer starts, it is not possible to serve # This is important since once the transfer starts, it is not possible to serve
# new replicas arriving, that will be queued for the next RDB transfer, so the server # new replicas arriving, that will be queued for the next RDB transfer, so the
# waits a delay in order to let more replicas arrive. # server waits a delay in order to let more replicas arrive.
# #
# The delay is specified in seconds, and by default is 5 seconds. To disable # The delay is specified in seconds, and by default is 5 seconds. To disable
# it entirely just set it to 0 seconds and the transfer will start ASAP. # it entirely just set it to 0 seconds and the transfer will start ASAP.
repl-diskless-sync-delay 5 repl-diskless-sync-delay 5
# Replica can load the rdb it reads from the replication link directly from the # -----------------------------------------------------------------------------
# socket, or store the rdb to a file and read that file after it was completely # WARNING: RDB diskless load is experimental. Since in this setup the replica
# does not immediately store an RDB on disk, it may cause data loss during
# failovers. RDB diskless load + Redis modules not handling I/O reads may also
# cause Redis to abort in case of I/O errors during the initial synchronization
# stage with the master. Use only if your do what you are doing.
# -----------------------------------------------------------------------------
#
# Replica can load the RDB it reads from the replication link directly from the
# socket, or store the RDB to a file and read that file after it was completely
# recived from the master. # recived from the master.
#
# In many cases the disk is slower than the network, and storing and loading # In many cases the disk is slower than the network, and storing and loading
# the rdb file may increase replication time (and even increase the master's # the RDB file may increase replication time (and even increase the master's
# Copy on Write memory and salve buffers). # Copy on Write memory and salve buffers).
# However, parsing the rdb file directly from the socket may mean that we have # However, parsing the RDB file directly from the socket may mean that we have
# to flush the contents of the current database before the full rdb was received. # to flush the contents of the current database before the full rdb was
# for this reason we have the following options: # received. For this reason we have the following options:
#
# "disabled" - Don't use diskless load (store the rdb file to the disk first) # "disabled" - Don't use diskless load (store the rdb file to the disk first)
# "on-empty-db" - Use diskless load only when it is completely safe. # "on-empty-db" - Use diskless load only when it is completely safe.
# "swapdb" - Keep a copy of the current db contents in RAM while parsing # "swapdb" - Keep a copy of the current db contents in RAM while parsing
...@@ -393,9 +401,9 @@ repl-diskless-sync-delay 5 ...@@ -393,9 +401,9 @@ repl-diskless-sync-delay 5
# sufficient memory, if you don't have it, you risk an OOM kill. # sufficient memory, if you don't have it, you risk an OOM kill.
repl-diskless-load disabled repl-diskless-load disabled
# Replicas send PINGs to server in a predefined interval. It's possible to change # Replicas send PINGs to server in a predefined interval. It's possible to
# this interval with the repl_ping_replica_period option. The default value is 10 # change this interval with the repl_ping_replica_period option. The default
# seconds. # value is 10 seconds.
# #
# repl-ping-replica-period 10 # repl-ping-replica-period 10
...@@ -427,10 +435,10 @@ repl-diskless-load disabled ...@@ -427,10 +435,10 @@ repl-diskless-load disabled
repl-disable-tcp-nodelay no repl-disable-tcp-nodelay no
# Set the replication backlog size. The backlog is a buffer that accumulates # Set the replication backlog size. The backlog is a buffer that accumulates
# replica data when replicas are disconnected for some time, so that when a replica # replica data when replicas are disconnected for some time, so that when a
# wants to reconnect again, often a full resync is not needed, but a partial # replica wants to reconnect again, often a full resync is not needed, but a
# resync is enough, just passing the portion of data the replica missed while # partial resync is enough, just passing the portion of data the replica
# disconnected. # missed while disconnected.
# #
# The bigger the replication backlog, the longer the time the replica can be # The bigger the replication backlog, the longer the time the replica can be
# disconnected and later be able to perform a partial resynchronization. # disconnected and later be able to perform a partial resynchronization.
...@@ -452,13 +460,13 @@ repl-disable-tcp-nodelay no ...@@ -452,13 +460,13 @@ repl-disable-tcp-nodelay no
# #
# repl-backlog-ttl 3600 # repl-backlog-ttl 3600
# The replica priority is an integer number published by Redis in the INFO output. # The replica priority is an integer number published by Redis in the INFO
# It is used by Redis Sentinel in order to select a replica to promote into a # output. It is used by Redis Sentinel in order to select a replica to promote
# master if the master is no longer working correctly. # into a master if the master is no longer working correctly.
# #
# A replica with a low priority number is considered better for promotion, so # A replica with a low priority number is considered better for promotion, so
# for instance if there are three replicas with priority 10, 100, 25 Sentinel will # for instance if there are three replicas with priority 10, 100, 25 Sentinel
# pick the one with priority 10, that is the lowest. # will pick the one with priority 10, that is the lowest.
# #
# However a special priority of 0 marks the replica as not able to perform the # However a special priority of 0 marks the replica as not able to perform the
# role of master, so a replica with priority of 0 will never be selected by # role of master, so a replica with priority of 0 will never be selected by
...@@ -518,6 +526,39 @@ replica-priority 100 ...@@ -518,6 +526,39 @@ replica-priority 100
# replica-announce-ip 5.5.5.5 # replica-announce-ip 5.5.5.5
# replica-announce-port 1234 # replica-announce-port 1234
############################### KEYS TRACKING #################################
# Redis implements server assisted support for client side caching of values.
# This is implemented using an invalidation table that remembers, using
# 16 millions of slots, what clients may have certain subsets of keys. In turn
# this is used in order to send invalidation messages to clients. Please
# to understand more about the feature check this page:
#
# https://redis.io/topics/client-side-caching
#
# When tracking is enabled for a client, all the read only queries are assumed
# to be cached: this will force Redis to store information in the invalidation
# table. When keys are modified, such information is flushed away, and
# invalidation messages are sent to the clients. However if the workload is
# heavily dominated by reads, Redis could use more and more memory in order
# to track the keys fetched by many clients.
#
# For this reason it is possible to configure a maximum fill value for the
# invalidation table. By default it is set to 10%, and once this limit is
# reached, Redis will start to evict caching slots in the invalidation table
# even if keys are not modified, just to reclaim memory: this will in turn
# force the clients to invalidate the cached values. Basically the table
# maximum fill rate is a trade off between the memory you want to spend server
# side to track information about who cached what, and the ability of clients
# to retain cached objects in memory.
#
# If you set the value to 0, it means there are no limits, and all the 16
# millions of caching slots can be used at the same time. In the "stats"
# INFO section, you can find information about the amount of caching slots
# used at every given moment.
#
# tracking-table-max-fill 10
################################## SECURITY ################################### ################################## SECURITY ###################################
# Warning: since Redis is pretty fast an outside user can try up to # Warning: since Redis is pretty fast an outside user can try up to
...@@ -747,17 +788,17 @@ replica-priority 100 ...@@ -747,17 +788,17 @@ replica-priority 100
# DEL commands to the replica as keys evict in the master side. # DEL commands to the replica as keys evict in the master side.
# #
# This behavior ensures that masters and replicas stay consistent, and is usually # This behavior ensures that masters and replicas stay consistent, and is usually
# what you want, however if your replica is writable, or you want the replica to have # what you want, however if your replica is writable, or you want the replica
# a different memory setting, and you are sure all the writes performed to the # to have a different memory setting, and you are sure all the writes performed
# replica are idempotent, then you may change this default (but be sure to understand # to the replica are idempotent, then you may change this default (but be sure
# what you are doing). # to understand what you are doing).
# #
# Note that since the replica by default does not evict, it may end using more # Note that since the replica by default does not evict, it may end using more
# memory than the one set via maxmemory (there are certain buffers that may # memory than the one set via maxmemory (there are certain buffers that may
# be larger on the replica, or data structures may sometimes take more memory and so # be larger on the replica, or data structures may sometimes take more memory
# forth). So make sure you monitor your replicas and make sure they have enough # and so forth). So make sure you monitor your replicas and make sure they
# memory to never hit a real out-of-memory condition before the master hits # have enough memory to never hit a real out-of-memory condition before the
# the configured maxmemory setting. # master hits the configured maxmemory setting.
# #
# replica-ignore-maxmemory yes # replica-ignore-maxmemory yes
......
...@@ -13,4 +13,4 @@ then ...@@ -13,4 +13,4 @@ then
fi fi
make -C tests/modules && \ make -C tests/modules && \
$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter "${@}" $TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/testrdb "${@}"
...@@ -303,9 +303,7 @@ ssize_t aofWrite(int fd, const char *buf, size_t len) { ...@@ -303,9 +303,7 @@ ssize_t aofWrite(int fd, const char *buf, size_t len) {
nwritten = write(fd, buf, len); nwritten = write(fd, buf, len);
if (nwritten < 0) { if (nwritten < 0) {
if (errno == EINTR) { if (errno == EINTR) continue;
continue;
}
return totwritten ? totwritten : -1; return totwritten ? totwritten : -1;
} }
......
...@@ -4251,12 +4251,7 @@ NULL ...@@ -4251,12 +4251,7 @@ NULL
} }
} else if (!strcasecmp(c->argv[1]->ptr,"nodes") && c->argc == 2) { } else if (!strcasecmp(c->argv[1]->ptr,"nodes") && c->argc == 2) {
/* CLUSTER NODES */ /* CLUSTER NODES */
robj *o; addReplyBulkSds(c,clusterGenNodesDescription(0));
sds ci = clusterGenNodesDescription(0);
o = createObject(OBJ_STRING,ci);
addReplyBulk(c,o);
decrRefCount(o);
} else if (!strcasecmp(c->argv[1]->ptr,"myid") && c->argc == 2) { } else if (!strcasecmp(c->argv[1]->ptr,"myid") && c->argc == 2) {
/* CLUSTER MYID */ /* CLUSTER MYID */
addReplyBulkCBuffer(c,myself->name, CLUSTER_NAMELEN); addReplyBulkCBuffer(c,myself->name, CLUSTER_NAMELEN);
...@@ -4832,7 +4827,7 @@ int verifyDumpPayload(unsigned char *p, size_t len) { ...@@ -4832,7 +4827,7 @@ int verifyDumpPayload(unsigned char *p, size_t len) {
* DUMP is actually not used by Redis Cluster but it is the obvious * DUMP is actually not used by Redis Cluster but it is the obvious
* complement of RESTORE and can be useful for different applications. */ * complement of RESTORE and can be useful for different applications. */
void dumpCommand(client *c) { void dumpCommand(client *c) {
robj *o, *dumpobj; robj *o;
rio payload; rio payload;
/* Check if the key is here. */ /* Check if the key is here. */
...@@ -4845,9 +4840,7 @@ void dumpCommand(client *c) { ...@@ -4845,9 +4840,7 @@ void dumpCommand(client *c) {
createDumpPayload(&payload,o,c->argv[1]); createDumpPayload(&payload,o,c->argv[1]);
/* Transfer to the client */ /* Transfer to the client */
dumpobj = createObject(OBJ_STRING,payload.io.buffer.ptr); addReplyBulkSds(c,payload.io.buffer.ptr);
addReplyBulk(c,dumpobj);
decrRefCount(dumpobj);
return; return;
} }
......
...@@ -686,6 +686,17 @@ void loadServerConfigFromString(char *config) { ...@@ -686,6 +686,17 @@ void loadServerConfigFromString(char *config) {
} }
} else if (!strcasecmp(argv[0],"slowlog-max-len") && argc == 2) { } else if (!strcasecmp(argv[0],"slowlog-max-len") && argc == 2) {
server.slowlog_max_len = strtoll(argv[1],NULL,10); server.slowlog_max_len = strtoll(argv[1],NULL,10);
} else if (!strcasecmp(argv[0],"tracking-table-max-fill") &&
argc == 2)
{
server.tracking_table_max_fill = strtoll(argv[1],NULL,10);
if (server.tracking_table_max_fill > 100 ||
server.tracking_table_max_fill < 0)
{
err = "The tracking table fill percentage must be an "
"integer between 0 and 100";
goto loaderr;
}
} else if (!strcasecmp(argv[0],"client-output-buffer-limit") && } else if (!strcasecmp(argv[0],"client-output-buffer-limit") &&
argc == 5) argc == 5)
{ {
...@@ -1133,6 +1144,8 @@ void configSetCommand(client *c) { ...@@ -1133,6 +1144,8 @@ void configSetCommand(client *c) {
"slowlog-max-len",ll,0,LONG_MAX) { "slowlog-max-len",ll,0,LONG_MAX) {
/* Cast to unsigned. */ /* Cast to unsigned. */
server.slowlog_max_len = (unsigned long)ll; server.slowlog_max_len = (unsigned long)ll;
} config_set_numerical_field(
"tracking-table-max-fill",server.tracking_table_max_fill,0,100) {
} config_set_numerical_field( } config_set_numerical_field(
"latency-monitor-threshold",server.latency_monitor_threshold,0,LLONG_MAX){ "latency-monitor-threshold",server.latency_monitor_threshold,0,LLONG_MAX){
} config_set_numerical_field( } config_set_numerical_field(
...@@ -1338,8 +1351,8 @@ void configGetCommand(client *c) { ...@@ -1338,8 +1351,8 @@ void configGetCommand(client *c) {
server.slowlog_log_slower_than); server.slowlog_log_slower_than);
config_get_numerical_field("latency-monitor-threshold", config_get_numerical_field("latency-monitor-threshold",
server.latency_monitor_threshold); server.latency_monitor_threshold);
config_get_numerical_field("slowlog-max-len", config_get_numerical_field("slowlog-max-len", server.slowlog_max_len);
server.slowlog_max_len); config_get_numerical_field("tracking-table-max-fill", server.tracking_table_max_fill);
config_get_numerical_field("port",server.port); config_get_numerical_field("port",server.port);
config_get_numerical_field("cluster-announce-port",server.cluster_announce_port); config_get_numerical_field("cluster-announce-port",server.cluster_announce_port);
config_get_numerical_field("cluster-announce-bus-port",server.cluster_announce_bus_port); config_get_numerical_field("cluster-announce-bus-port",server.cluster_announce_bus_port);
...@@ -1470,12 +1483,10 @@ void configGetCommand(client *c) { ...@@ -1470,12 +1483,10 @@ void configGetCommand(client *c) {
matches++; matches++;
} }
if (stringmatch(pattern,"notify-keyspace-events",1)) { if (stringmatch(pattern,"notify-keyspace-events",1)) {
robj *flagsobj = createObject(OBJ_STRING, sds flags = keyspaceEventsFlagsToString(server.notify_keyspace_events);
keyspaceEventsFlagsToString(server.notify_keyspace_events));
addReplyBulkCString(c,"notify-keyspace-events"); addReplyBulkCString(c,"notify-keyspace-events");
addReplyBulk(c,flagsobj); addReplyBulkSds(c,flags);
decrRefCount(flagsobj);
matches++; matches++;
} }
if (stringmatch(pattern,"bind",1)) { if (stringmatch(pattern,"bind",1)) {
...@@ -2167,6 +2178,7 @@ int rewriteConfig(char *path) { ...@@ -2167,6 +2178,7 @@ int rewriteConfig(char *path) {
rewriteConfigNumericalOption(state,"slowlog-log-slower-than",server.slowlog_log_slower_than,CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN); rewriteConfigNumericalOption(state,"slowlog-log-slower-than",server.slowlog_log_slower_than,CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN);
rewriteConfigNumericalOption(state,"latency-monitor-threshold",server.latency_monitor_threshold,CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD); rewriteConfigNumericalOption(state,"latency-monitor-threshold",server.latency_monitor_threshold,CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD);
rewriteConfigNumericalOption(state,"slowlog-max-len",server.slowlog_max_len,CONFIG_DEFAULT_SLOWLOG_MAX_LEN); rewriteConfigNumericalOption(state,"slowlog-max-len",server.slowlog_max_len,CONFIG_DEFAULT_SLOWLOG_MAX_LEN);
rewriteConfigNumericalOption(state,"tracking-table-max-fill",server.tracking_table_max_fill,CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL);
rewriteConfigNotifykeyspaceeventsOption(state); rewriteConfigNotifykeyspaceeventsOption(state);
rewriteConfigNumericalOption(state,"hash-max-ziplist-entries",server.hash_max_ziplist_entries,OBJ_HASH_MAX_ZIPLIST_ENTRIES); rewriteConfigNumericalOption(state,"hash-max-ziplist-entries",server.hash_max_ziplist_entries,OBJ_HASH_MAX_ZIPLIST_ENTRIES);
rewriteConfigNumericalOption(state,"hash-max-ziplist-value",server.hash_max_ziplist_value,OBJ_HASH_MAX_ZIPLIST_VALUE); rewriteConfigNumericalOption(state,"hash-max-ziplist-value",server.hash_max_ziplist_value,OBJ_HASH_MAX_ZIPLIST_VALUE);
......
...@@ -353,6 +353,11 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)( ...@@ -353,6 +353,11 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(
return -1; return -1;
} }
/* Make sure the WATCHed keys are affected by the FLUSH* commands.
* Note that we need to call the function while the keys are still
* there. */
signalFlushedDb(dbnum);
int startdb, enddb; int startdb, enddb;
if (dbnum == -1) { if (dbnum == -1) {
startdb = 0; startdb = 0;
...@@ -412,11 +417,12 @@ long long dbTotalServerKeyCount() { ...@@ -412,11 +417,12 @@ long long dbTotalServerKeyCount() {
void signalModifiedKey(redisDb *db, robj *key) { void signalModifiedKey(redisDb *db, robj *key) {
touchWatchedKey(db,key); touchWatchedKey(db,key);
if (server.tracking_clients) trackingInvalidateKey(key); trackingInvalidateKey(key);
} }
void signalFlushedDb(int dbid) { void signalFlushedDb(int dbid) {
touchWatchedKeysOnFlush(dbid); touchWatchedKeysOnFlush(dbid);
trackingInvalidateKeysOnFlush(dbid);
} }
/*----------------------------------------------------------------------------- /*-----------------------------------------------------------------------------
...@@ -452,7 +458,6 @@ void flushdbCommand(client *c) { ...@@ -452,7 +458,6 @@ void flushdbCommand(client *c) {
int flags; int flags;
if (getFlushCommandFlags(c,&flags) == C_ERR) return; if (getFlushCommandFlags(c,&flags) == C_ERR) return;
signalFlushedDb(c->db->id);
server.dirty += emptyDb(c->db->id,flags,NULL); server.dirty += emptyDb(c->db->id,flags,NULL);
addReply(c,shared.ok); addReply(c,shared.ok);
} }
...@@ -464,7 +469,6 @@ void flushallCommand(client *c) { ...@@ -464,7 +469,6 @@ void flushallCommand(client *c) {
int flags; int flags;
if (getFlushCommandFlags(c,&flags) == C_ERR) return; if (getFlushCommandFlags(c,&flags) == C_ERR) return;
signalFlushedDb(-1);
server.dirty += emptyDb(-1,flags,NULL); server.dirty += emptyDb(-1,flags,NULL);
addReply(c,shared.ok); addReply(c,shared.ok);
if (server.rdb_child_pid != -1) killRDBChild(); if (server.rdb_child_pid != -1) killRDBChild();
......
...@@ -64,7 +64,7 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) { ...@@ -64,7 +64,7 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) {
dbSyncDelete(db,keyobj); dbSyncDelete(db,keyobj);
notifyKeyspaceEvent(NOTIFY_EXPIRED, notifyKeyspaceEvent(NOTIFY_EXPIRED,
"expired",keyobj,db->id); "expired",keyobj,db->id);
if (server.tracking_clients) trackingInvalidateKey(keyobj); trackingInvalidateKey(keyobj);
decrRefCount(keyobj); decrRefCount(keyobj);
server.stat_expiredkeys++; server.stat_expiredkeys++;
return 1; return 1;
......
...@@ -701,6 +701,7 @@ int hllSparseSet(robj *o, long index, uint8_t count) { ...@@ -701,6 +701,7 @@ int hllSparseSet(robj *o, long index, uint8_t count) {
first += span; first += span;
} }
if (span == 0) return -1; /* Invalid format. */ if (span == 0) return -1; /* Invalid format. */
if (span >= end) return -1; /* Invalid format. */
next = HLL_SPARSE_IS_XZERO(p) ? p+2 : p+1; next = HLL_SPARSE_IS_XZERO(p) ? p+2 : p+1;
if (next >= end) next = NULL; if (next >= end) next = NULL;
......
...@@ -29,6 +29,7 @@ ...@@ -29,6 +29,7 @@
#include "server.h" #include "server.h"
#include "cluster.h" #include "cluster.h"
#include "rdb.h"
#include <dlfcn.h> #include <dlfcn.h>
#define REDISMODULE_CORE 1 #define REDISMODULE_CORE 1
...@@ -3092,6 +3093,11 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver, ...@@ -3092,6 +3093,11 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver,
moduleTypeMemUsageFunc mem_usage; moduleTypeMemUsageFunc mem_usage;
moduleTypeDigestFunc digest; moduleTypeDigestFunc digest;
moduleTypeFreeFunc free; moduleTypeFreeFunc free;
struct {
moduleTypeAuxLoadFunc aux_load;
moduleTypeAuxSaveFunc aux_save;
int aux_save_triggers;
} v2;
} *tms = (struct typemethods*) typemethods_ptr; } *tms = (struct typemethods*) typemethods_ptr;
moduleType *mt = zcalloc(sizeof(*mt)); moduleType *mt = zcalloc(sizeof(*mt));
...@@ -3103,6 +3109,11 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver, ...@@ -3103,6 +3109,11 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver,
mt->mem_usage = tms->mem_usage; mt->mem_usage = tms->mem_usage;
mt->digest = tms->digest; mt->digest = tms->digest;
mt->free = tms->free; mt->free = tms->free;
if (tms->version >= 2) {
mt->aux_load = tms->v2.aux_load;
mt->aux_save = tms->v2.aux_save;
mt->aux_save_triggers = tms->v2.aux_save_triggers;
}
memcpy(mt->name,name,sizeof(mt->name)); memcpy(mt->name,name,sizeof(mt->name));
listAddNodeTail(ctx->module->types,mt); listAddNodeTail(ctx->module->types,mt);
return mt; return mt;
...@@ -3405,6 +3416,36 @@ loaderr: ...@@ -3405,6 +3416,36 @@ loaderr:
return 0; return 0;
} }
/* Iterate over modules, and trigger rdb aux saving for the ones modules types
* who asked for it. */
ssize_t rdbSaveModulesAux(rio *rdb, int when) {
size_t total_written = 0;
dictIterator *di = dictGetIterator(modules);
dictEntry *de;
while ((de = dictNext(di)) != NULL) {
struct RedisModule *module = dictGetVal(de);
listIter li;
listNode *ln;
listRewind(module->types,&li);
while((ln = listNext(&li))) {
moduleType *mt = ln->value;
if (!mt->aux_save || !(mt->aux_save_triggers & when))
continue;
ssize_t ret = rdbSaveSingleModuleAux(rdb, when, mt);
if (ret==-1) {
dictReleaseIterator(di);
return -1;
}
total_written += ret;
}
}
dictReleaseIterator(di);
return total_written;
}
/* -------------------------------------------------------------------------- /* --------------------------------------------------------------------------
* Key digest API (DEBUG DIGEST interface for modules types) * Key digest API (DEBUG DIGEST interface for modules types)
* -------------------------------------------------------------------------- */ * -------------------------------------------------------------------------- */
...@@ -3565,7 +3606,7 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li ...@@ -3565,7 +3606,7 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li
if (level < server.verbosity) return; if (level < server.verbosity) return;
name_len = snprintf(msg, sizeof(msg),"<%s> ", module->name); name_len = snprintf(msg, sizeof(msg),"<%s> ", module? module->name: "module");
vsnprintf(msg + name_len, sizeof(msg) - name_len, fmt, ap); vsnprintf(msg + name_len, sizeof(msg) - name_len, fmt, ap);
serverLogRaw(level,msg); serverLogRaw(level,msg);
} }
...@@ -3583,13 +3624,15 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li ...@@ -3583,13 +3624,15 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li
* There is a fixed limit to the length of the log line this function is able * There is a fixed limit to the length of the log line this function is able
* to emit, this limit is not specified but is guaranteed to be more than * to emit, this limit is not specified but is guaranteed to be more than
* a few lines of text. * a few lines of text.
*
* The ctx argument may be NULL if cannot be provided in the context of the
* caller for instance threads or callbacks, in which case a generic "module"
* will be used instead of the module name.
*/ */
void RM_Log(RedisModuleCtx *ctx, const char *levelstr, const char *fmt, ...) { void RM_Log(RedisModuleCtx *ctx, const char *levelstr, const char *fmt, ...) {
if (!ctx->module) return; /* Can only log if module is initialized */
va_list ap; va_list ap;
va_start(ap, fmt); va_start(ap, fmt);
RM_LogRaw(ctx->module,levelstr,fmt,ap); RM_LogRaw(ctx? ctx->module: NULL,levelstr,fmt,ap);
va_end(ap); va_end(ap);
} }
......
This diff is collapsed.
...@@ -145,6 +145,7 @@ size_t rdbSavedObjectLen(robj *o); ...@@ -145,6 +145,7 @@ size_t rdbSavedObjectLen(robj *o);
robj *rdbLoadObject(int type, rio *rdb, robj *key); robj *rdbLoadObject(int type, rio *rdb, robj *key);
void backgroundSaveDoneHandler(int exitcode, int bysignal); void backgroundSaveDoneHandler(int exitcode, int bysignal);
int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime); int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime);
ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt);
robj *rdbLoadStringObject(rio *rdb); robj *rdbLoadStringObject(rio *rdb);
ssize_t rdbSaveStringObject(rio *rdb, robj *obj); ssize_t rdbSaveStringObject(rio *rdb, robj *obj);
ssize_t rdbSaveRawString(rio *rdb, unsigned char *s, size_t len); ssize_t rdbSaveRawString(rio *rdb, unsigned char *s, size_t len);
......
...@@ -104,6 +104,7 @@ static struct config { ...@@ -104,6 +104,7 @@ static struct config {
int is_fetching_slots; int is_fetching_slots;
int is_updating_slots; int is_updating_slots;
int slots_last_update; int slots_last_update;
int enable_tracking;
/* Thread mutexes to be used as fallbacks by atomicvar.h */ /* Thread mutexes to be used as fallbacks by atomicvar.h */
pthread_mutex_t requests_issued_mutex; pthread_mutex_t requests_issued_mutex;
pthread_mutex_t requests_finished_mutex; pthread_mutex_t requests_finished_mutex;
...@@ -255,7 +256,7 @@ static redisConfig *getRedisConfig(const char *ip, int port, ...@@ -255,7 +256,7 @@ static redisConfig *getRedisConfig(const char *ip, int port,
goto fail; goto fail;
} }
if(config.auth){ if(config.auth) {
void *authReply = NULL; void *authReply = NULL;
redisAppendCommand(c, "AUTH %s", config.auth); redisAppendCommand(c, "AUTH %s", config.auth);
if (REDIS_OK != redisGetReply(c, &authReply)) goto fail; if (REDIS_OK != redisGetReply(c, &authReply)) goto fail;
...@@ -633,6 +634,14 @@ static client createClient(char *cmd, size_t len, client from, int thread_id) { ...@@ -633,6 +634,14 @@ static client createClient(char *cmd, size_t len, client from, int thread_id) {
c->prefix_pending++; c->prefix_pending++;
} }
if (config.enable_tracking) {
char *buf = NULL;
int len = redisFormatCommand(&buf, "CLIENT TRACKING on");
c->obuf = sdscatlen(c->obuf, buf, len);
free(buf);
c->prefix_pending++;
}
/* If a DB number different than zero is selected, prefix our request /* If a DB number different than zero is selected, prefix our request
* buffer with the SELECT command, that will be discarded the first * buffer with the SELECT command, that will be discarded the first
* time the replies are received, so if the client is reused the * time the replies are received, so if the client is reused the
...@@ -1350,6 +1359,8 @@ int parseOptions(int argc, const char **argv) { ...@@ -1350,6 +1359,8 @@ int parseOptions(int argc, const char **argv) {
} else if (config.num_threads < 0) config.num_threads = 0; } else if (config.num_threads < 0) config.num_threads = 0;
} else if (!strcmp(argv[i],"--cluster")) { } else if (!strcmp(argv[i],"--cluster")) {
config.cluster_mode = 1; config.cluster_mode = 1;
} else if (!strcmp(argv[i],"--enable-tracking")) {
config.enable_tracking = 1;
} else if (!strcmp(argv[i],"--help")) { } else if (!strcmp(argv[i],"--help")) {
exit_status = 0; exit_status = 0;
goto usage; goto usage;
...@@ -1380,6 +1391,7 @@ usage: ...@@ -1380,6 +1391,7 @@ usage:
" --dbnum <db> SELECT the specified db number (default 0)\n" " --dbnum <db> SELECT the specified db number (default 0)\n"
" --threads <num> Enable multi-thread mode.\n" " --threads <num> Enable multi-thread mode.\n"
" --cluster Enable cluster mode.\n" " --cluster Enable cluster mode.\n"
" --enable-tracking Send CLIENT TRACKING on before starting benchmark.\n"
" -k <boolean> 1=keep alive 0=reconnect (default 1)\n" " -k <boolean> 1=keep alive 0=reconnect (default 1)\n"
" -r <keyspacelen> Use random keys for SET/GET/INCR, random values for SADD\n" " -r <keyspacelen> Use random keys for SET/GET/INCR, random values for SADD\n"
" Using this option the benchmark will expand the string __rand_int__\n" " Using this option the benchmark will expand the string __rand_int__\n"
...@@ -1504,6 +1516,7 @@ int main(int argc, const char **argv) { ...@@ -1504,6 +1516,7 @@ int main(int argc, const char **argv) {
config.is_fetching_slots = 0; config.is_fetching_slots = 0;
config.is_updating_slots = 0; config.is_updating_slots = 0;
config.slots_last_update = 0; config.slots_last_update = 0;
config.enable_tracking = 0;
i = parseOptions(argc,argv); i = parseOptions(argc,argv);
argc -= i; argc -= i;
......
...@@ -216,14 +216,16 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { ...@@ -216,14 +216,16 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) {
/* EXPIRETIME: load an expire associated with the next key /* EXPIRETIME: load an expire associated with the next key
* to load. Note that after loading an expire we need to * to load. Note that after loading an expire we need to
* load the actual type, and continue. */ * load the actual type, and continue. */
if ((expiretime = rdbLoadTime(&rdb)) == -1) goto eoferr; expiretime = rdbLoadTime(&rdb);
expiretime *= 1000; expiretime *= 1000;
if (rioGetReadError(&rdb)) goto eoferr;
continue; /* Read next opcode. */ continue; /* Read next opcode. */
} else if (type == RDB_OPCODE_EXPIRETIME_MS) { } else if (type == RDB_OPCODE_EXPIRETIME_MS) {
/* EXPIRETIME_MS: milliseconds precision expire times introduced /* EXPIRETIME_MS: milliseconds precision expire times introduced
* with RDB v3. Like EXPIRETIME but no with more precision. */ * with RDB v3. Like EXPIRETIME but no with more precision. */
rdbstate.doing = RDB_CHECK_DOING_READ_EXPIRE; rdbstate.doing = RDB_CHECK_DOING_READ_EXPIRE;
if ((expiretime = rdbLoadMillisecondTime(&rdb, rdbver)) == -1) goto eoferr; expiretime = rdbLoadMillisecondTime(&rdb, rdbver);
if (rioGetReadError(&rdb)) goto eoferr;
continue; /* Read next opcode. */ continue; /* Read next opcode. */
} else if (type == RDB_OPCODE_FREQ) { } else if (type == RDB_OPCODE_FREQ) {
/* FREQ: LFU frequency. */ /* FREQ: LFU frequency. */
......
...@@ -129,6 +129,10 @@ ...@@ -129,6 +129,10 @@
#define REDISMODULE_NOT_USED(V) ((void) V) #define REDISMODULE_NOT_USED(V) ((void) V)
/* Bit flags for aux_save_triggers and the aux_load and aux_save callbacks */
#define REDISMODULE_AUX_BEFORE_RDB (1<<0)
#define REDISMODULE_AUX_AFTER_RDB (1<<1)
/* This type represents a timer handle, and is returned when a timer is /* This type represents a timer handle, and is returned when a timer is
* registered and used in order to invalidate a timer. It's just a 64 bit * registered and used in order to invalidate a timer. It's just a 64 bit
* number, because this is how each timer is represented inside the radix tree * number, because this is how each timer is represented inside the radix tree
...@@ -169,6 +173,8 @@ typedef void (*RedisModuleDisconnectFunc)(RedisModuleCtx *ctx, RedisModuleBlocke ...@@ -169,6 +173,8 @@ typedef void (*RedisModuleDisconnectFunc)(RedisModuleCtx *ctx, RedisModuleBlocke
typedef int (*RedisModuleNotificationFunc)(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key); typedef int (*RedisModuleNotificationFunc)(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key);
typedef void *(*RedisModuleTypeLoadFunc)(RedisModuleIO *rdb, int encver); typedef void *(*RedisModuleTypeLoadFunc)(RedisModuleIO *rdb, int encver);
typedef void (*RedisModuleTypeSaveFunc)(RedisModuleIO *rdb, void *value); typedef void (*RedisModuleTypeSaveFunc)(RedisModuleIO *rdb, void *value);
typedef int (*RedisModuleTypeAuxLoadFunc)(RedisModuleIO *rdb, int encver, int when);
typedef void (*RedisModuleTypeAuxSaveFunc)(RedisModuleIO *rdb, int when);
typedef void (*RedisModuleTypeRewriteFunc)(RedisModuleIO *aof, RedisModuleString *key, void *value); typedef void (*RedisModuleTypeRewriteFunc)(RedisModuleIO *aof, RedisModuleString *key, void *value);
typedef size_t (*RedisModuleTypeMemUsageFunc)(const void *value); typedef size_t (*RedisModuleTypeMemUsageFunc)(const void *value);
typedef void (*RedisModuleTypeDigestFunc)(RedisModuleDigest *digest, void *value); typedef void (*RedisModuleTypeDigestFunc)(RedisModuleDigest *digest, void *value);
...@@ -177,7 +183,7 @@ typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const cha ...@@ -177,7 +183,7 @@ typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const cha
typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data); typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data);
typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter); typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter);
#define REDISMODULE_TYPE_METHOD_VERSION 1 #define REDISMODULE_TYPE_METHOD_VERSION 2
typedef struct RedisModuleTypeMethods { typedef struct RedisModuleTypeMethods {
uint64_t version; uint64_t version;
RedisModuleTypeLoadFunc rdb_load; RedisModuleTypeLoadFunc rdb_load;
...@@ -186,6 +192,9 @@ typedef struct RedisModuleTypeMethods { ...@@ -186,6 +192,9 @@ typedef struct RedisModuleTypeMethods {
RedisModuleTypeMemUsageFunc mem_usage; RedisModuleTypeMemUsageFunc mem_usage;
RedisModuleTypeDigestFunc digest; RedisModuleTypeDigestFunc digest;
RedisModuleTypeFreeFunc free; RedisModuleTypeFreeFunc free;
RedisModuleTypeAuxLoadFunc aux_load;
RedisModuleTypeAuxSaveFunc aux_save;
int aux_save_triggers;
} RedisModuleTypeMethods; } RedisModuleTypeMethods;
#define REDISMODULE_GET_API(name) \ #define REDISMODULE_GET_API(name) \
......
...@@ -92,6 +92,7 @@ static const rio rioBufferIO = { ...@@ -92,6 +92,7 @@ static const rio rioBufferIO = {
rioBufferFlush, rioBufferFlush,
NULL, /* update_checksum */ NULL, /* update_checksum */
0, /* current checksum */ 0, /* current checksum */
0, /* flags */
0, /* bytes read or written */ 0, /* bytes read or written */
0, /* read/write chunk size */ 0, /* read/write chunk size */
{ { NULL, 0 } } /* union for io-specific vars */ { { NULL, 0 } } /* union for io-specific vars */
...@@ -145,6 +146,7 @@ static const rio rioFileIO = { ...@@ -145,6 +146,7 @@ static const rio rioFileIO = {
rioFileFlush, rioFileFlush,
NULL, /* update_checksum */ NULL, /* update_checksum */
0, /* current checksum */ 0, /* current checksum */
0, /* flags */
0, /* bytes read or written */ 0, /* bytes read or written */
0, /* read/write chunk size */ 0, /* read/write chunk size */
{ { NULL, 0 } } /* union for io-specific vars */ { { NULL, 0 } } /* union for io-specific vars */
...@@ -239,6 +241,7 @@ static const rio rioFdIO = { ...@@ -239,6 +241,7 @@ static const rio rioFdIO = {
rioFdFlush, rioFdFlush,
NULL, /* update_checksum */ NULL, /* update_checksum */
0, /* current checksum */ 0, /* current checksum */
0, /* flags */
0, /* bytes read or written */ 0, /* bytes read or written */
0, /* read/write chunk size */ 0, /* read/write chunk size */
{ { NULL, 0 } } /* union for io-specific vars */ { { NULL, 0 } } /* union for io-specific vars */
...@@ -374,6 +377,7 @@ static const rio rioFdsetIO = { ...@@ -374,6 +377,7 @@ static const rio rioFdsetIO = {
rioFdsetFlush, rioFdsetFlush,
NULL, /* update_checksum */ NULL, /* update_checksum */
0, /* current checksum */ 0, /* current checksum */
0, /* flags */
0, /* bytes read or written */ 0, /* bytes read or written */
0, /* read/write chunk size */ 0, /* read/write chunk size */
{ { NULL, 0 } } /* union for io-specific vars */ { { NULL, 0 } } /* union for io-specific vars */
......
/* /*
* Copyright (c) 2009-2012, Pieter Noordhuis <pcnoordhuis at gmail dot com> * Copyright (c) 2009-2012, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com> * Copyright (c) 2009-2019, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
...@@ -36,6 +36,9 @@ ...@@ -36,6 +36,9 @@
#include <stdint.h> #include <stdint.h>
#include "sds.h" #include "sds.h"
#define RIO_FLAG_READ_ERROR (1<<0)
#define RIO_FLAG_WRITE_ERROR (1<<1)
struct _rio { struct _rio {
/* Backend functions. /* Backend functions.
* Since this functions do not tolerate short writes or reads the return * Since this functions do not tolerate short writes or reads the return
...@@ -51,8 +54,8 @@ struct _rio { ...@@ -51,8 +54,8 @@ struct _rio {
* computation. */ * computation. */
void (*update_cksum)(struct _rio *, const void *buf, size_t len); void (*update_cksum)(struct _rio *, const void *buf, size_t len);
/* The current checksum */ /* The current checksum and flags (see RIO_FLAG_*) */
uint64_t cksum; uint64_t cksum, flags;
/* number of bytes read or written */ /* number of bytes read or written */
size_t processed_bytes; size_t processed_bytes;
...@@ -99,11 +102,14 @@ typedef struct _rio rio; ...@@ -99,11 +102,14 @@ typedef struct _rio rio;
* if needed. */ * if needed. */
static inline size_t rioWrite(rio *r, const void *buf, size_t len) { static inline size_t rioWrite(rio *r, const void *buf, size_t len) {
if (r->flags & RIO_FLAG_WRITE_ERROR) return 0;
while (len) { while (len) {
size_t bytes_to_write = (r->max_processing_chunk && r->max_processing_chunk < len) ? r->max_processing_chunk : len; size_t bytes_to_write = (r->max_processing_chunk && r->max_processing_chunk < len) ? r->max_processing_chunk : len;
if (r->update_cksum) r->update_cksum(r,buf,bytes_to_write); if (r->update_cksum) r->update_cksum(r,buf,bytes_to_write);
if (r->write(r,buf,bytes_to_write) == 0) if (r->write(r,buf,bytes_to_write) == 0) {
r->flags |= RIO_FLAG_WRITE_ERROR;
return 0; return 0;
}
buf = (char*)buf + bytes_to_write; buf = (char*)buf + bytes_to_write;
len -= bytes_to_write; len -= bytes_to_write;
r->processed_bytes += bytes_to_write; r->processed_bytes += bytes_to_write;
...@@ -112,10 +118,13 @@ static inline size_t rioWrite(rio *r, const void *buf, size_t len) { ...@@ -112,10 +118,13 @@ static inline size_t rioWrite(rio *r, const void *buf, size_t len) {
} }
static inline size_t rioRead(rio *r, void *buf, size_t len) { static inline size_t rioRead(rio *r, void *buf, size_t len) {
if (r->flags & RIO_FLAG_READ_ERROR) return 0;
while (len) { while (len) {
size_t bytes_to_read = (r->max_processing_chunk && r->max_processing_chunk < len) ? r->max_processing_chunk : len; size_t bytes_to_read = (r->max_processing_chunk && r->max_processing_chunk < len) ? r->max_processing_chunk : len;
if (r->read(r,buf,bytes_to_read) == 0) if (r->read(r,buf,bytes_to_read) == 0) {
r->flags |= RIO_FLAG_READ_ERROR;
return 0; return 0;
}
if (r->update_cksum) r->update_cksum(r,buf,bytes_to_read); if (r->update_cksum) r->update_cksum(r,buf,bytes_to_read);
buf = (char*)buf + bytes_to_read; buf = (char*)buf + bytes_to_read;
len -= bytes_to_read; len -= bytes_to_read;
...@@ -132,6 +141,22 @@ static inline int rioFlush(rio *r) { ...@@ -132,6 +141,22 @@ static inline int rioFlush(rio *r) {
return r->flush(r); return r->flush(r);
} }
/* This function allows to know if there was a read error in any past
* operation, since the rio stream was created or since the last call
* to rioClearError(). */
static inline int rioGetReadError(rio *r) {
return (r->flags & RIO_FLAG_READ_ERROR) != 0;
}
/* Like rioGetReadError() but for write errors. */
static inline int rioGetWriteError(rio *r) {
return (r->flags & RIO_FLAG_READ_ERROR) != 0;
}
static inline void rioClearErrors(rio *r) {
r->flags &= ~(RIO_FLAG_READ_ERROR|RIO_FLAG_WRITE_ERROR);
}
void rioInitWithFile(rio *r, FILE *fp); void rioInitWithFile(rio *r, FILE *fp);
void rioInitWithBuffer(rio *r, sds s); void rioInitWithBuffer(rio *r, sds s);
void rioInitWithFd(rio *r, int fd, size_t read_limit); void rioInitWithFd(rio *r, int fd, size_t read_limit);
......
...@@ -2403,6 +2403,9 @@ void initServerConfig(void) { ...@@ -2403,6 +2403,9 @@ void initServerConfig(void) {
/* Latency monitor */ /* Latency monitor */
server.latency_monitor_threshold = CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD; server.latency_monitor_threshold = CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD;
/* Tracking. */
server.tracking_table_max_fill = CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL;
/* Debugging */ /* Debugging */
server.assert_failed = "<no assertion failed>"; server.assert_failed = "<no assertion failed>";
server.assert_file = "<no file>"; server.assert_file = "<no file>";
...@@ -3401,6 +3404,10 @@ int processCommand(client *c) { ...@@ -3401,6 +3404,10 @@ int processCommand(client *c) {
} }
} }
/* Make sure to use a reasonable amount of memory for client side
* caching metadata. */
if (server.tracking_clients) trackingLimitUsedSlots();
/* Don't accept write commands if there are problems persisting on disk /* Don't accept write commands if there are problems persisting on disk
* and if this is a master instance. */ * and if this is a master instance. */
int deny_write_type = writeCommandsDeniedByDiskError(); int deny_write_type = writeCommandsDeniedByDiskError();
...@@ -4140,7 +4147,8 @@ sds genRedisInfoString(char *section) { ...@@ -4140,7 +4147,8 @@ sds genRedisInfoString(char *section) {
"active_defrag_hits:%lld\r\n" "active_defrag_hits:%lld\r\n"
"active_defrag_misses:%lld\r\n" "active_defrag_misses:%lld\r\n"
"active_defrag_key_hits:%lld\r\n" "active_defrag_key_hits:%lld\r\n"
"active_defrag_key_misses:%lld\r\n", "active_defrag_key_misses:%lld\r\n"
"tracking_used_slots:%lld\r\n",
server.stat_numconnections, server.stat_numconnections,
server.stat_numcommands, server.stat_numcommands,
getInstantaneousMetric(STATS_METRIC_COMMAND), getInstantaneousMetric(STATS_METRIC_COMMAND),
...@@ -4166,7 +4174,8 @@ sds genRedisInfoString(char *section) { ...@@ -4166,7 +4174,8 @@ sds genRedisInfoString(char *section) {
server.stat_active_defrag_hits, server.stat_active_defrag_hits,
server.stat_active_defrag_misses, server.stat_active_defrag_misses,
server.stat_active_defrag_key_hits, server.stat_active_defrag_key_hits,
server.stat_active_defrag_key_misses); server.stat_active_defrag_key_misses,
trackingGetUsedSlots());
} }
/* Replication */ /* Replication */
......
...@@ -171,6 +171,7 @@ typedef long long mstime_t; /* millisecond time type. */ ...@@ -171,6 +171,7 @@ typedef long long mstime_t; /* millisecond time type. */
#define CONFIG_DEFAULT_DEFRAG_CYCLE_MAX 75 /* 75% CPU max (at upper threshold) */ #define CONFIG_DEFAULT_DEFRAG_CYCLE_MAX 75 /* 75% CPU max (at upper threshold) */
#define CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS 1000 /* keys with more than 1000 fields will be processed separately */ #define CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS 1000 /* keys with more than 1000 fields will be processed separately */
#define CONFIG_DEFAULT_PROTO_MAX_BULK_LEN (512ll*1024*1024) /* Bulk request max size */ #define CONFIG_DEFAULT_PROTO_MAX_BULK_LEN (512ll*1024*1024) /* Bulk request max size */
#define CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL 10 /* 10% tracking table max fill. */
#define ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP 20 /* Loopkups per loop. */ #define ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP 20 /* Loopkups per loop. */
#define ACTIVE_EXPIRE_CYCLE_FAST_DURATION 1000 /* Microseconds */ #define ACTIVE_EXPIRE_CYCLE_FAST_DURATION 1000 /* Microseconds */
...@@ -536,6 +537,10 @@ typedef long long mstime_t; /* millisecond time type. */ ...@@ -536,6 +537,10 @@ typedef long long mstime_t; /* millisecond time type. */
#define REDISMODULE_TYPE_ENCVER(id) (id & REDISMODULE_TYPE_ENCVER_MASK) #define REDISMODULE_TYPE_ENCVER(id) (id & REDISMODULE_TYPE_ENCVER_MASK)
#define REDISMODULE_TYPE_SIGN(id) ((id & ~((uint64_t)REDISMODULE_TYPE_ENCVER_MASK)) >>REDISMODULE_TYPE_ENCVER_BITS) #define REDISMODULE_TYPE_SIGN(id) ((id & ~((uint64_t)REDISMODULE_TYPE_ENCVER_MASK)) >>REDISMODULE_TYPE_ENCVER_BITS)
/* Bit flags for moduleTypeAuxSaveFunc */
#define REDISMODULE_AUX_BEFORE_RDB (1<<0)
#define REDISMODULE_AUX_AFTER_RDB (1<<1)
struct RedisModule; struct RedisModule;
struct RedisModuleIO; struct RedisModuleIO;
struct RedisModuleDigest; struct RedisModuleDigest;
...@@ -548,6 +553,8 @@ struct redisObject; ...@@ -548,6 +553,8 @@ struct redisObject;
* is deleted. */ * is deleted. */
typedef void *(*moduleTypeLoadFunc)(struct RedisModuleIO *io, int encver); typedef void *(*moduleTypeLoadFunc)(struct RedisModuleIO *io, int encver);
typedef void (*moduleTypeSaveFunc)(struct RedisModuleIO *io, void *value); typedef void (*moduleTypeSaveFunc)(struct RedisModuleIO *io, void *value);
typedef int (*moduleTypeAuxLoadFunc)(struct RedisModuleIO *rdb, int encver, int when);
typedef void (*moduleTypeAuxSaveFunc)(struct RedisModuleIO *rdb, int when);
typedef void (*moduleTypeRewriteFunc)(struct RedisModuleIO *io, struct redisObject *key, void *value); typedef void (*moduleTypeRewriteFunc)(struct RedisModuleIO *io, struct redisObject *key, void *value);
typedef void (*moduleTypeDigestFunc)(struct RedisModuleDigest *digest, void *value); typedef void (*moduleTypeDigestFunc)(struct RedisModuleDigest *digest, void *value);
typedef size_t (*moduleTypeMemUsageFunc)(const void *value); typedef size_t (*moduleTypeMemUsageFunc)(const void *value);
...@@ -564,6 +571,9 @@ typedef struct RedisModuleType { ...@@ -564,6 +571,9 @@ typedef struct RedisModuleType {
moduleTypeMemUsageFunc mem_usage; moduleTypeMemUsageFunc mem_usage;
moduleTypeDigestFunc digest; moduleTypeDigestFunc digest;
moduleTypeFreeFunc free; moduleTypeFreeFunc free;
moduleTypeAuxLoadFunc aux_load;
moduleTypeAuxSaveFunc aux_save;
int aux_save_triggers;
char name[10]; /* 9 bytes name + null term. Charset: A-Z a-z 0-9 _- */ char name[10]; /* 9 bytes name + null term. Charset: A-Z a-z 0-9 _- */
} moduleType; } moduleType;
...@@ -1316,6 +1326,7 @@ struct redisServer { ...@@ -1316,6 +1326,7 @@ struct redisServer {
list *ready_keys; /* List of readyList structures for BLPOP & co */ list *ready_keys; /* List of readyList structures for BLPOP & co */
/* Client side caching. */ /* Client side caching. */
unsigned int tracking_clients; /* # of clients with tracking enabled.*/ unsigned int tracking_clients; /* # of clients with tracking enabled.*/
int tracking_table_max_fill; /* Max fill percentage. */
/* Sort parameters - qsort_r() is only available under BSD so we /* Sort parameters - qsort_r() is only available under BSD so we
* have to take this state global, in order to pass it to sortCompare() */ * have to take this state global, in order to pass it to sortCompare() */
int sort_desc; int sort_desc;
...@@ -1528,6 +1539,7 @@ void moduleAcquireGIL(void); ...@@ -1528,6 +1539,7 @@ void moduleAcquireGIL(void);
void moduleReleaseGIL(void); void moduleReleaseGIL(void);
void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid);
void moduleCallCommandFilters(client *c); void moduleCallCommandFilters(client *c);
ssize_t rdbSaveModulesAux(rio *rdb, int when);
int moduleAllDatatypesHandleErrors(); int moduleAllDatatypesHandleErrors();
/* Utils */ /* Utils */
...@@ -1639,6 +1651,9 @@ void enableTracking(client *c, uint64_t redirect_to); ...@@ -1639,6 +1651,9 @@ void enableTracking(client *c, uint64_t redirect_to);
void disableTracking(client *c); void disableTracking(client *c);
void trackingRememberKeys(client *c); void trackingRememberKeys(client *c);
void trackingInvalidateKey(robj *keyobj); void trackingInvalidateKey(robj *keyobj);
void trackingInvalidateKeysOnFlush(int dbid);
void trackingLimitUsedSlots(void);
unsigned long long trackingGetUsedSlots(void);
/* List data type */ /* List data type */
void listTypeTryConversion(robj *subject, robj *value); void listTypeTryConversion(robj *subject, robj *value);
......
...@@ -109,5 +109,6 @@ streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id); ...@@ -109,5 +109,6 @@ streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id);
streamNACK *streamCreateNACK(streamConsumer *consumer); streamNACK *streamCreateNACK(streamConsumer *consumer);
void streamDecodeID(void *buf, streamID *id); void streamDecodeID(void *buf, streamID *id);
int streamCompareID(streamID *a, streamID *b); int streamCompareID(streamID *a, streamID *b);
void streamFreeNACK(streamNACK *na);
#endif #endif
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment