Unverified Commit 0b2d0be3 authored by Wang Yuan's avatar Wang Yuan Committed by GitHub
Browse files

Make full use of replication backlog memory (#8966)

According jemalloc size classes, we may allocate much more memory
than our setting of repl_backlog_size, but we don't make full use of it.
parent a59e75a4
...@@ -2242,10 +2242,10 @@ static int updateJemallocBgThread(int val, int prev, const char **err) { ...@@ -2242,10 +2242,10 @@ static int updateJemallocBgThread(int val, int prev, const char **err) {
} }
static int updateReplBacklogSize(long long val, long long prev, const char **err) { static int updateReplBacklogSize(long long val, long long prev, const char **err) {
/* resizeReplicationBacklog sets server.repl_backlog_size, and relies on /* resizeReplicationBacklog sets server.cfg_repl_backlog_size, and relies on
* being able to tell when the size changes, so restore prev before calling it. */ * being able to tell when the size changes, so restore prev before calling it. */
UNUSED(err); UNUSED(err);
server.repl_backlog_size = prev; server.cfg_repl_backlog_size = prev;
resizeReplicationBacklog(val); resizeReplicationBacklog(val);
return 1; return 1;
} }
...@@ -2523,7 +2523,7 @@ standardConfig configs[] = { ...@@ -2523,7 +2523,7 @@ standardConfig configs[] = {
createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, 0, INTEGER_CONFIG, NULL, NULL), createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, 0, INTEGER_CONFIG, NULL, NULL),
createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 1024*1024, LONG_MAX, server.proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */ createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 1024*1024, LONG_MAX, server.proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */
createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.stream_node_max_entries, 100, INTEGER_CONFIG, NULL, NULL), createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.stream_node_max_entries, 100, INTEGER_CONFIG, NULL, NULL),
createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */ createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, server.cfg_repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */
/* Unsigned Long Long configs */ /* Unsigned Long Long configs */
createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.maxmemory, 0, MEMORY_CONFIG, NULL, updateMaxmemory), createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.maxmemory, 0, MEMORY_CONFIG, NULL, updateMaxmemory),
......
...@@ -109,7 +109,8 @@ int bg_unlink(const char *filename) { ...@@ -109,7 +109,8 @@ int bg_unlink(const char *filename) {
void createReplicationBacklog(void) { void createReplicationBacklog(void) {
serverAssert(server.repl_backlog == NULL); serverAssert(server.repl_backlog == NULL);
server.repl_backlog = zmalloc(server.repl_backlog_size); server.repl_backlog = zmalloc(server.cfg_repl_backlog_size);
server.repl_backlog_size = zmalloc_usable_size(server.repl_backlog);
server.repl_backlog_histlen = 0; server.repl_backlog_histlen = 0;
server.repl_backlog_idx = 0; server.repl_backlog_idx = 0;
...@@ -121,16 +122,16 @@ void createReplicationBacklog(void) { ...@@ -121,16 +122,16 @@ void createReplicationBacklog(void) {
/* This function is called when the user modifies the replication backlog /* This function is called when the user modifies the replication backlog
* size at runtime. It is up to the function to both update the * size at runtime. It is up to the function to both update the
* server.repl_backlog_size and to resize the buffer and setup it so that * server.cfg_repl_backlog_size and to resize the buffer and setup it so that
* it contains the same data as the previous one (possibly less data, but * it contains the same data as the previous one (possibly less data, but
* the most recent bytes, or the same data and more free space in case the * the most recent bytes, or the same data and more free space in case the
* buffer is enlarged). */ * buffer is enlarged). */
void resizeReplicationBacklog(long long newsize) { void resizeReplicationBacklog(long long newsize) {
if (newsize < CONFIG_REPL_BACKLOG_MIN_SIZE) if (newsize < CONFIG_REPL_BACKLOG_MIN_SIZE)
newsize = CONFIG_REPL_BACKLOG_MIN_SIZE; newsize = CONFIG_REPL_BACKLOG_MIN_SIZE;
if (server.repl_backlog_size == newsize) return; if (server.cfg_repl_backlog_size == newsize) return;
server.repl_backlog_size = newsize; server.cfg_repl_backlog_size = newsize;
if (server.repl_backlog != NULL) { if (server.repl_backlog != NULL) {
/* What we actually do is to flush the old buffer and realloc a new /* What we actually do is to flush the old buffer and realloc a new
* empty one. It will refill with new data incrementally. * empty one. It will refill with new data incrementally.
...@@ -138,7 +139,8 @@ void resizeReplicationBacklog(long long newsize) { ...@@ -138,7 +139,8 @@ void resizeReplicationBacklog(long long newsize) {
* worse often we need to alloc additional space before freeing the * worse often we need to alloc additional space before freeing the
* old buffer. */ * old buffer. */
zfree(server.repl_backlog); zfree(server.repl_backlog);
server.repl_backlog = zmalloc(server.repl_backlog_size); server.repl_backlog = zmalloc(server.cfg_repl_backlog_size);
server.repl_backlog_size = zmalloc_usable_size(server.repl_backlog);
server.repl_backlog_histlen = 0; server.repl_backlog_histlen = 0;
server.repl_backlog_idx = 0; server.repl_backlog_idx = 0;
/* Next byte we have is... the next since the buffer is empty. */ /* Next byte we have is... the next since the buffer is empty. */
......
...@@ -1432,6 +1432,7 @@ struct redisServer { ...@@ -1432,6 +1432,7 @@ struct redisServer {
int repl_ping_slave_period; /* Master pings the slave every N seconds */ int repl_ping_slave_period; /* Master pings the slave every N seconds */
char *repl_backlog; /* Replication backlog for partial syncs */ char *repl_backlog; /* Replication backlog for partial syncs */
long long repl_backlog_size; /* Backlog circular buffer size */ long long repl_backlog_size; /* Backlog circular buffer size */
long long cfg_repl_backlog_size;/* Backlog circular buffer size in config */
long long repl_backlog_histlen; /* Backlog actual data length */ long long repl_backlog_histlen; /* Backlog actual data length */
long long repl_backlog_idx; /* Backlog circular buffer current offset, long long repl_backlog_idx; /* Backlog circular buffer current offset,
that is the next byte will'll write to.*/ that is the next byte will'll write to.*/
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment