Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
94658303
Unverified
Commit
94658303
authored
Jun 16, 2018
by
Salvatore Sanfilippo
Committed by
GitHub
Jun 16, 2018
Browse files
Merge pull request #4758 from soloestoy/rdb-save-incremental-fsync
Rdb save incremental fsync
parents
6a66b93b
fd0c4c02
Changes
9
Hide whitespace changes
Inline
Side-by-side
redis.conf
View file @
94658303
...
@@ -1211,6 +1211,12 @@ hz 10
...
@@ -1211,6 +1211,12 @@ hz 10
# big latency spikes.
# big latency spikes.
aof
-
rewrite
-
incremental
-
fsync
yes
aof
-
rewrite
-
incremental
-
fsync
yes
# When redis saves RDB file, if the following option is enabled
# the file will be fsync-ed every 32 MB of data generated. This is useful
# in order to commit the file to the disk more incrementally and avoid
# big latency spikes.
rdb
-
save
-
incremental
-
fsync
yes
# Redis LFU eviction (see maxmemory setting) can be tuned. However it is a good
# Redis LFU eviction (see maxmemory setting) can be tuned. However it is a good
# idea to start with the default settings and only change them after investigating
# idea to start with the default settings and only change them after investigating
# how to improve the performances and how the keys LFU change over time, which
# how to improve the performances and how the keys LFU change over time, which
...
...
src/aof.c
View file @
94658303
...
@@ -228,7 +228,7 @@ static void killAppendOnlyChild(void) {
...
@@ -228,7 +228,7 @@ static void killAppendOnlyChild(void) {
void
stopAppendOnly
(
void
)
{
void
stopAppendOnly
(
void
)
{
serverAssert
(
server
.
aof_state
!=
AOF_OFF
);
serverAssert
(
server
.
aof_state
!=
AOF_OFF
);
flushAppendOnlyFile
(
1
);
flushAppendOnlyFile
(
1
);
aof
_fsync
(
server
.
aof_fd
);
redis
_fsync
(
server
.
aof_fd
);
close
(
server
.
aof_fd
);
close
(
server
.
aof_fd
);
server
.
aof_fd
=
-
1
;
server
.
aof_fd
=
-
1
;
...
@@ -476,10 +476,10 @@ void flushAppendOnlyFile(int force) {
...
@@ -476,10 +476,10 @@ void flushAppendOnlyFile(int force) {
/* Perform the fsync if needed. */
/* Perform the fsync if needed. */
if
(
server
.
aof_fsync
==
AOF_FSYNC_ALWAYS
)
{
if
(
server
.
aof_fsync
==
AOF_FSYNC_ALWAYS
)
{
/*
aof
_fsync is defined as fdatasync() for Linux in order to avoid
/*
redis
_fsync is defined as fdatasync() for Linux in order to avoid
* flushing metadata. */
* flushing metadata. */
latencyStartMonitor
(
latency
);
latencyStartMonitor
(
latency
);
aof
_fsync
(
server
.
aof_fd
);
/* Let's try to get this data on the disk */
redis
_fsync
(
server
.
aof_fd
);
/* Let's try to get this data on the disk */
latencyEndMonitor
(
latency
);
latencyEndMonitor
(
latency
);
latencyAddSampleIfNeeded
(
"aof-fsync-always"
,
latency
);
latencyAddSampleIfNeeded
(
"aof-fsync-always"
,
latency
);
server
.
aof_last_fsync
=
server
.
unixtime
;
server
.
aof_last_fsync
=
server
.
unixtime
;
...
@@ -1322,7 +1322,7 @@ int rewriteAppendOnlyFile(char *filename) {
...
@@ -1322,7 +1322,7 @@ int rewriteAppendOnlyFile(char *filename) {
rioInitWithFile
(
&
aof
,
fp
);
rioInitWithFile
(
&
aof
,
fp
);
if
(
server
.
aof_rewrite_incremental_fsync
)
if
(
server
.
aof_rewrite_incremental_fsync
)
rioSetAutoSync
(
&
aof
,
AOF
_AUTOSYNC_BYTES
);
rioSetAutoSync
(
&
aof
,
REDIS
_AUTOSYNC_BYTES
);
if
(
server
.
aof_use_rdb_preamble
)
{
if
(
server
.
aof_use_rdb_preamble
)
{
int
error
;
int
error
;
...
@@ -1690,7 +1690,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
...
@@ -1690,7 +1690,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
oldfd
=
server
.
aof_fd
;
oldfd
=
server
.
aof_fd
;
server
.
aof_fd
=
newfd
;
server
.
aof_fd
=
newfd
;
if
(
server
.
aof_fsync
==
AOF_FSYNC_ALWAYS
)
if
(
server
.
aof_fsync
==
AOF_FSYNC_ALWAYS
)
aof
_fsync
(
newfd
);
redis
_fsync
(
newfd
);
else
if
(
server
.
aof_fsync
==
AOF_FSYNC_EVERYSEC
)
else
if
(
server
.
aof_fsync
==
AOF_FSYNC_EVERYSEC
)
aof_background_fsync
(
newfd
);
aof_background_fsync
(
newfd
);
server
.
aof_selected_db
=
-
1
;
/* Make sure SELECT is re-issued */
server
.
aof_selected_db
=
-
1
;
/* Make sure SELECT is re-issued */
...
...
src/bio.c
View file @
94658303
...
@@ -187,7 +187,7 @@ void *bioProcessBackgroundJobs(void *arg) {
...
@@ -187,7 +187,7 @@ void *bioProcessBackgroundJobs(void *arg) {
if
(
type
==
BIO_CLOSE_FILE
)
{
if
(
type
==
BIO_CLOSE_FILE
)
{
close
((
long
)
job
->
arg1
);
close
((
long
)
job
->
arg1
);
}
else
if
(
type
==
BIO_AOF_FSYNC
)
{
}
else
if
(
type
==
BIO_AOF_FSYNC
)
{
aof
_fsync
((
long
)
job
->
arg1
);
redis
_fsync
((
long
)
job
->
arg1
);
}
else
if
(
type
==
BIO_LAZY_FREE
)
{
}
else
if
(
type
==
BIO_LAZY_FREE
)
{
/* What we free changes depending on what arguments are set:
/* What we free changes depending on what arguments are set:
* arg1 -> free the object at pointer.
* arg1 -> free the object at pointer.
...
...
src/config.c
View file @
94658303
...
@@ -483,6 +483,13 @@ void loadServerConfigFromString(char *config) {
...
@@ -483,6 +483,13 @@ void loadServerConfigFromString(char *config) {
yesnotoi
(
argv
[
1
]))
==
-
1
)
{
yesnotoi
(
argv
[
1
]))
==
-
1
)
{
err
=
"argument must be 'yes' or 'no'"
;
goto
loaderr
;
err
=
"argument must be 'yes' or 'no'"
;
goto
loaderr
;
}
}
}
else
if
(
!
strcasecmp
(
argv
[
0
],
"rdb-save-incremental-fsync"
)
&&
argc
==
2
)
{
if
((
server
.
rdb_save_incremental_fsync
=
yesnotoi
(
argv
[
1
]))
==
-
1
)
{
err
=
"argument must be 'yes' or 'no'"
;
goto
loaderr
;
}
}
else
if
(
!
strcasecmp
(
argv
[
0
],
"aof-load-truncated"
)
&&
argc
==
2
)
{
}
else
if
(
!
strcasecmp
(
argv
[
0
],
"aof-load-truncated"
)
&&
argc
==
2
)
{
if
((
server
.
aof_load_truncated
=
yesnotoi
(
argv
[
1
]))
==
-
1
)
{
if
((
server
.
aof_load_truncated
=
yesnotoi
(
argv
[
1
]))
==
-
1
)
{
err
=
"argument must be 'yes' or 'no'"
;
goto
loaderr
;
err
=
"argument must be 'yes' or 'no'"
;
goto
loaderr
;
...
@@ -1021,6 +1028,8 @@ void configSetCommand(client *c) {
...
@@ -1021,6 +1028,8 @@ void configSetCommand(client *c) {
"cluster-slave-no-failover"
,
server
.
cluster_slave_no_failover
)
{
"cluster-slave-no-failover"
,
server
.
cluster_slave_no_failover
)
{
}
config_set_bool_field
(
}
config_set_bool_field
(
"aof-rewrite-incremental-fsync"
,
server
.
aof_rewrite_incremental_fsync
)
{
"aof-rewrite-incremental-fsync"
,
server
.
aof_rewrite_incremental_fsync
)
{
}
config_set_bool_field
(
"rdb-save-incremental-fsync"
,
server
.
rdb_save_incremental_fsync
)
{
}
config_set_bool_field
(
}
config_set_bool_field
(
"aof-load-truncated"
,
server
.
aof_load_truncated
)
{
"aof-load-truncated"
,
server
.
aof_load_truncated
)
{
}
config_set_bool_field
(
}
config_set_bool_field
(
...
@@ -1347,6 +1356,8 @@ void configGetCommand(client *c) {
...
@@ -1347,6 +1356,8 @@ void configGetCommand(client *c) {
server
.
repl_diskless_sync
);
server
.
repl_diskless_sync
);
config_get_bool_field
(
"aof-rewrite-incremental-fsync"
,
config_get_bool_field
(
"aof-rewrite-incremental-fsync"
,
server
.
aof_rewrite_incremental_fsync
);
server
.
aof_rewrite_incremental_fsync
);
config_get_bool_field
(
"rdb-save-incremental-fsync"
,
server
.
rdb_save_incremental_fsync
);
config_get_bool_field
(
"aof-load-truncated"
,
config_get_bool_field
(
"aof-load-truncated"
,
server
.
aof_load_truncated
);
server
.
aof_load_truncated
);
config_get_bool_field
(
"aof-use-rdb-preamble"
,
config_get_bool_field
(
"aof-use-rdb-preamble"
,
...
@@ -2084,6 +2095,7 @@ int rewriteConfig(char *path) {
...
@@ -2084,6 +2095,7 @@ int rewriteConfig(char *path) {
rewriteConfigClientoutputbufferlimitOption
(
state
);
rewriteConfigClientoutputbufferlimitOption
(
state
);
rewriteConfigNumericalOption
(
state
,
"hz"
,
server
.
hz
,
CONFIG_DEFAULT_HZ
);
rewriteConfigNumericalOption
(
state
,
"hz"
,
server
.
hz
,
CONFIG_DEFAULT_HZ
);
rewriteConfigYesNoOption
(
state
,
"aof-rewrite-incremental-fsync"
,
server
.
aof_rewrite_incremental_fsync
,
CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC
);
rewriteConfigYesNoOption
(
state
,
"aof-rewrite-incremental-fsync"
,
server
.
aof_rewrite_incremental_fsync
,
CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC
);
rewriteConfigYesNoOption
(
state
,
"rdb-save-incremental-fsync"
,
server
.
rdb_save_incremental_fsync
,
CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC
);
rewriteConfigYesNoOption
(
state
,
"aof-load-truncated"
,
server
.
aof_load_truncated
,
CONFIG_DEFAULT_AOF_LOAD_TRUNCATED
);
rewriteConfigYesNoOption
(
state
,
"aof-load-truncated"
,
server
.
aof_load_truncated
,
CONFIG_DEFAULT_AOF_LOAD_TRUNCATED
);
rewriteConfigYesNoOption
(
state
,
"aof-use-rdb-preamble"
,
server
.
aof_use_rdb_preamble
,
CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE
);
rewriteConfigYesNoOption
(
state
,
"aof-use-rdb-preamble"
,
server
.
aof_use_rdb_preamble
,
CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE
);
rewriteConfigEnumOption
(
state
,
"supervised"
,
server
.
supervised_mode
,
supervised_mode_enum
,
SUPERVISED_NONE
);
rewriteConfigEnumOption
(
state
,
"supervised"
,
server
.
supervised_mode
,
supervised_mode_enum
,
SUPERVISED_NONE
);
...
...
src/config.h
View file @
94658303
...
@@ -87,11 +87,11 @@
...
@@ -87,11 +87,11 @@
#endif
#endif
#endif
#endif
/* Define
aof
_fsync to fdatasync() in Linux and fsync() for all the rest */
/* Define
redis
_fsync to fdatasync() in Linux and fsync() for all the rest */
#ifdef __linux__
#ifdef __linux__
#define
aof
_fsync fdatasync
#define
redis
_fsync fdatasync
#else
#else
#define
aof
_fsync fsync
#define
redis
_fsync fsync
#endif
#endif
/* Define rdb_fsync_range to sync_file_range() on Linux, otherwise we use
/* Define rdb_fsync_range to sync_file_range() on Linux, otherwise we use
...
...
src/rdb.c
View file @
94658303
...
@@ -1241,6 +1241,10 @@ int rdbSave(char *filename, rdbSaveInfo *rsi) {
...
@@ -1241,6 +1241,10 @@ int rdbSave(char *filename, rdbSaveInfo *rsi) {
}
}
rioInitWithFile
(
&
rdb
,
fp
);
rioInitWithFile
(
&
rdb
,
fp
);
if
(
server
.
rdb_save_incremental_fsync
)
rioSetAutoSync
(
&
rdb
,
REDIS_AUTOSYNC_BYTES
);
if
(
rdbSaveRio
(
&
rdb
,
&
error
,
RDB_SAVE_NONE
,
rsi
)
==
C_ERR
)
{
if
(
rdbSaveRio
(
&
rdb
,
&
error
,
RDB_SAVE_NONE
,
rsi
)
==
C_ERR
)
{
errno
=
error
;
errno
=
error
;
goto
werr
;
goto
werr
;
...
...
src/rio.c
View file @
94658303
...
@@ -116,7 +116,7 @@ static size_t rioFileWrite(rio *r, const void *buf, size_t len) {
...
@@ -116,7 +116,7 @@ static size_t rioFileWrite(rio *r, const void *buf, size_t len) {
r
->
io
.
file
.
buffered
>=
r
->
io
.
file
.
autosync
)
r
->
io
.
file
.
buffered
>=
r
->
io
.
file
.
autosync
)
{
{
fflush
(
r
->
io
.
file
.
fp
);
fflush
(
r
->
io
.
file
.
fp
);
aof
_fsync
(
fileno
(
r
->
io
.
file
.
fp
));
redis
_fsync
(
fileno
(
r
->
io
.
file
.
fp
));
r
->
io
.
file
.
buffered
=
0
;
r
->
io
.
file
.
buffered
=
0
;
}
}
return
retval
;
return
retval
;
...
...
src/server.c
View file @
94658303
...
@@ -1457,6 +1457,7 @@ void initServerConfig(void) {
...
@@ -1457,6 +1457,7 @@ void initServerConfig(void) {
server
.
aof_selected_db
=
-
1
;
/* Make sure the first time will not match */
server
.
aof_selected_db
=
-
1
;
/* Make sure the first time will not match */
server
.
aof_flush_postponed_start
=
0
;
server
.
aof_flush_postponed_start
=
0
;
server
.
aof_rewrite_incremental_fsync
=
CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC
;
server
.
aof_rewrite_incremental_fsync
=
CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC
;
server
.
rdb_save_incremental_fsync
=
CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC
;
server
.
aof_load_truncated
=
CONFIG_DEFAULT_AOF_LOAD_TRUNCATED
;
server
.
aof_load_truncated
=
CONFIG_DEFAULT_AOF_LOAD_TRUNCATED
;
server
.
aof_use_rdb_preamble
=
CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE
;
server
.
aof_use_rdb_preamble
=
CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE
;
server
.
pidfile
=
NULL
;
server
.
pidfile
=
NULL
;
...
@@ -2638,7 +2639,7 @@ int prepareForShutdown(int flags) {
...
@@ -2638,7 +2639,7 @@ int prepareForShutdown(int flags) {
/* Append only file: flush buffers and fsync() the AOF at exit */
/* Append only file: flush buffers and fsync() the AOF at exit */
serverLog
(
LL_NOTICE
,
"Calling fsync() on the AOF file."
);
serverLog
(
LL_NOTICE
,
"Calling fsync() on the AOF file."
);
flushAppendOnlyFile
(
1
);
flushAppendOnlyFile
(
1
);
aof
_fsync
(
server
.
aof_fd
);
redis
_fsync
(
server
.
aof_fd
);
}
}
/* Create a new RDB file before exiting. */
/* Create a new RDB file before exiting. */
...
...
src/server.h
View file @
94658303
...
@@ -142,6 +142,7 @@ typedef long long mstime_t; /* millisecond time type. */
...
@@ -142,6 +142,7 @@ typedef long long mstime_t; /* millisecond time type. */
#define CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE 1
#define CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE 1
#define CONFIG_DEFAULT_ACTIVE_REHASHING 1
#define CONFIG_DEFAULT_ACTIVE_REHASHING 1
#define CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC 1
#define CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC 1
#define CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC 1
#define CONFIG_DEFAULT_MIN_SLAVES_TO_WRITE 0
#define CONFIG_DEFAULT_MIN_SLAVES_TO_WRITE 0
#define CONFIG_DEFAULT_MIN_SLAVES_MAX_LAG 10
#define CONFIG_DEFAULT_MIN_SLAVES_MAX_LAG 10
#define NET_IP_STR_LEN 46
/* INET6_ADDRSTRLEN is 46, but we need to be sure */
#define NET_IP_STR_LEN 46
/* INET6_ADDRSTRLEN is 46, but we need to be sure */
...
@@ -183,7 +184,7 @@ typedef long long mstime_t; /* millisecond time type. */
...
@@ -183,7 +184,7 @@ typedef long long mstime_t; /* millisecond time type. */
#define PROTO_INLINE_MAX_SIZE (1024*64)
/* Max size of inline reads */
#define PROTO_INLINE_MAX_SIZE (1024*64)
/* Max size of inline reads */
#define PROTO_MBULK_BIG_ARG (1024*32)
#define PROTO_MBULK_BIG_ARG (1024*32)
#define LONG_STR_SIZE 21
/* Bytes needed for long -> str + '\0' */
#define LONG_STR_SIZE 21
/* Bytes needed for long -> str + '\0' */
#define
AOF
_AUTOSYNC_BYTES (1024*1024*32)
/* fdatasync every 32MB */
#define
REDIS
_AUTOSYNC_BYTES (1024*1024*32)
/* fdatasync every 32MB */
/* When configuring the server eventloop, we setup it so that the total number
/* When configuring the server eventloop, we setup it so that the total number
* of file descriptors we can handle are server.maxclients + RESERVED_FDS +
* of file descriptors we can handle are server.maxclients + RESERVED_FDS +
...
@@ -1046,7 +1047,8 @@ struct redisServer {
...
@@ -1046,7 +1047,8 @@ struct redisServer {
time_t
aof_rewrite_time_start
;
/* Current AOF rewrite start time. */
time_t
aof_rewrite_time_start
;
/* Current AOF rewrite start time. */
int
aof_lastbgrewrite_status
;
/* C_OK or C_ERR */
int
aof_lastbgrewrite_status
;
/* C_OK or C_ERR */
unsigned
long
aof_delayed_fsync
;
/* delayed AOF fsync() counter */
unsigned
long
aof_delayed_fsync
;
/* delayed AOF fsync() counter */
int
aof_rewrite_incremental_fsync
;
/* fsync incrementally while rewriting? */
int
aof_rewrite_incremental_fsync
;
/* fsync incrementally while aof rewriting? */
int
rdb_save_incremental_fsync
;
/* fsync incrementally while rdb saving? */
int
aof_last_write_status
;
/* C_OK or C_ERR */
int
aof_last_write_status
;
/* C_OK or C_ERR */
int
aof_last_write_errno
;
/* Valid if aof_last_write_status is ERR */
int
aof_last_write_errno
;
/* Valid if aof_last_write_status is ERR */
int
aof_load_truncated
;
/* Don't stop on unexpected AOF EOF. */
int
aof_load_truncated
;
/* Don't stop on unexpected AOF EOF. */
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment