Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
54cae05e
Commit
54cae05e
authored
Mar 16, 2018
by
zhaozhao.zz
Browse files
rdb: incremental fsync when redis saves rdb
parent
84b28120
Changes
8
Hide whitespace changes
Inline
Side-by-side
src/aof.c
View file @
54cae05e
...
@@ -228,7 +228,7 @@ static void killAppendOnlyChild(void) {
...
@@ -228,7 +228,7 @@ static void killAppendOnlyChild(void) {
void
stopAppendOnly
(
void
)
{
void
stopAppendOnly
(
void
)
{
serverAssert
(
server
.
aof_state
!=
AOF_OFF
);
serverAssert
(
server
.
aof_state
!=
AOF_OFF
);
flushAppendOnlyFile
(
1
);
flushAppendOnlyFile
(
1
);
aof
_fsync
(
server
.
aof_fd
);
redis
_fsync
(
server
.
aof_fd
);
close
(
server
.
aof_fd
);
close
(
server
.
aof_fd
);
server
.
aof_fd
=
-
1
;
server
.
aof_fd
=
-
1
;
...
@@ -476,10 +476,10 @@ void flushAppendOnlyFile(int force) {
...
@@ -476,10 +476,10 @@ void flushAppendOnlyFile(int force) {
/* Perform the fsync if needed. */
/* Perform the fsync if needed. */
if
(
server
.
aof_fsync
==
AOF_FSYNC_ALWAYS
)
{
if
(
server
.
aof_fsync
==
AOF_FSYNC_ALWAYS
)
{
/*
aof
_fsync is defined as fdatasync() for Linux in order to avoid
/*
redis
_fsync is defined as fdatasync() for Linux in order to avoid
* flushing metadata. */
* flushing metadata. */
latencyStartMonitor
(
latency
);
latencyStartMonitor
(
latency
);
aof
_fsync
(
server
.
aof_fd
);
/* Let's try to get this data on the disk */
redis
_fsync
(
server
.
aof_fd
);
/* Let's try to get this data on the disk */
latencyEndMonitor
(
latency
);
latencyEndMonitor
(
latency
);
latencyAddSampleIfNeeded
(
"aof-fsync-always"
,
latency
);
latencyAddSampleIfNeeded
(
"aof-fsync-always"
,
latency
);
server
.
aof_last_fsync
=
server
.
unixtime
;
server
.
aof_last_fsync
=
server
.
unixtime
;
...
@@ -1241,7 +1241,7 @@ int rewriteAppendOnlyFile(char *filename) {
...
@@ -1241,7 +1241,7 @@ int rewriteAppendOnlyFile(char *filename) {
rioInitWithFile
(
&
aof
,
fp
);
rioInitWithFile
(
&
aof
,
fp
);
if
(
server
.
aof_rewrite_incremental_fsync
)
if
(
server
.
aof_rewrite_incremental_fsync
)
rioSetAutoSync
(
&
aof
,
AOF
_AUTOSYNC_BYTES
);
rioSetAutoSync
(
&
aof
,
REDIS
_AUTOSYNC_BYTES
);
if
(
server
.
aof_use_rdb_preamble
)
{
if
(
server
.
aof_use_rdb_preamble
)
{
int
error
;
int
error
;
...
@@ -1609,7 +1609,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
...
@@ -1609,7 +1609,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
oldfd
=
server
.
aof_fd
;
oldfd
=
server
.
aof_fd
;
server
.
aof_fd
=
newfd
;
server
.
aof_fd
=
newfd
;
if
(
server
.
aof_fsync
==
AOF_FSYNC_ALWAYS
)
if
(
server
.
aof_fsync
==
AOF_FSYNC_ALWAYS
)
aof
_fsync
(
newfd
);
redis
_fsync
(
newfd
);
else
if
(
server
.
aof_fsync
==
AOF_FSYNC_EVERYSEC
)
else
if
(
server
.
aof_fsync
==
AOF_FSYNC_EVERYSEC
)
aof_background_fsync
(
newfd
);
aof_background_fsync
(
newfd
);
server
.
aof_selected_db
=
-
1
;
/* Make sure SELECT is re-issued */
server
.
aof_selected_db
=
-
1
;
/* Make sure SELECT is re-issued */
...
...
src/bio.c
View file @
54cae05e
...
@@ -187,7 +187,7 @@ void *bioProcessBackgroundJobs(void *arg) {
...
@@ -187,7 +187,7 @@ void *bioProcessBackgroundJobs(void *arg) {
if
(
type
==
BIO_CLOSE_FILE
)
{
if
(
type
==
BIO_CLOSE_FILE
)
{
close
((
long
)
job
->
arg1
);
close
((
long
)
job
->
arg1
);
}
else
if
(
type
==
BIO_AOF_FSYNC
)
{
}
else
if
(
type
==
BIO_AOF_FSYNC
)
{
aof
_fsync
((
long
)
job
->
arg1
);
redis
_fsync
((
long
)
job
->
arg1
);
}
else
if
(
type
==
BIO_LAZY_FREE
)
{
}
else
if
(
type
==
BIO_LAZY_FREE
)
{
/* What we free changes depending on what arguments are set:
/* What we free changes depending on what arguments are set:
* arg1 -> free the object at pointer.
* arg1 -> free the object at pointer.
...
...
src/config.c
View file @
54cae05e
...
@@ -483,6 +483,13 @@ void loadServerConfigFromString(char *config) {
...
@@ -483,6 +483,13 @@ void loadServerConfigFromString(char *config) {
yesnotoi
(
argv
[
1
]))
==
-
1
)
{
yesnotoi
(
argv
[
1
]))
==
-
1
)
{
err
=
"argument must be 'yes' or 'no'"
;
goto
loaderr
;
err
=
"argument must be 'yes' or 'no'"
;
goto
loaderr
;
}
}
}
else
if
(
!
strcasecmp
(
argv
[
0
],
"rdb-save-incremental-fsync"
)
&&
argc
==
2
)
{
if
((
server
.
rdb_save_incremental_fsync
=
yesnotoi
(
argv
[
1
]))
==
-
1
)
{
err
=
"argument must be 'yes' or 'no'"
;
goto
loaderr
;
}
}
else
if
(
!
strcasecmp
(
argv
[
0
],
"aof-load-truncated"
)
&&
argc
==
2
)
{
}
else
if
(
!
strcasecmp
(
argv
[
0
],
"aof-load-truncated"
)
&&
argc
==
2
)
{
if
((
server
.
aof_load_truncated
=
yesnotoi
(
argv
[
1
]))
==
-
1
)
{
if
((
server
.
aof_load_truncated
=
yesnotoi
(
argv
[
1
]))
==
-
1
)
{
err
=
"argument must be 'yes' or 'no'"
;
goto
loaderr
;
err
=
"argument must be 'yes' or 'no'"
;
goto
loaderr
;
...
@@ -999,6 +1006,8 @@ void configSetCommand(client *c) {
...
@@ -999,6 +1006,8 @@ void configSetCommand(client *c) {
"cluster-require-full-coverage"
,
server
.
cluster_require_full_coverage
)
{
"cluster-require-full-coverage"
,
server
.
cluster_require_full_coverage
)
{
}
config_set_bool_field
(
}
config_set_bool_field
(
"aof-rewrite-incremental-fsync"
,
server
.
aof_rewrite_incremental_fsync
)
{
"aof-rewrite-incremental-fsync"
,
server
.
aof_rewrite_incremental_fsync
)
{
}
config_set_bool_field
(
"rdb-save-incremental-fsync"
,
server
.
rdb_save_incremental_fsync
)
{
}
config_set_bool_field
(
}
config_set_bool_field
(
"aof-load-truncated"
,
server
.
aof_load_truncated
)
{
"aof-load-truncated"
,
server
.
aof_load_truncated
)
{
}
config_set_bool_field
(
}
config_set_bool_field
(
...
@@ -1311,6 +1320,8 @@ void configGetCommand(client *c) {
...
@@ -1311,6 +1320,8 @@ void configGetCommand(client *c) {
server
.
repl_diskless_sync
);
server
.
repl_diskless_sync
);
config_get_bool_field
(
"aof-rewrite-incremental-fsync"
,
config_get_bool_field
(
"aof-rewrite-incremental-fsync"
,
server
.
aof_rewrite_incremental_fsync
);
server
.
aof_rewrite_incremental_fsync
);
config_get_bool_field
(
"rdb-save-incremental-fsync"
,
server
.
rdb_save_incremental_fsync
);
config_get_bool_field
(
"aof-load-truncated"
,
config_get_bool_field
(
"aof-load-truncated"
,
server
.
aof_load_truncated
);
server
.
aof_load_truncated
);
config_get_bool_field
(
"aof-use-rdb-preamble"
,
config_get_bool_field
(
"aof-use-rdb-preamble"
,
...
@@ -2044,6 +2055,7 @@ int rewriteConfig(char *path) {
...
@@ -2044,6 +2055,7 @@ int rewriteConfig(char *path) {
rewriteConfigClientoutputbufferlimitOption
(
state
);
rewriteConfigClientoutputbufferlimitOption
(
state
);
rewriteConfigNumericalOption
(
state
,
"hz"
,
server
.
hz
,
CONFIG_DEFAULT_HZ
);
rewriteConfigNumericalOption
(
state
,
"hz"
,
server
.
hz
,
CONFIG_DEFAULT_HZ
);
rewriteConfigYesNoOption
(
state
,
"aof-rewrite-incremental-fsync"
,
server
.
aof_rewrite_incremental_fsync
,
CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC
);
rewriteConfigYesNoOption
(
state
,
"aof-rewrite-incremental-fsync"
,
server
.
aof_rewrite_incremental_fsync
,
CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC
);
rewriteConfigYesNoOption
(
state
,
"rdb-save-incremental-fsync"
,
server
.
rdb_save_incremental_fsync
,
CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC
);
rewriteConfigYesNoOption
(
state
,
"aof-load-truncated"
,
server
.
aof_load_truncated
,
CONFIG_DEFAULT_AOF_LOAD_TRUNCATED
);
rewriteConfigYesNoOption
(
state
,
"aof-load-truncated"
,
server
.
aof_load_truncated
,
CONFIG_DEFAULT_AOF_LOAD_TRUNCATED
);
rewriteConfigYesNoOption
(
state
,
"aof-use-rdb-preamble"
,
server
.
aof_use_rdb_preamble
,
CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE
);
rewriteConfigYesNoOption
(
state
,
"aof-use-rdb-preamble"
,
server
.
aof_use_rdb_preamble
,
CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE
);
rewriteConfigEnumOption
(
state
,
"supervised"
,
server
.
supervised_mode
,
supervised_mode_enum
,
SUPERVISED_NONE
);
rewriteConfigEnumOption
(
state
,
"supervised"
,
server
.
supervised_mode
,
supervised_mode_enum
,
SUPERVISED_NONE
);
...
...
src/config.h
View file @
54cae05e
...
@@ -87,11 +87,11 @@
...
@@ -87,11 +87,11 @@
#endif
#endif
#endif
#endif
/* Define
aof
_fsync to fdatasync() in Linux and fsync() for all the rest */
/* Define
redis
_fsync to fdatasync() in Linux and fsync() for all the rest */
#ifdef __linux__
#ifdef __linux__
#define
aof
_fsync fdatasync
#define
redis
_fsync fdatasync
#else
#else
#define
aof
_fsync fsync
#define
redis
_fsync fsync
#endif
#endif
/* Define rdb_fsync_range to sync_file_range() on Linux, otherwise we use
/* Define rdb_fsync_range to sync_file_range() on Linux, otherwise we use
...
...
src/rdb.c
View file @
54cae05e
...
@@ -1060,6 +1060,10 @@ int rdbSave(char *filename, rdbSaveInfo *rsi) {
...
@@ -1060,6 +1060,10 @@ int rdbSave(char *filename, rdbSaveInfo *rsi) {
}
}
rioInitWithFile
(
&
rdb
,
fp
);
rioInitWithFile
(
&
rdb
,
fp
);
if
(
server
.
rdb_save_incremental_fsync
)
rioSetAutoSync
(
&
rdb
,
REDIS_AUTOSYNC_BYTES
);
if
(
rdbSaveRio
(
&
rdb
,
&
error
,
RDB_SAVE_NONE
,
rsi
)
==
C_ERR
)
{
if
(
rdbSaveRio
(
&
rdb
,
&
error
,
RDB_SAVE_NONE
,
rsi
)
==
C_ERR
)
{
errno
=
error
;
errno
=
error
;
goto
werr
;
goto
werr
;
...
...
src/rio.c
View file @
54cae05e
...
@@ -116,7 +116,7 @@ static size_t rioFileWrite(rio *r, const void *buf, size_t len) {
...
@@ -116,7 +116,7 @@ static size_t rioFileWrite(rio *r, const void *buf, size_t len) {
r
->
io
.
file
.
buffered
>=
r
->
io
.
file
.
autosync
)
r
->
io
.
file
.
buffered
>=
r
->
io
.
file
.
autosync
)
{
{
fflush
(
r
->
io
.
file
.
fp
);
fflush
(
r
->
io
.
file
.
fp
);
aof
_fsync
(
fileno
(
r
->
io
.
file
.
fp
));
redis
_fsync
(
fileno
(
r
->
io
.
file
.
fp
));
r
->
io
.
file
.
buffered
=
0
;
r
->
io
.
file
.
buffered
=
0
;
}
}
return
retval
;
return
retval
;
...
...
src/server.c
View file @
54cae05e
...
@@ -1416,6 +1416,7 @@ void initServerConfig(void) {
...
@@ -1416,6 +1416,7 @@ void initServerConfig(void) {
server
.
aof_selected_db
=
-
1
;
/* Make sure the first time will not match */
server
.
aof_selected_db
=
-
1
;
/* Make sure the first time will not match */
server
.
aof_flush_postponed_start
=
0
;
server
.
aof_flush_postponed_start
=
0
;
server
.
aof_rewrite_incremental_fsync
=
CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC
;
server
.
aof_rewrite_incremental_fsync
=
CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC
;
server
.
rdb_save_incremental_fsync
=
CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC
;
server
.
aof_load_truncated
=
CONFIG_DEFAULT_AOF_LOAD_TRUNCATED
;
server
.
aof_load_truncated
=
CONFIG_DEFAULT_AOF_LOAD_TRUNCATED
;
server
.
aof_use_rdb_preamble
=
CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE
;
server
.
aof_use_rdb_preamble
=
CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE
;
server
.
pidfile
=
NULL
;
server
.
pidfile
=
NULL
;
...
@@ -2585,7 +2586,7 @@ int prepareForShutdown(int flags) {
...
@@ -2585,7 +2586,7 @@ int prepareForShutdown(int flags) {
/* Append only file: flush buffers and fsync() the AOF at exit */
/* Append only file: flush buffers and fsync() the AOF at exit */
serverLog
(
LL_NOTICE
,
"Calling fsync() on the AOF file."
);
serverLog
(
LL_NOTICE
,
"Calling fsync() on the AOF file."
);
flushAppendOnlyFile
(
1
);
flushAppendOnlyFile
(
1
);
aof
_fsync
(
server
.
aof_fd
);
redis
_fsync
(
server
.
aof_fd
);
}
}
/* Create a new RDB file before exiting. */
/* Create a new RDB file before exiting. */
...
...
src/server.h
View file @
54cae05e
...
@@ -143,6 +143,7 @@ typedef long long mstime_t; /* millisecond time type. */
...
@@ -143,6 +143,7 @@ typedef long long mstime_t; /* millisecond time type. */
#define CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE 0
#define CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE 0
#define CONFIG_DEFAULT_ACTIVE_REHASHING 1
#define CONFIG_DEFAULT_ACTIVE_REHASHING 1
#define CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC 1
#define CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC 1
#define CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC 1
#define CONFIG_DEFAULT_MIN_SLAVES_TO_WRITE 0
#define CONFIG_DEFAULT_MIN_SLAVES_TO_WRITE 0
#define CONFIG_DEFAULT_MIN_SLAVES_MAX_LAG 10
#define CONFIG_DEFAULT_MIN_SLAVES_MAX_LAG 10
#define NET_IP_STR_LEN 46
/* INET6_ADDRSTRLEN is 46, but we need to be sure */
#define NET_IP_STR_LEN 46
/* INET6_ADDRSTRLEN is 46, but we need to be sure */
...
@@ -183,7 +184,7 @@ typedef long long mstime_t; /* millisecond time type. */
...
@@ -183,7 +184,7 @@ typedef long long mstime_t; /* millisecond time type. */
#define PROTO_INLINE_MAX_SIZE (1024*64)
/* Max size of inline reads */
#define PROTO_INLINE_MAX_SIZE (1024*64)
/* Max size of inline reads */
#define PROTO_MBULK_BIG_ARG (1024*32)
#define PROTO_MBULK_BIG_ARG (1024*32)
#define LONG_STR_SIZE 21
/* Bytes needed for long -> str + '\0' */
#define LONG_STR_SIZE 21
/* Bytes needed for long -> str + '\0' */
#define
AOF
_AUTOSYNC_BYTES (1024*1024*32)
/* fdatasync every 32MB */
#define
REDIS
_AUTOSYNC_BYTES (1024*1024*32)
/* fdatasync every 32MB */
/* When configuring the server eventloop, we setup it so that the total number
/* When configuring the server eventloop, we setup it so that the total number
* of file descriptors we can handle are server.maxclients + RESERVED_FDS +
* of file descriptors we can handle are server.maxclients + RESERVED_FDS +
...
@@ -1021,7 +1022,8 @@ struct redisServer {
...
@@ -1021,7 +1022,8 @@ struct redisServer {
time_t
aof_rewrite_time_start
;
/* Current AOF rewrite start time. */
time_t
aof_rewrite_time_start
;
/* Current AOF rewrite start time. */
int
aof_lastbgrewrite_status
;
/* C_OK or C_ERR */
int
aof_lastbgrewrite_status
;
/* C_OK or C_ERR */
unsigned
long
aof_delayed_fsync
;
/* delayed AOF fsync() counter */
unsigned
long
aof_delayed_fsync
;
/* delayed AOF fsync() counter */
int
aof_rewrite_incremental_fsync
;
/* fsync incrementally while rewriting? */
int
aof_rewrite_incremental_fsync
;
/* fsync incrementally while aof rewriting? */
int
rdb_save_incremental_fsync
;
/* fsync incrementally while rdb saving? */
int
aof_last_write_status
;
/* C_OK or C_ERR */
int
aof_last_write_status
;
/* C_OK or C_ERR */
int
aof_last_write_errno
;
/* Valid if aof_last_write_status is ERR */
int
aof_last_write_errno
;
/* Valid if aof_last_write_status is ERR */
int
aof_load_truncated
;
/* Don't stop on unexpected AOF EOF. */
int
aof_load_truncated
;
/* Don't stop on unexpected AOF EOF. */
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment