Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
b333e239
Commit
b333e239
authored
Jun 10, 2011
by
antirez
Browse files
automatic AOF rewrite first implementation. Still to be tested.
parent
936c4ab6
Changes
6
Show whitespace changes
Inline
Side-by-side
redis.conf
View file @
b333e239
...
@@ -292,6 +292,26 @@ appendfsync everysec
...
@@ -292,6 +292,26 @@ appendfsync everysec
# "no" that is the safest pick from the point of view of durability.
# "no" that is the safest pick from the point of view of durability.
no
-
appendfsync
-
on
-
rewrite
no
no
-
appendfsync
-
on
-
rewrite
no
# Automatic rewrite of the append only file.
# Redis is able to automatically rewrite the log file implicitly calling
# BGREWRITEAOF when the AOF log size will growth by the specified percentage.
#
# This is how it works: Redis remembers the size of the AOF file after the
# latest rewrite (or if no rewrite happened since the restart, the size of
# the AOF at startup is used).
#
# This base size is compared to the current size. If the current size is
# bigger than the specified percentage, the rewrite is triggered. Also
# you need to specify a minimal size for the AOF file to be rewritten, this
# is useful to avoid rewriting the AOF file even if the percentage increase
# is reached but it is still pretty small.
#
# Specify a precentage of zero in order to disable the automatic AOF
# rewrite feature.
auto
-
aof
-
rewrite
-
percentage
100
auto
-
aof
-
rewrite
-
min
-
size
64
mb
#################################### DISK STORE ###############################
#################################### DISK STORE ###############################
# When disk store is active Redis works as an on-disk database, where memory
# When disk store is active Redis works as an on-disk database, where memory
...
...
src/aof.c
View file @
b333e239
...
@@ -8,6 +8,8 @@
...
@@ -8,6 +8,8 @@
#include <sys/resource.h>
#include <sys/resource.h>
#include <sys/wait.h>
#include <sys/wait.h>
void
aofUpdateCurrentSize
(
void
);
/* Called when the user switches from "appendonly yes" to "appendonly no"
/* Called when the user switches from "appendonly yes" to "appendonly no"
* at runtime using the CONFIG command. */
* at runtime using the CONFIG command. */
void
stopAppendOnly
(
void
)
{
void
stopAppendOnly
(
void
)
{
...
@@ -19,15 +21,15 @@ void stopAppendOnly(void) {
...
@@ -19,15 +21,15 @@ void stopAppendOnly(void) {
server
.
appendseldb
=
-
1
;
server
.
appendseldb
=
-
1
;
server
.
appendonly
=
0
;
server
.
appendonly
=
0
;
/* rewrite operation in progress? kill it, wait child exit */
/* rewrite operation in progress? kill it, wait child exit */
if
(
server
.
bg
sav
echildpid
!=
-
1
)
{
if
(
server
.
bg
rewrit
echildpid
!=
-
1
)
{
int
statloc
;
int
statloc
;
if
(
kill
(
server
.
bg
sav
echildpid
,
SIGKILL
)
!=
-
1
)
if
(
kill
(
server
.
bg
rewrit
echildpid
,
SIGKILL
)
!=
-
1
)
wait3
(
&
statloc
,
0
,
NULL
);
wait3
(
&
statloc
,
0
,
NULL
);
/* reset the buffer accumulating changes while the child saves */
/* reset the buffer accumulating changes while the child saves */
sdsfree
(
server
.
bgrewritebuf
);
sdsfree
(
server
.
bgrewritebuf
);
server
.
bgrewritebuf
=
sdsempty
();
server
.
bgrewritebuf
=
sdsempty
();
server
.
bg
sav
echildpid
=
-
1
;
server
.
bg
rewrit
echildpid
=
-
1
;
}
}
}
}
...
@@ -82,6 +84,7 @@ void flushAppendOnlyFile(void) {
...
@@ -82,6 +84,7 @@ void flushAppendOnlyFile(void) {
}
}
sdsfree
(
server
.
aofbuf
);
sdsfree
(
server
.
aofbuf
);
server
.
aofbuf
=
sdsempty
();
server
.
aofbuf
=
sdsempty
();
server
.
appendonly_current_size
+=
nwritten
;
/* Don't Fsync if no-appendfsync-on-rewrite is set to yes and we have
/* Don't Fsync if no-appendfsync-on-rewrite is set to yes and we have
* childs performing heavy I/O on disk. */
* childs performing heavy I/O on disk. */
...
@@ -221,6 +224,7 @@ int loadAppendOnlyFile(char *filename) {
...
@@ -221,6 +224,7 @@ int loadAppendOnlyFile(char *filename) {
long
loops
=
0
;
long
loops
=
0
;
if
(
fp
&&
redis_fstat
(
fileno
(
fp
),
&
sb
)
!=
-
1
&&
sb
.
st_size
==
0
)
{
if
(
fp
&&
redis_fstat
(
fileno
(
fp
),
&
sb
)
!=
-
1
&&
sb
.
st_size
==
0
)
{
server
.
appendonly_current_size
=
0
;
fclose
(
fp
);
fclose
(
fp
);
return
REDIS_ERR
;
return
REDIS_ERR
;
}
}
...
@@ -299,6 +303,7 @@ int loadAppendOnlyFile(char *filename) {
...
@@ -299,6 +303,7 @@ int loadAppendOnlyFile(char *filename) {
freeFakeClient
(
fakeClient
);
freeFakeClient
(
fakeClient
);
server
.
appendonly
=
appendonly
;
server
.
appendonly
=
appendonly
;
stopLoading
();
stopLoading
();
aofUpdateCurrentSize
();
return
REDIS_OK
;
return
REDIS_OK
;
readerr:
readerr:
...
@@ -611,9 +616,10 @@ int rewriteAppendOnlyFileBackground(void) {
...
@@ -611,9 +616,10 @@ int rewriteAppendOnlyFileBackground(void) {
void
bgrewriteaofCommand
(
redisClient
*
c
)
{
void
bgrewriteaofCommand
(
redisClient
*
c
)
{
if
(
server
.
bgrewritechildpid
!=
-
1
)
{
if
(
server
.
bgrewritechildpid
!=
-
1
)
{
addReplyError
(
c
,
"Background append only file rewriting already in progress"
);
addReplyError
(
c
,
"Background append only file rewriting already in progress"
);
return
;
}
else
if
(
server
.
bgsavechildpid
!=
-
1
)
{
}
server
.
aofrewrite_scheduled
=
1
;
if
(
rewriteAppendOnlyFileBackground
()
==
REDIS_OK
)
{
addReplyStatus
(
c
,
"Background append only file rewriting started"
);
}
else
if
(
rewriteAppendOnlyFileBackground
()
==
REDIS_OK
)
{
addReplyStatus
(
c
,
"Background append only file rewriting started"
);
addReplyStatus
(
c
,
"Background append only file rewriting started"
);
}
else
{
}
else
{
addReply
(
c
,
shared
.
err
);
addReply
(
c
,
shared
.
err
);
...
@@ -627,6 +633,21 @@ void aofRemoveTempFile(pid_t childpid) {
...
@@ -627,6 +633,21 @@ void aofRemoveTempFile(pid_t childpid) {
unlink
(
tmpfile
);
unlink
(
tmpfile
);
}
}
/* Update the server.appendonly_current_size filed explicitly using stat(2)
* to check the size of the file. This is useful after a rewrite or after
* a restart, normally the size is updated just adding the write length
* to the current lenght, that is much faster. */
void
aofUpdateCurrentSize
(
void
)
{
struct
redis_stat
sb
;
if
(
redis_fstat
(
server
.
appendfd
,
&
sb
)
==
-
1
)
{
redisLog
(
REDIS_WARNING
,
"Unable to check the AOF length: %s"
,
strerror
(
errno
));
}
else
{
server
.
appendonly_current_size
=
sb
.
st_size
;
}
}
/* A background append only file rewriting (BGREWRITEAOF) terminated its work.
/* A background append only file rewriting (BGREWRITEAOF) terminated its work.
* Handle this. */
* Handle this. */
void
backgroundRewriteDoneHandler
(
int
exitcode
,
int
bysignal
)
{
void
backgroundRewriteDoneHandler
(
int
exitcode
,
int
bysignal
)
{
...
@@ -667,6 +688,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
...
@@ -667,6 +688,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
if
(
server
.
appendfsync
!=
APPENDFSYNC_NO
)
aof_fsync
(
fd
);
if
(
server
.
appendfsync
!=
APPENDFSYNC_NO
)
aof_fsync
(
fd
);
server
.
appendseldb
=
-
1
;
/* Make sure it will issue SELECT */
server
.
appendseldb
=
-
1
;
/* Make sure it will issue SELECT */
redisLog
(
REDIS_NOTICE
,
"The new append only file was selected for future appends."
);
redisLog
(
REDIS_NOTICE
,
"The new append only file was selected for future appends."
);
aofUpdateCurrentSize
();
}
else
{
}
else
{
/* If append only is disabled we just generate a dump in this
/* If append only is disabled we just generate a dump in this
* format. Why not? */
* format. Why not? */
...
...
src/config.c
View file @
b333e239
...
@@ -231,6 +231,18 @@ void loadServerConfig(char *filename) {
...
@@ -231,6 +231,18 @@ void loadServerConfig(char *filename) {
err
=
"argument must be 'no', 'always' or 'everysec'"
;
err
=
"argument must be 'no', 'always' or 'everysec'"
;
goto
loaderr
;
goto
loaderr
;
}
}
}
else
if
(
!
strcasecmp
(
argv
[
0
],
"auto-aof-rewrite-percentage"
)
&&
argc
==
2
)
{
server
.
auto_aofrewrite_perc
=
atoi
(
argv
[
2
]);
if
(
server
.
auto_aofrewrite_perc
<
0
)
{
err
=
"Invalid negative percentage for AOF auto rewrite"
;
goto
loaderr
;
}
}
else
if
(
!
strcasecmp
(
argv
[
0
],
"auto-aof-rewrite-min-size"
)
&&
argc
==
2
)
{
server
.
auto_aofrewrite_min_size
=
memtoll
(
argv
[
1
],
NULL
);
}
else
if
(
!
strcasecmp
(
argv
[
0
],
"requirepass"
)
&&
argc
==
2
)
{
}
else
if
(
!
strcasecmp
(
argv
[
0
],
"requirepass"
)
&&
argc
==
2
)
{
server
.
requirepass
=
zstrdup
(
argv
[
1
]);
server
.
requirepass
=
zstrdup
(
argv
[
1
]);
}
else
if
(
!
strcasecmp
(
argv
[
0
],
"pidfile"
)
&&
argc
==
2
)
{
}
else
if
(
!
strcasecmp
(
argv
[
0
],
"pidfile"
)
&&
argc
==
2
)
{
...
...
src/rdb.c
View file @
b333e239
...
@@ -1039,6 +1039,9 @@ void bgsaveCommand(redisClient *c) {
...
@@ -1039,6 +1039,9 @@ void bgsaveCommand(redisClient *c) {
if
(
server
.
bgsavechildpid
!=
-
1
||
server
.
bgsavethread
!=
(
pthread_t
)
-
1
)
{
if
(
server
.
bgsavechildpid
!=
-
1
||
server
.
bgsavethread
!=
(
pthread_t
)
-
1
)
{
addReplyError
(
c
,
"Background save already in progress"
);
addReplyError
(
c
,
"Background save already in progress"
);
return
;
return
;
}
else
if
(
server
.
bgrewritechildpid
!=
-
1
)
{
addReplyError
(
c
,
"Can't BGSAVE while AOF log rewriting is in progress"
);
return
;
}
}
if
(
rdbSaveBackground
(
server
.
dbfilename
)
==
REDIS_OK
)
{
if
(
rdbSaveBackground
(
server
.
dbfilename
)
==
REDIS_OK
)
{
addReplyStatus
(
c
,
"Background saving started"
);
addReplyStatus
(
c
,
"Background saving started"
);
...
...
src/redis.c
View file @
b333e239
...
@@ -633,6 +633,14 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
...
@@ -633,6 +633,14 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
if
((
server
.
maxidletime
&&
!
(
loops
%
100
))
||
server
.
bpop_blocked_clients
)
if
((
server
.
maxidletime
&&
!
(
loops
%
100
))
||
server
.
bpop_blocked_clients
)
closeTimedoutClients
();
closeTimedoutClients
();
/* Start a scheduled AOF rewrite if this was requested by the user while
* a BGSAVE was in progress. */
if
(
server
.
bgsavechildpid
==
-
1
&&
server
.
bgrewritechildpid
==
-
1
&&
server
.
aofrewrite_scheduled
)
{
rewriteAppendOnlyFileBackground
();
}
/* Check if a background saving or AOF rewrite in progress terminated. */
/* Check if a background saving or AOF rewrite in progress terminated. */
if
(
server
.
bgsavechildpid
!=
-
1
||
server
.
bgrewritechildpid
!=
-
1
)
{
if
(
server
.
bgsavechildpid
!=
-
1
||
server
.
bgrewritechildpid
!=
-
1
)
{
int
statloc
;
int
statloc
;
...
@@ -667,9 +675,10 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
...
@@ -667,9 +675,10 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
}
}
}
}
}
else
if
(
!
server
.
ds_enabled
)
{
}
else
if
(
!
server
.
ds_enabled
)
{
/* If there is not a background saving in progress check if
* we have to save now */
time_t
now
=
time
(
NULL
);
time_t
now
=
time
(
NULL
);
/* If there is not a background saving/rewrite in progress check if
* we have to save/rewrite now */
for
(
j
=
0
;
j
<
server
.
saveparamslen
;
j
++
)
{
for
(
j
=
0
;
j
<
server
.
saveparamslen
;
j
++
)
{
struct
saveparam
*
sp
=
server
.
saveparams
+
j
;
struct
saveparam
*
sp
=
server
.
saveparams
+
j
;
...
@@ -681,6 +690,18 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
...
@@ -681,6 +690,18 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
break
;
break
;
}
}
}
}
/* Trigger an AOF rewrite if needed */
if
(
server
.
auto_aofrewrite_perc
&&
server
.
appendonly_current_size
>
server
.
auto_aofrewrite_min_size
)
{
int
growth
=
(
server
.
appendonly_current_size
*
100
/
server
.
auto_aofrewrite_base_size
);
if
(
growth
>=
server
.
auto_aofrewrite_perc
)
{
redisLog
(
REDIS_NOTICE
,
"Starting automatic rewriting of AOF on %d growth"
,
growth
);
rewriteAppendOnlyFileBackground
();
}
}
}
}
/* Expire a few keys per cycle, only if this is a master.
/* Expire a few keys per cycle, only if this is a master.
...
@@ -828,6 +849,10 @@ void initServerConfig() {
...
@@ -828,6 +849,10 @@ void initServerConfig() {
server
.
appendonly
=
0
;
server
.
appendonly
=
0
;
server
.
appendfsync
=
APPENDFSYNC_EVERYSEC
;
server
.
appendfsync
=
APPENDFSYNC_EVERYSEC
;
server
.
no_appendfsync_on_rewrite
=
0
;
server
.
no_appendfsync_on_rewrite
=
0
;
server
.
auto_aofrewrite_perc
=
REDIS_AUTO_AOFREWRITE_PERC
;
server
.
auto_aofrewrite_min_size
=
REDIS_AUTO_AOFREWRITE_MIN_SIZE
;
server
.
auto_aofrewrite_base_size
=
0
;
server
.
aofrewrite_scheduled
=
0
;
server
.
lastfsync
=
time
(
NULL
);
server
.
lastfsync
=
time
(
NULL
);
server
.
appendfd
=
-
1
;
server
.
appendfd
=
-
1
;
server
.
appendseldb
=
-
1
;
/* Make sure the first time will not match */
server
.
appendseldb
=
-
1
;
/* Make sure the first time will not match */
...
...
src/redis.h
View file @
b333e239
...
@@ -50,6 +50,8 @@
...
@@ -50,6 +50,8 @@
#define REDIS_SHARED_INTEGERS 10000
#define REDIS_SHARED_INTEGERS 10000
#define REDIS_REPLY_CHUNK_BYTES (5*1500)
/* 5 TCP packets with default MTU */
#define REDIS_REPLY_CHUNK_BYTES (5*1500)
/* 5 TCP packets with default MTU */
#define REDIS_MAX_LOGMSG_LEN 1024
/* Default maximum length of syslog messages */
#define REDIS_MAX_LOGMSG_LEN 1024
/* Default maximum length of syslog messages */
#define REDIS_AUTO_AOFREWRITE_PERC 100
#define REDIS_AUTO_AOFREWRITE_MIN_SIZE (1024*1024)
/* Hash table parameters */
/* Hash table parameters */
#define REDIS_HT_MINFILL 10
/* Minimal hash table fill 10% */
#define REDIS_HT_MINFILL 10
/* Minimal hash table fill 10% */
...
@@ -555,6 +557,11 @@ struct redisServer {
...
@@ -555,6 +557,11 @@ struct redisServer {
int
appendonly
;
int
appendonly
;
int
appendfsync
;
int
appendfsync
;
int
no_appendfsync_on_rewrite
;
int
no_appendfsync_on_rewrite
;
int
auto_aofrewrite_perc
;
/* Rewrite AOF if % growth is > M and... */
off_t
auto_aofrewrite_min_size
;
/* the AOF file is at least N bytes. */
off_t
auto_aofrewrite_base_size
;
/* AOF size on latest startup or rewrite. */
off_t
appendonly_current_size
;
/* AOF current size. */
int
aofrewrite_scheduled
;
/* Rewrite once BGSAVE terminates. */
int
shutdown_asap
;
int
shutdown_asap
;
int
activerehashing
;
int
activerehashing
;
char
*
requirepass
;
char
*
requirepass
;
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment