Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
97e7f8ae
Commit
97e7f8ae
authored
Nov 08, 2010
by
antirez
Browse files
non blocking loading of the DB / AOF with informations and ETA in INFO output
parent
57c9babd
Changes
4
Hide whitespace changes
Inline
Side-by-side
src/aof.c
View file @
97e7f8ae
...
@@ -218,6 +218,7 @@ int loadAppendOnlyFile(char *filename) {
...
@@ -218,6 +218,7 @@ int loadAppendOnlyFile(char *filename) {
FILE
*
fp
=
fopen
(
filename
,
"r"
);
FILE
*
fp
=
fopen
(
filename
,
"r"
);
struct
redis_stat
sb
;
struct
redis_stat
sb
;
int
appendonly
=
server
.
appendonly
;
int
appendonly
=
server
.
appendonly
;
long
loops
=
0
;
if
(
redis_fstat
(
fileno
(
fp
),
&
sb
)
!=
-
1
&&
sb
.
st_size
==
0
)
if
(
redis_fstat
(
fileno
(
fp
),
&
sb
)
!=
-
1
&&
sb
.
st_size
==
0
)
return
REDIS_ERR
;
return
REDIS_ERR
;
...
@@ -232,6 +233,8 @@ int loadAppendOnlyFile(char *filename) {
...
@@ -232,6 +233,8 @@ int loadAppendOnlyFile(char *filename) {
server
.
appendonly
=
0
;
server
.
appendonly
=
0
;
fakeClient
=
createFakeClient
();
fakeClient
=
createFakeClient
();
startLoading
(
fp
);
while
(
1
)
{
while
(
1
)
{
int
argc
,
j
;
int
argc
,
j
;
unsigned
long
len
;
unsigned
long
len
;
...
@@ -241,6 +244,12 @@ int loadAppendOnlyFile(char *filename) {
...
@@ -241,6 +244,12 @@ int loadAppendOnlyFile(char *filename) {
struct
redisCommand
*
cmd
;
struct
redisCommand
*
cmd
;
int
force_swapout
;
int
force_swapout
;
/* Serve the clients from time to time */
if
(
!
(
loops
++
%
1000
))
{
loadingProgress
(
ftello
(
fp
));
aeProcessEvents
(
server
.
el
,
AE_FILE_EVENTS
|
AE_DONT_WAIT
);
}
if
(
fgets
(
buf
,
sizeof
(
buf
),
fp
)
==
NULL
)
{
if
(
fgets
(
buf
,
sizeof
(
buf
),
fp
)
==
NULL
)
{
if
(
feof
(
fp
))
if
(
feof
(
fp
))
break
;
break
;
...
@@ -297,6 +306,7 @@ int loadAppendOnlyFile(char *filename) {
...
@@ -297,6 +306,7 @@ int loadAppendOnlyFile(char *filename) {
fclose
(
fp
);
fclose
(
fp
);
freeFakeClient
(
fakeClient
);
freeFakeClient
(
fakeClient
);
server
.
appendonly
=
appendonly
;
server
.
appendonly
=
appendonly
;
stopLoading
();
return
REDIS_OK
;
return
REDIS_OK
;
readerr:
readerr:
...
...
src/rdb.c
View file @
97e7f8ae
...
@@ -7,6 +7,7 @@
...
@@ -7,6 +7,7 @@
#include <sys/resource.h>
#include <sys/resource.h>
#include <sys/wait.h>
#include <sys/wait.h>
#include <arpa/inet.h>
#include <arpa/inet.h>
#include <sys/stat.h>
int
rdbSaveType
(
FILE
*
fp
,
unsigned
char
type
)
{
int
rdbSaveType
(
FILE
*
fp
,
unsigned
char
type
)
{
if
(
fwrite
(
&
type
,
1
,
1
,
fp
)
==
0
)
return
-
1
;
if
(
fwrite
(
&
type
,
1
,
1
,
fp
)
==
0
)
return
-
1
;
...
@@ -793,6 +794,31 @@ robj *rdbLoadObject(int type, FILE *fp) {
...
@@ -793,6 +794,31 @@ robj *rdbLoadObject(int type, FILE *fp) {
return
o
;
return
o
;
}
}
/* Mark that we are loading in the global state and setup the fields
* needed to provide loading stats. */
void
startLoading
(
FILE
*
fp
)
{
struct
stat
sb
;
/* Load the DB */
server
.
loading
=
1
;
server
.
loading_start_time
=
time
(
NULL
);
if
(
fstat
(
fileno
(
fp
),
&
sb
)
==
-
1
)
{
server
.
loading_total_bytes
=
1
;
/* just to avoid division by zero */
}
else
{
server
.
loading_total_bytes
=
sb
.
st_size
;
}
}
/* Refresh the loading progress info */
void
loadingProgress
(
off_t
pos
)
{
server
.
loading_loaded_bytes
=
pos
;
}
/* Loading finished */
void
stopLoading
(
void
)
{
server
.
loading
=
0
;
}
int
rdbLoad
(
char
*
filename
)
{
int
rdbLoad
(
char
*
filename
)
{
FILE
*
fp
;
FILE
*
fp
;
uint32_t
dbid
;
uint32_t
dbid
;
...
@@ -801,6 +827,7 @@ int rdbLoad(char *filename) {
...
@@ -801,6 +827,7 @@ int rdbLoad(char *filename) {
redisDb
*
db
=
server
.
db
+
0
;
redisDb
*
db
=
server
.
db
+
0
;
char
buf
[
1024
];
char
buf
[
1024
];
time_t
expiretime
,
now
=
time
(
NULL
);
time_t
expiretime
,
now
=
time
(
NULL
);
long
loops
=
0
;
fp
=
fopen
(
filename
,
"r"
);
fp
=
fopen
(
filename
,
"r"
);
if
(
!
fp
)
return
REDIS_ERR
;
if
(
!
fp
)
return
REDIS_ERR
;
...
@@ -817,11 +844,20 @@ int rdbLoad(char *filename) {
...
@@ -817,11 +844,20 @@ int rdbLoad(char *filename) {
redisLog
(
REDIS_WARNING
,
"Can't handle RDB format version %d"
,
rdbver
);
redisLog
(
REDIS_WARNING
,
"Can't handle RDB format version %d"
,
rdbver
);
return
REDIS_ERR
;
return
REDIS_ERR
;
}
}
startLoading
(
fp
);
while
(
1
)
{
while
(
1
)
{
robj
*
key
,
*
val
;
robj
*
key
,
*
val
;
int
force_swapout
;
int
force_swapout
;
expiretime
=
-
1
;
expiretime
=
-
1
;
/* Serve the clients from time to time */
if
(
!
(
loops
++
%
1000
))
{
loadingProgress
(
ftello
(
fp
));
aeProcessEvents
(
server
.
el
,
AE_FILE_EVENTS
|
AE_DONT_WAIT
);
}
/* Read type. */
/* Read type. */
if
((
type
=
rdbLoadType
(
fp
))
==
-
1
)
goto
eoferr
;
if
((
type
=
rdbLoadType
(
fp
))
==
-
1
)
goto
eoferr
;
if
(
type
==
REDIS_EXPIRETIME
)
{
if
(
type
==
REDIS_EXPIRETIME
)
{
...
@@ -900,6 +936,7 @@ int rdbLoad(char *filename) {
...
@@ -900,6 +936,7 @@ int rdbLoad(char *filename) {
}
}
}
}
fclose
(
fp
);
fclose
(
fp
);
stopLoading
();
return
REDIS_OK
;
return
REDIS_OK
;
eoferr:
/* unexpected end of file is handled here with a fatal exit */
eoferr:
/* unexpected end of file is handled here with a fatal exit */
...
...
src/redis.c
View file @
97e7f8ae
...
@@ -702,6 +702,8 @@ void createSharedObjects(void) {
...
@@ -702,6 +702,8 @@ void createSharedObjects(void) {
"-ERR source and destination objects are the same
\r\n
"
));
"-ERR source and destination objects are the same
\r\n
"
));
shared
.
outofrangeerr
=
createObject
(
REDIS_STRING
,
sdsnew
(
shared
.
outofrangeerr
=
createObject
(
REDIS_STRING
,
sdsnew
(
"-ERR index out of range
\r\n
"
));
"-ERR index out of range
\r\n
"
));
shared
.
loadingerr
=
createObject
(
REDIS_STRING
,
sdsnew
(
"-LOADING Redis is loading the dataset in memory
\r\n
"
));
shared
.
space
=
createObject
(
REDIS_STRING
,
sdsnew
(
" "
));
shared
.
space
=
createObject
(
REDIS_STRING
,
sdsnew
(
" "
));
shared
.
colon
=
createObject
(
REDIS_STRING
,
sdsnew
(
":"
));
shared
.
colon
=
createObject
(
REDIS_STRING
,
sdsnew
(
":"
));
shared
.
plus
=
createObject
(
REDIS_STRING
,
sdsnew
(
"+"
));
shared
.
plus
=
createObject
(
REDIS_STRING
,
sdsnew
(
"+"
));
...
@@ -739,6 +741,7 @@ void initServerConfig() {
...
@@ -739,6 +741,7 @@ void initServerConfig() {
server
.
verbosity
=
REDIS_VERBOSE
;
server
.
verbosity
=
REDIS_VERBOSE
;
server
.
maxidletime
=
REDIS_MAXIDLETIME
;
server
.
maxidletime
=
REDIS_MAXIDLETIME
;
server
.
saveparams
=
NULL
;
server
.
saveparams
=
NULL
;
server
.
loading
=
0
;
server
.
logfile
=
NULL
;
/* NULL = log on standard output */
server
.
logfile
=
NULL
;
/* NULL = log on standard output */
server
.
glueoutputbuf
=
1
;
server
.
glueoutputbuf
=
1
;
server
.
daemonize
=
0
;
server
.
daemonize
=
0
;
...
@@ -1006,6 +1009,12 @@ int processCommand(redisClient *c) {
...
@@ -1006,6 +1009,12 @@ int processCommand(redisClient *c) {
return
REDIS_OK
;
return
REDIS_OK
;
}
}
/* Loading DB? Return an error if the command is not INFO */
if
(
server
.
loading
&&
cmd
->
proc
!=
infoCommand
)
{
addReply
(
c
,
shared
.
loadingerr
);
return
REDIS_OK
;
}
/* Exec the command */
/* Exec the command */
if
(
c
->
flags
&
REDIS_MULTI
&&
if
(
c
->
flags
&
REDIS_MULTI
&&
cmd
->
proc
!=
execCommand
&&
cmd
->
proc
!=
discardCommand
&&
cmd
->
proc
!=
execCommand
&&
cmd
->
proc
!=
discardCommand
&&
...
@@ -1133,6 +1142,8 @@ sds genRedisInfoString(void) {
...
@@ -1133,6 +1142,8 @@ sds genRedisInfoString(void) {
"used_memory_rss:%zu
\r\n
"
"used_memory_rss:%zu
\r\n
"
"mem_fragmentation_ratio:%.2f
\r\n
"
"mem_fragmentation_ratio:%.2f
\r\n
"
"use_tcmalloc:%d
\r\n
"
"use_tcmalloc:%d
\r\n
"
"loading:%d
\r\n
"
"aof_enabled:%d
\r\n
"
"changes_since_last_save:%lld
\r\n
"
"changes_since_last_save:%lld
\r\n
"
"bgsave_in_progress:%d
\r\n
"
"bgsave_in_progress:%d
\r\n
"
"last_save_time:%ld
\r\n
"
"last_save_time:%ld
\r\n
"
...
@@ -1173,6 +1184,8 @@ sds genRedisInfoString(void) {
...
@@ -1173,6 +1184,8 @@ sds genRedisInfoString(void) {
#else
#else
0
,
0
,
#endif
#endif
server
.
loading
,
server
.
appendonly
,
server
.
dirty
,
server
.
dirty
,
server
.
bgsavechildpid
!=
-
1
,
server
.
bgsavechildpid
!=
-
1
,
server
.
lastsave
,
server
.
lastsave
,
...
@@ -1243,6 +1256,35 @@ sds genRedisInfoString(void) {
...
@@ -1243,6 +1256,35 @@ sds genRedisInfoString(void) {
);
);
unlockThreadedIO
();
unlockThreadedIO
();
}
}
if
(
server
.
loading
)
{
double
perc
;
time_t
eta
,
elapsed
;
off_t
remaining_bytes
=
server
.
loading_total_bytes
-
server
.
loading_loaded_bytes
;
perc
=
((
double
)
server
.
loading_loaded_bytes
/
server
.
loading_total_bytes
)
*
100
;
elapsed
=
time
(
NULL
)
-
server
.
loading_start_time
;
if
(
elapsed
==
0
)
{
eta
=
1
;
/* A fake 1 second figure if we don't have enough info */
}
else
{
eta
=
(
elapsed
*
remaining_bytes
)
/
server
.
loading_loaded_bytes
;
}
info
=
sdscatprintf
(
info
,
"loading_start_time:%ld
\r\n
"
"loading_total_bytes:%llu
\r\n
"
"loading_loaded_bytes:%llu
\r\n
"
"loading_loaded_perc:%.2f
\r\n
"
"loading_eta_seconds:%ld
\r\n
"
,(
unsigned
long
)
server
.
loading_start_time
,
(
unsigned
long
long
)
server
.
loading_total_bytes
,
(
unsigned
long
long
)
server
.
loading_loaded_bytes
,
perc
,
eta
);
}
for
(
j
=
0
;
j
<
server
.
dbnum
;
j
++
)
{
for
(
j
=
0
;
j
<
server
.
dbnum
;
j
++
)
{
long
long
keys
,
vkeys
;
long
long
keys
,
vkeys
;
...
...
src/redis.h
View file @
97e7f8ae
...
@@ -340,7 +340,7 @@ struct sharedObjectsStruct {
...
@@ -340,7 +340,7 @@ struct sharedObjectsStruct {
robj
*
crlf
,
*
ok
,
*
err
,
*
emptybulk
,
*
czero
,
*
cone
,
*
cnegone
,
*
pong
,
*
space
,
robj
*
crlf
,
*
ok
,
*
err
,
*
emptybulk
,
*
czero
,
*
cone
,
*
cnegone
,
*
pong
,
*
space
,
*
colon
,
*
nullbulk
,
*
nullmultibulk
,
*
queued
,
*
colon
,
*
nullbulk
,
*
nullmultibulk
,
*
queued
,
*
emptymultibulk
,
*
wrongtypeerr
,
*
nokeyerr
,
*
syntaxerr
,
*
sameobjecterr
,
*
emptymultibulk
,
*
wrongtypeerr
,
*
nokeyerr
,
*
syntaxerr
,
*
sameobjecterr
,
*
outofrangeerr
,
*
plus
,
*
outofrangeerr
,
*
loadingerr
,
*
plus
,
*
select0
,
*
select1
,
*
select2
,
*
select3
,
*
select4
,
*
select0
,
*
select1
,
*
select2
,
*
select3
,
*
select4
,
*
select5
,
*
select6
,
*
select7
,
*
select8
,
*
select9
,
*
select5
,
*
select6
,
*
select7
,
*
select8
,
*
select9
,
*
messagebulk
,
*
pmessagebulk
,
*
subscribebulk
,
*
unsubscribebulk
,
*
mbulk3
,
*
messagebulk
,
*
pmessagebulk
,
*
subscribebulk
,
*
unsubscribebulk
,
*
mbulk3
,
...
@@ -361,6 +361,11 @@ struct redisServer {
...
@@ -361,6 +361,11 @@ struct redisServer {
long
long
dirty_before_bgsave
;
/* used to restore dirty on failed BGSAVE */
long
long
dirty_before_bgsave
;
/* used to restore dirty on failed BGSAVE */
list
*
clients
;
list
*
clients
;
dict
*
commands
;
/* Command table hahs table */
dict
*
commands
;
/* Command table hahs table */
/* RDB / AOF loading information */
int
loading
;
off_t
loading_total_bytes
;
off_t
loading_loaded_bytes
;
time_t
loading_start_time
;
/* Fast pointers to often looked up command */
/* Fast pointers to often looked up command */
struct
redisCommand
*
delCommand
,
*
multiCommand
;
struct
redisCommand
*
delCommand
,
*
multiCommand
;
list
*
slaves
,
*
monitors
;
list
*
slaves
,
*
monitors
;
...
@@ -726,6 +731,11 @@ int syncWithMaster(void);
...
@@ -726,6 +731,11 @@ int syncWithMaster(void);
void
updateSlavesWaitingBgsave
(
int
bgsaveerr
);
void
updateSlavesWaitingBgsave
(
int
bgsaveerr
);
void
replicationCron
(
void
);
void
replicationCron
(
void
);
/* Generic persistence functions */
void
startLoading
(
FILE
*
fp
);
void
loadingProgress
(
off_t
pos
);
void
stopLoading
(
void
);
/* RDB persistence */
/* RDB persistence */
int
rdbLoad
(
char
*
filename
);
int
rdbLoad
(
char
*
filename
);
int
rdbSaveBackground
(
char
*
filename
);
int
rdbSaveBackground
(
char
*
filename
);
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment