Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
fe63046e
Commit
fe63046e
authored
Mar 02, 2015
by
Oran Agra
Browse files
diskless replication on slave side
parent
c77081a4
Changes
14
Hide whitespace changes
Inline
Side-by-side
src/anet.c
View file @
fe63046e
...
...
@@ -193,6 +193,20 @@ int anetSendTimeout(char *err, int fd, long long ms) {
return
ANET_OK
;
}
/* Set the socket receive timeout (SO_RCVTIMEO socket option) to the specified
* number of milliseconds, or disable it if the 'ms' argument is zero. */
int
anetRecvTimeout
(
char
*
err
,
int
fd
,
long
long
ms
)
{
struct
timeval
tv
;
tv
.
tv_sec
=
ms
/
1000
;
tv
.
tv_usec
=
(
ms
%
1000
)
*
1000
;
if
(
setsockopt
(
fd
,
SOL_SOCKET
,
SO_RCVTIMEO
,
&
tv
,
sizeof
(
tv
))
==
-
1
)
{
anetSetError
(
err
,
"setsockopt SO_RCVTIMEO: %s"
,
strerror
(
errno
));
return
ANET_ERR
;
}
return
ANET_OK
;
}
/* anetGenericResolve() is called by anetResolve() and anetResolveIP() to
* do the actual work. It resolves the hostname "host" and set the string
* representation of the IP address into the buffer pointed by "ipbuf".
...
...
src/anet.h
View file @
fe63046e
...
...
@@ -67,6 +67,7 @@ int anetEnableTcpNoDelay(char *err, int fd);
int
anetDisableTcpNoDelay
(
char
*
err
,
int
fd
);
int
anetTcpKeepAlive
(
char
*
err
,
int
fd
);
int
anetSendTimeout
(
char
*
err
,
int
fd
,
long
long
ms
);
int
anetRecvTimeout
(
char
*
err
,
int
fd
,
long
long
ms
);
int
anetPeerToString
(
int
fd
,
char
*
ip
,
size_t
ip_len
,
int
*
port
);
int
anetKeepAlive
(
char
*
err
,
int
fd
,
int
interval
);
int
anetSockName
(
int
fd
,
char
*
ip
,
size_t
ip_len
,
int
*
port
);
...
...
src/aof.c
View file @
fe63046e
...
...
@@ -620,7 +620,7 @@ int loadAppendOnlyFile(char *filename) {
server
.
aof_state
=
REDIS_AOF_OFF
;
fakeClient
=
createFakeClient
();
startLoading
(
fp
);
startLoading
File
(
fp
);
while
(
1
)
{
int
argc
,
j
;
...
...
src/rdb.c
View file @
fe63046e
...
...
@@ -1187,18 +1187,19 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
/* Mark that we are loading in the global state and setup the fields
* needed to provide loading stats. */
void
startLoading
(
FILE
*
fp
)
{
struct
stat
sb
;
void
startLoading
(
size_t
size
)
{
/* Load the DB */
server
.
loading
=
1
;
server
.
loading_start_time
=
time
(
NULL
);
server
.
loading_loaded_bytes
=
0
;
if
(
fstat
(
fileno
(
fp
),
&
sb
)
==
-
1
)
{
server
.
loading_total_bytes
=
0
;
}
else
{
server
.
loading_total_bytes
=
sb
.
st_size
;
}
server
.
loading_total_bytes
=
size
;
}
void
startLoadingFile
(
FILE
*
fp
)
{
struct
stat
sb
;
if
(
fstat
(
fileno
(
fp
),
&
sb
)
==
-
1
)
sb
.
st_size
=
0
;
startLoading
(
sb
.
st_size
);
}
/* Refresh the loading progress info */
...
...
@@ -1233,66 +1234,74 @@ void rdbLoadProgressCallback(rio *r, const void *buf, size_t len) {
}
int
rdbLoad
(
char
*
filename
)
{
FILE
*
fp
;
int
retval
;
rio
rdb
;
struct
stat
sb
;
if
((
fp
=
fopen
(
filename
,
"r"
))
==
NULL
)
return
REDIS_ERR
;
if
(
fstat
(
fileno
(
fp
),
&
sb
)
==
-
1
)
sb
.
st_size
=
0
;
rioInitWithFile
(
&
rdb
,
fp
);
retval
=
rdbLoadRio
(
&
rdb
,
sb
.
st_size
);
fclose
(
fp
);
return
retval
;
}
int
rdbLoadRio
(
rio
*
rdb
,
size_t
size
)
{
uint32_t
dbid
;
int
type
,
rdbver
;
redisDb
*
db
=
server
.
db
+
0
;
char
buf
[
1024
];
long
long
expiretime
,
now
=
mstime
();
FILE
*
fp
;
rio
rdb
;
if
((
fp
=
fopen
(
filename
,
"r"
))
==
NULL
)
return
REDIS_ERR
;
rioInitWithFile
(
&
rdb
,
fp
);
rdb
.
update_cksum
=
rdbLoadProgressCallback
;
rdb
.
max_processing_chunk
=
server
.
loading_process_events_interval_bytes
;
if
(
rioRead
(
&
rdb
,
buf
,
9
)
==
0
)
goto
eoferr
;
rdb
->
update_cksum
=
rdbLoadProgressCallback
;
rdb
->
max_processing_chunk
=
server
.
loading_process_events_interval_bytes
;
if
(
rioRead
(
rdb
,
buf
,
9
)
==
0
)
goto
eoferr
;
buf
[
9
]
=
'\0'
;
if
(
memcmp
(
buf
,
"REDIS"
,
5
)
!=
0
)
{
fclose
(
fp
);
redisLog
(
REDIS_WARNING
,
"Wrong signature trying to load DB from file"
);
errno
=
EINVAL
;
return
REDIS_ERR
;
}
rdbver
=
atoi
(
buf
+
5
);
if
(
rdbver
<
1
||
rdbver
>
REDIS_RDB_VERSION
)
{
fclose
(
fp
);
redisLog
(
REDIS_WARNING
,
"Can't handle RDB format version %d"
,
rdbver
);
errno
=
EINVAL
;
return
REDIS_ERR
;
}
startLoading
(
fp
);
startLoading
(
size
);
while
(
1
)
{
robj
*
key
,
*
val
;
expiretime
=
-
1
;
/* Read type. */
if
((
type
=
rdbLoadType
(
&
rdb
))
==
-
1
)
goto
eoferr
;
if
((
type
=
rdbLoadType
(
rdb
))
==
-
1
)
goto
eoferr
;
/* Handle special types. */
if
(
type
==
REDIS_RDB_OPCODE_EXPIRETIME
)
{
/* EXPIRETIME: load an expire associated with the next key
* to load. Note that after loading an expire we need to
* load the actual type, and continue. */
if
((
expiretime
=
rdbLoadTime
(
&
rdb
))
==
-
1
)
goto
eoferr
;
if
((
expiretime
=
rdbLoadTime
(
rdb
))
==
-
1
)
goto
eoferr
;
/* We read the time so we need to read the object type again. */
if
((
type
=
rdbLoadType
(
&
rdb
))
==
-
1
)
goto
eoferr
;
if
((
type
=
rdbLoadType
(
rdb
))
==
-
1
)
goto
eoferr
;
/* the EXPIRETIME opcode specifies time in seconds, so convert
* into milliseconds. */
expiretime
*=
1000
;
}
else
if
(
type
==
REDIS_RDB_OPCODE_EXPIRETIME_MS
)
{
/* EXPIRETIME_MS: milliseconds precision expire times introduced
* with RDB v3. Like EXPIRETIME but no with more precision. */
if
((
expiretime
=
rdbLoadMillisecondTime
(
&
rdb
))
==
-
1
)
goto
eoferr
;
if
((
expiretime
=
rdbLoadMillisecondTime
(
rdb
))
==
-
1
)
goto
eoferr
;
/* We read the time so we need to read the object type again. */
if
((
type
=
rdbLoadType
(
&
rdb
))
==
-
1
)
goto
eoferr
;
if
((
type
=
rdbLoadType
(
rdb
))
==
-
1
)
goto
eoferr
;
}
else
if
(
type
==
REDIS_RDB_OPCODE_EOF
)
{
/* EOF: End of file, exit the main loop. */
break
;
}
else
if
(
type
==
REDIS_RDB_OPCODE_SELECTDB
)
{
/* SELECTDB: Select the specified database. */
if
((
dbid
=
rdbLoadLen
(
&
rdb
,
NULL
))
==
REDIS_RDB_LENERR
)
if
((
dbid
=
rdbLoadLen
(
rdb
,
NULL
))
==
REDIS_RDB_LENERR
)
goto
eoferr
;
if
(
dbid
>=
(
unsigned
)
server
.
dbnum
)
{
redisLog
(
REDIS_WARNING
,
...
...
@@ -1307,9 +1316,9 @@ int rdbLoad(char *filename) {
/* RESIZEDB: Hint about the size of the keys in the currently
* selected data base, in order to avoid useless rehashing. */
uint32_t
db_size
,
expires_size
;
if
((
db_size
=
rdbLoadLen
(
&
rdb
,
NULL
))
==
REDIS_RDB_LENERR
)
if
((
db_size
=
rdbLoadLen
(
rdb
,
NULL
))
==
REDIS_RDB_LENERR
)
goto
eoferr
;
if
((
expires_size
=
rdbLoadLen
(
&
rdb
,
NULL
))
==
REDIS_RDB_LENERR
)
if
((
expires_size
=
rdbLoadLen
(
rdb
,
NULL
))
==
REDIS_RDB_LENERR
)
goto
eoferr
;
dictExpand
(
db
->
dict
,
db_size
);
dictExpand
(
db
->
expires
,
expires_size
);
...
...
@@ -1321,8 +1330,8 @@ int rdbLoad(char *filename) {
*
* An AUX field is composed of two strings: key and value. */
robj
*
auxkey
,
*
auxval
;
if
((
auxkey
=
rdbLoadStringObject
(
&
rdb
))
==
NULL
)
goto
eoferr
;
if
((
auxval
=
rdbLoadStringObject
(
&
rdb
))
==
NULL
)
goto
eoferr
;
if
((
auxkey
=
rdbLoadStringObject
(
rdb
))
==
NULL
)
goto
eoferr
;
if
((
auxval
=
rdbLoadStringObject
(
rdb
))
==
NULL
)
goto
eoferr
;
if
(((
char
*
)
auxkey
->
ptr
)[
0
]
==
'%'
)
{
/* All the fields with a name staring with '%' are considered
...
...
@@ -1344,9 +1353,9 @@ int rdbLoad(char *filename) {
}
/* Read key */
if
((
key
=
rdbLoadStringObject
(
&
rdb
))
==
NULL
)
goto
eoferr
;
if
((
key
=
rdbLoadStringObject
(
rdb
))
==
NULL
)
goto
eoferr
;
/* Read value */
if
((
val
=
rdbLoadObject
(
type
,
&
rdb
))
==
NULL
)
goto
eoferr
;
if
((
val
=
rdbLoadObject
(
type
,
rdb
))
==
NULL
)
goto
eoferr
;
/* Check if the key already expired. This function is used when loading
* an RDB file from disk, either at startup, or when an RDB was
* received from the master. In the latter case, the master is
...
...
@@ -1367,9 +1376,9 @@ int rdbLoad(char *filename) {
}
/* Verify the checksum if RDB version is >= 5 */
if
(
rdbver
>=
5
&&
server
.
rdb_checksum
)
{
uint64_t
cksum
,
expected
=
rdb
.
cksum
;
uint64_t
cksum
,
expected
=
rdb
->
cksum
;
if
(
rioRead
(
&
rdb
,
&
cksum
,
8
)
==
0
)
goto
eoferr
;
if
(
rioRead
(
rdb
,
&
cksum
,
8
)
==
0
)
goto
eoferr
;
memrev64ifbe
(
&
cksum
);
if
(
cksum
==
0
)
{
redisLog
(
REDIS_WARNING
,
"RDB file was saved with checksum disabled: no check performed."
);
...
...
@@ -1379,7 +1388,6 @@ int rdbLoad(char *filename) {
}
}
fclose
(
fp
);
stopLoading
();
return
REDIS_OK
;
...
...
@@ -1572,7 +1580,7 @@ int rdbSaveToSlavesSockets(void) {
clientids
[
numfds
]
=
slave
->
id
;
fds
[
numfds
++
]
=
slave
->
fd
;
slave
->
replstate
=
REDIS_REPL_WAIT_BGSAVE_END
;
/* Put the socket in
non-
blocking mode to simplify RDB transfer.
/* Put the socket in blocking mode to simplify RDB transfer.
* We'll restore it when the children returns (since duped socket
* will share the O_NONBLOCK attribute with the parent). */
anetBlock
(
NULL
,
slave
->
fd
);
...
...
@@ -1646,6 +1654,7 @@ int rdbSaveToSlavesSockets(void) {
zfree
(
msg
);
}
zfree
(
clientids
);
rioFreeFdset
(
&
slave_sockets
);
exitFromChild
((
retval
==
REDIS_OK
)
?
0
:
1
);
}
else
{
/* Parent */
...
...
src/rdb.h
View file @
fe63046e
...
...
@@ -105,6 +105,7 @@ uint32_t rdbLoadLen(rio *rdb, int *isencoded);
int
rdbSaveObjectType
(
rio
*
rdb
,
robj
*
o
);
int
rdbLoadObjectType
(
rio
*
rdb
);
int
rdbLoad
(
char
*
filename
);
int
rdbLoadRio
(
rio
*
rdb
,
size_t
size
);
int
rdbSaveBackground
(
char
*
filename
);
int
rdbSaveToSlavesSockets
(
void
);
void
rdbRemoveTempFile
(
pid_t
childpid
);
...
...
src/redis.c
View file @
fe63046e
...
...
@@ -1490,6 +1490,9 @@ void initServerConfig(void) {
server
.
cached_master
=
NULL
;
server
.
repl_master_initial_offset
=
-
1
;
server
.
repl_state
=
REDIS_REPL_NONE
;
server
.
repl_transfer_tmpfile
=
NULL
;
server
.
repl_transfer_fd
=
-
1
;
server
.
repl_transfer_s
=
-
1
;
server
.
repl_syncio_timeout
=
REDIS_REPL_SYNCIO_TIMEOUT
;
server
.
repl_serve_stale_data
=
REDIS_DEFAULT_SLAVE_SERVE_STALE_DATA
;
server
.
repl_slave_ro
=
REDIS_DEFAULT_SLAVE_READ_ONLY
;
...
...
src/redis.h
View file @
fe63046e
...
...
@@ -1195,7 +1195,8 @@ long long replicationGetSlaveOffset(void);
char
*
replicationGetSlaveName
(
redisClient
*
c
);
/* Generic persistence functions */
void
startLoading
(
FILE
*
fp
);
void
startLoading
(
size_t
size
);
void
startLoadingFile
(
FILE
*
fp
);
void
loadingProgress
(
off_t
pos
);
void
stopLoading
(
void
);
...
...
src/replication.c
View file @
fe63046e
...
...
@@ -823,9 +823,13 @@ void replicationAbortSyncTransfer(void) {
aeDeleteFileEvent
(
server
.
el
,
server
.
repl_transfer_s
,
AE_READABLE
);
close
(
server
.
repl_transfer_s
);
close
(
server
.
repl_transfer_fd
);
unlink
(
server
.
repl_transfer_tmpfile
);
zfree
(
server
.
repl_transfer_tmpfile
);
if
(
!
server
.
repl_diskless_sync
)
{
close
(
server
.
repl_transfer_fd
);
unlink
(
server
.
repl_transfer_tmpfile
);
zfree
(
server
.
repl_transfer_tmpfile
);
server
.
repl_transfer_tmpfile
=
NULL
;
server
.
repl_transfer_fd
=
-
1
;
}
server
.
repl_state
=
REDIS_REPL_CONNECT
;
}
...
...
@@ -877,6 +881,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) {
char
buf
[
4096
];
ssize_t
nread
,
readlen
;
off_t
left
;
sds
initialCommandStream
=
NULL
;
REDIS_NOTUSED
(
el
);
REDIS_NOTUSED
(
privdata
);
REDIS_NOTUSED
(
mask
);
...
...
@@ -942,118 +947,165 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) {
return
;
}
/* Read bulk data */
if
(
usemark
)
{
readlen
=
sizeof
(
buf
);
}
else
{
left
=
server
.
repl_transfer_size
-
server
.
repl_transfer_read
;
readlen
=
(
left
<
(
signed
)
sizeof
(
buf
))
?
left
:
(
signed
)
sizeof
(
buf
);
}
if
(
!
server
.
repl_diskless_sync
)
{
/* read the data from the socket, store it to a file and search for the EOF */
if
(
usemark
)
{
readlen
=
sizeof
(
buf
);
}
else
{
left
=
server
.
repl_transfer_size
-
server
.
repl_transfer_read
;
readlen
=
(
left
<
(
signed
)
sizeof
(
buf
))
?
left
:
(
signed
)
sizeof
(
buf
);
}
nread
=
read
(
fd
,
buf
,
readlen
);
if
(
nread
<=
0
)
{
redisLog
(
REDIS_WARNING
,
"I/O error trying to sync with MASTER: %s"
,
(
nread
==
-
1
)
?
strerror
(
errno
)
:
"connection lost"
);
replicationAbortSyncTransfer
();
return
;
}
server
.
stat_net_input_bytes
+=
nread
;
nread
=
read
(
fd
,
buf
,
readlen
);
if
(
nread
<=
0
)
{
redisLog
(
REDIS_WARNING
,
"I/O error trying to sync with MASTER: %s"
,
(
nread
==
-
1
)
?
strerror
(
errno
)
:
"connection lost"
);
replicationAbortSyncTransfer
();
return
;
}
server
.
stat_net_input_bytes
+=
nread
;
/* When a mark is used, we want to detect EOF asap in order to avoid
* writing the EOF mark into the file... */
int
eof_reached
=
0
;
/* When a mark is used, we want to detect EOF asap in order to avoid
* writing the EOF mark into the file... */
int
eof_reached
=
0
;
if
(
usemark
)
{
/* Update the last bytes array, and check if it matches our delimiter.*/
if
(
nread
>=
REDIS_RUN_ID_SIZE
)
{
memcpy
(
lastbytes
,
buf
+
nread
-
REDIS_RUN_ID_SIZE
,
REDIS_RUN_ID_SIZE
);
}
else
{
int
rem
=
REDIS_RUN_ID_SIZE
-
nread
;
memmove
(
lastbytes
,
lastbytes
+
nread
,
rem
);
memcpy
(
lastbytes
+
rem
,
buf
,
nread
);
if
(
usemark
)
{
/* Update the last bytes array, and check if it matches our delimiter.*/
if
(
nread
>=
REDIS_RUN_ID_SIZE
)
{
memcpy
(
lastbytes
,
buf
+
nread
-
REDIS_RUN_ID_SIZE
,
REDIS_RUN_ID_SIZE
);
}
else
{
int
rem
=
REDIS_RUN_ID_SIZE
-
nread
;
memmove
(
lastbytes
,
lastbytes
+
nread
,
rem
);
memcpy
(
lastbytes
+
rem
,
buf
,
nread
);
}
if
(
memcmp
(
lastbytes
,
eofmark
,
REDIS_RUN_ID_SIZE
)
==
0
)
eof_reached
=
1
;
}
if
(
memcmp
(
lastbytes
,
eofmark
,
REDIS_RUN_ID_SIZE
)
==
0
)
eof_reached
=
1
;
}
server
.
repl_transfer_lastio
=
server
.
unixtime
;
if
(
write
(
server
.
repl_transfer_fd
,
buf
,
nread
)
!=
nread
)
{
redisLog
(
REDIS_WARNING
,
"Write error or short write writing to the DB dump file needed for MASTER <-> SLAVE synchronization: %s"
,
strerror
(
errno
));
goto
error
;
}
server
.
repl_transfer_read
+=
nread
;
/* Delete the last 40 bytes from the file if we reached EOF. */
if
(
usemark
&&
eof_reached
)
{
if
(
ftruncate
(
server
.
repl_transfer_fd
,
server
.
repl_transfer_read
-
REDIS_RUN_ID_SIZE
)
==
-
1
)
{
redisLog
(
REDIS_WARNING
,
"Error truncating the RDB file received from the master for SYNC: %s"
,
strerror
(
errno
));
server
.
repl_transfer_lastio
=
server
.
unixtime
;
if
(
write
(
server
.
repl_transfer_fd
,
buf
,
nread
)
!=
nread
)
{
redisLog
(
REDIS_WARNING
,
"Write error or short write writing to the DB dump file needed for MASTER <-> SLAVE synchronization: %s"
,
strerror
(
errno
));
goto
error
;
}
}
server
.
repl_transfer_read
+=
nread
;
/* Sync data on disk from time to time, otherwise at the end of the transfer
* we may suffer a big delay as the memory buffers are copied into the
* actual disk. */
if
(
server
.
repl_transfer_read
>=
server
.
repl_transfer_last_fsync_off
+
REPL_MAX_WRITTEN_BEFORE_FSYNC
)
{
off_t
sync_size
=
server
.
repl_transfer_read
-
server
.
repl_transfer_last_fsync_off
;
rdb_fsync_range
(
server
.
repl_transfer_fd
,
server
.
repl_transfer_last_fsync_off
,
sync_size
);
server
.
repl_transfer_last_fsync_off
+=
sync_size
;
}
/* Delete the last 40 bytes from the file if we reached EOF. */
if
(
usemark
&&
eof_reached
)
{
if
(
ftruncate
(
server
.
repl_transfer_fd
,
server
.
repl_transfer_read
-
REDIS_RUN_ID_SIZE
)
==
-
1
)
{
redisLog
(
REDIS_WARNING
,
"Error truncating the RDB file received from the master for SYNC: %s"
,
strerror
(
errno
));
goto
error
;
}
}
/* Check if the transfer is now complete */
if
(
!
usemark
)
{
if
(
server
.
repl_transfer_read
==
server
.
repl_transfer_size
)
eof_reached
=
1
;
/* Sync data on disk from time to time, otherwise at the end of the transfer
* we may suffer a big delay as the memory buffers are copied into the
* actual disk. */
if
(
server
.
repl_transfer_read
>=
server
.
repl_transfer_last_fsync_off
+
REPL_MAX_WRITTEN_BEFORE_FSYNC
)
{
off_t
sync_size
=
server
.
repl_transfer_read
-
server
.
repl_transfer_last_fsync_off
;
rdb_fsync_range
(
server
.
repl_transfer_fd
,
server
.
repl_transfer_last_fsync_off
,
sync_size
);
server
.
repl_transfer_last_fsync_off
+=
sync_size
;
}
/* Check if the transfer is now complete */
if
(
!
usemark
)
{
if
(
server
.
repl_transfer_read
==
server
.
repl_transfer_size
)
eof_reached
=
1
;
}
if
(
!
eof_reached
)
return
;
}
if
(
eof_reached
)
{
/* We reach here when the slave is using diskless replication,
* or when we are done reading from the socket to the rdb file. */
redisLog
(
REDIS_NOTICE
,
"MASTER <-> SLAVE sync: Flushing old data"
);
signalFlushedDb
(
-
1
);
emptyDb
(
replicationEmptyDbCallback
);
/* Before loading the DB into memory we need to delete the readable
* handler, otherwise it will get called recursively since
* rdbLoad() will call the event loop to process events from time to
* time for non blocking loading. */
aeDeleteFileEvent
(
server
.
el
,
server
.
repl_transfer_s
,
AE_READABLE
);
redisLog
(
REDIS_NOTICE
,
"MASTER <-> SLAVE sync: Loading DB in memory"
);
if
(
server
.
repl_diskless_sync
)
{
rio
rdb
;
rioInitWithFd
(
&
rdb
,
fd
);
/* Put the socket in blocking mode to simplify RDB transfer.
* We'll restore it when the RDB is received. */
anetBlock
(
NULL
,
fd
);
anetRecvTimeout
(
NULL
,
fd
,
server
.
repl_timeout
*
1000
);
if
(
rdbLoadRio
(
&
rdb
,
server
.
repl_transfer_size
)
!=
REDIS_OK
)
{
redisLog
(
REDIS_WARNING
,
"Failed trying to load the MASTER synchronization DB from disk"
);
replicationAbortSyncTransfer
();
rioFreeFd
(
&
rdb
,
NULL
);
return
;
}
if
(
usemark
)
{
if
(
!
rioRead
(
&
rdb
,
buf
,
REDIS_RUN_ID_SIZE
)
||
memcmp
(
buf
,
eofmark
,
REDIS_RUN_ID_SIZE
)
!=
0
)
{
redisLog
(
REDIS_WARNING
,
"replication stream EOF marker is broken"
);
replicationAbortSyncTransfer
();
rioFreeFd
(
&
rdb
,
NULL
);
return
;
}
}
/* get the unread command stream from the rio buffer */
rioFreeFd
(
&
rdb
,
&
initialCommandStream
);
/* Restore the socket as non-blocking. */
anetNonBlock
(
NULL
,
fd
);
anetRecvTimeout
(
NULL
,
fd
,
0
);
}
else
{
if
(
rename
(
server
.
repl_transfer_tmpfile
,
server
.
rdb_filename
)
==
-
1
)
{
redisLog
(
REDIS_WARNING
,
"Failed trying to rename the temp DB into dump.rdb in MASTER <-> SLAVE synchronization: %s"
,
strerror
(
errno
));
replicationAbortSyncTransfer
();
return
;
}
redisLog
(
REDIS_NOTICE
,
"MASTER <-> SLAVE sync: Flushing old data"
);
signalFlushedDb
(
-
1
);
emptyDb
(
replicationEmptyDbCallback
);
/* Before loading the DB into memory we need to delete the readable
* handler, otherwise it will get called recursively since
* rdbLoad() will call the event loop to process events from time to
* time for non blocking loading. */
aeDeleteFileEvent
(
server
.
el
,
server
.
repl_transfer_s
,
AE_READABLE
);
redisLog
(
REDIS_NOTICE
,
"MASTER <-> SLAVE sync: Loading DB in memory"
);
if
(
rdbLoad
(
server
.
rdb_filename
)
!=
REDIS_OK
)
{
redisLog
(
REDIS_WARNING
,
"Failed trying to load the MASTER synchronization DB from disk"
);
replicationAbortSyncTransfer
();
return
;
}
/* Final setup of the connected slave <- master link */
zfree
(
server
.
repl_transfer_tmpfile
);
close
(
server
.
repl_transfer_fd
);
replicationCreateMasterClient
(
server
.
repl_transfer_s
);
redisLog
(
REDIS_NOTICE
,
"MASTER <-> SLAVE sync: Finished with success"
);
/* Restart the AOF subsystem now that we finished the sync. This
* will trigger an AOF rewrite, and when done will start appending
* to the new file. */
if
(
server
.
aof_state
!=
REDIS_AOF_OFF
)
{
int
retry
=
10
;
stopAppendOnly
();
while
(
retry
--
&&
startAppendOnly
()
==
REDIS_ERR
)
{
redisLog
(
REDIS_WARNING
,
"Failed enabling the AOF after successful master synchronization! Trying it again in one second."
);
sleep
(
1
);
}
if
(
!
retry
)
{
redisLog
(
REDIS_WARNING
,
"FATAL: this slave instance finished the synchronization with its master, but the AOF can't be turned on. Exiting now."
);
exit
(
1
);
}
server
.
repl_transfer_fd
=
-
1
;
server
.
repl_transfer_tmpfile
=
NULL
;
}
/* Final setup of the connected slave <- master link */
replicationCreateMasterClient
(
server
.
repl_transfer_s
);
/* feed the initial command stream into the client input buffer */
if
(
initialCommandStream
)
{
int
nread
=
sdslen
(
initialCommandStream
);
server
.
current_client
=
server
.
master
;
server
.
master
->
querybuf
=
sdscatsds
(
server
.
master
->
querybuf
,
initialCommandStream
);
server
.
master
->
reploff
+=
nread
;
server
.
master
->
lastinteraction
=
server
.
unixtime
;
server
.
stat_net_input_bytes
+=
nread
;
processInputBuffer
(
server
.
master
);
server
.
current_client
=
NULL
;
sdsfree
(
initialCommandStream
);
}
redisLog
(
REDIS_NOTICE
,
"MASTER <-> SLAVE sync: Finished with success"
);
/* Restart the AOF subsystem now that we finished the sync. This
* will trigger an AOF rewrite, and when done will start appending
* to the new file. */
if
(
server
.
aof_state
!=
REDIS_AOF_OFF
)
{
int
retry
=
10
;
stopAppendOnly
();
while
(
retry
--
&&
startAppendOnly
()
==
REDIS_ERR
)
{
redisLog
(
REDIS_WARNING
,
"Failed enabling the AOF after successful master synchronization! Trying it again in one second."
);
sleep
(
1
);
}
if
(
!
retry
)
{
redisLog
(
REDIS_WARNING
,
"FATAL: this slave instance finished the synchronization with its master, but the AOF can't be turned on. Exiting now."
);
exit
(
1
);
}
}
return
;
error:
...
...
@@ -1341,16 +1393,20 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) {
}
/* Prepare a suitable temp file for bulk transfer */
while
(
maxtries
--
)
{
snprintf
(
tmpfile
,
256
,
"temp-%d.%ld.rdb"
,(
int
)
server
.
unixtime
,(
long
int
)
getpid
());
dfd
=
open
(
tmpfile
,
O_CREAT
|
O_WRONLY
|
O_EXCL
,
0644
);
if
(
dfd
!=
-
1
)
break
;
sleep
(
1
);
}
if
(
dfd
==
-
1
)
{
redisLog
(
REDIS_WARNING
,
"Opening the temp file needed for MASTER <-> SLAVE synchronization: %s"
,
strerror
(
errno
));
goto
error
;
if
(
!
server
.
repl_diskless_sync
)
{
while
(
maxtries
--
)
{
snprintf
(
tmpfile
,
256
,
"temp-%d.%ld.rdb"
,(
int
)
server
.
unixtime
,(
long
int
)
getpid
());
dfd
=
open
(
tmpfile
,
O_CREAT
|
O_WRONLY
|
O_EXCL
,
0644
);
if
(
dfd
!=
-
1
)
break
;
sleep
(
1
);
}
if
(
dfd
==
-
1
)
{
redisLog
(
REDIS_WARNING
,
"Opening the temp file needed for MASTER <-> SLAVE synchronization: %s"
,
strerror
(
errno
));
goto
error
;
}
server
.
repl_transfer_tmpfile
=
zstrdup
(
tmpfile
);
server
.
repl_transfer_fd
=
dfd
;
}
/* Setup the non blocking download of the bulk file. */
...
...
@@ -1367,13 +1423,17 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) {
server
.
repl_transfer_size
=
-
1
;
server
.
repl_transfer_read
=
0
;
server
.
repl_transfer_last_fsync_off
=
0
;
server
.
repl_transfer_fd
=
dfd
;
server
.
repl_transfer_lastio
=
server
.
unixtime
;
server
.
repl_transfer_tmpfile
=
zstrdup
(
tmpfile
);
return
;
error:
close
(
fd
);
if
(
server
.
repl_transfer_fd
!=
-
1
)
close
(
server
.
repl_transfer_fd
);
if
(
server
.
repl_transfer_tmpfile
)
zfree
(
server
.
repl_transfer_tmpfile
);
server
.
repl_transfer_tmpfile
=
NULL
;
server
.
repl_transfer_fd
=
-
1
;
server
.
repl_transfer_s
=
-
1
;
server
.
repl_state
=
REDIS_REPL_CONNECT
;
return
;
...
...
src/rio.c
View file @
fe63046e
...
...
@@ -157,13 +157,106 @@ void rioInitWithFile(rio *r, FILE *fp) {
r
->
io
.
file
.
autosync
=
0
;
}
/* ------------------- File descriptor implementation ------------------- */
static
size_t
rioFdWrite
(
rio
*
r
,
const
void
*
buf
,
size_t
len
)
{
REDIS_NOTUSED
(
r
);
REDIS_NOTUSED
(
buf
);
REDIS_NOTUSED
(
len
);
return
0
;
/* Error, this target does not yet support writing. */
}
/* Returns 1 or 0 for success/failure. */
static
size_t
rioFdRead
(
rio
*
r
,
void
*
buf
,
size_t
len
)
{
size_t
avail
=
sdslen
(
r
->
io
.
fd
.
buf
)
-
r
->
io
.
fd
.
pos
;
/* if the buffer is too small for the entire request: realloc */
if
(
sdslen
(
r
->
io
.
fd
.
buf
)
+
sdsavail
(
r
->
io
.
fd
.
buf
)
<
len
)
r
->
io
.
fd
.
buf
=
sdsMakeRoomFor
(
r
->
io
.
fd
.
buf
,
len
-
sdslen
(
r
->
io
.
fd
.
buf
));
/* if the remaining unused buffer is not large enough: memmove so that we can read the rest */
if
(
len
>
avail
&&
sdsavail
(
r
->
io
.
fd
.
buf
)
<
len
-
avail
)
{
sdsrange
(
r
->
io
.
fd
.
buf
,
r
->
io
.
fd
.
pos
,
-
1
);
r
->
io
.
fd
.
pos
=
0
;
}
/* if we don't already have all the data in the sds, read more */
while
(
len
>
sdslen
(
r
->
io
.
fd
.
buf
)
-
r
->
io
.
fd
.
pos
)
{
size_t
toread
=
len
-
(
sdslen
(
r
->
io
.
fd
.
buf
)
-
r
->
io
.
fd
.
pos
);
/* read either what's missing, or REDIS_IOBUF_LEN, the bigger of the two */
if
(
toread
<
REDIS_IOBUF_LEN
)
toread
=
REDIS_IOBUF_LEN
;
if
(
toread
>
sdsavail
(
r
->
io
.
fd
.
buf
))
toread
=
sdsavail
(
r
->
io
.
fd
.
buf
);
int
retval
=
read
(
r
->
io
.
fd
.
fd
,
(
char
*
)
r
->
io
.
fd
.
buf
+
sdslen
(
r
->
io
.
fd
.
buf
),
toread
);
if
(
retval
==
-
1
)
{
if
(
errno
==
EWOULDBLOCK
)
errno
=
ETIMEDOUT
;
return
0
;
}
sdsIncrLen
(
r
->
io
.
fd
.
buf
,
retval
);
}
memcpy
(
buf
,
(
char
*
)
r
->
io
.
fd
.
buf
+
r
->
io
.
fd
.
pos
,
len
);
r
->
io
.
fd
.
pos
+=
len
;
return
len
;
}
/* Returns read/write position in file. */
static
off_t
rioFdTell
(
rio
*
r
)
{
off_t
pos
=
lseek
(
r
->
io
.
fd
.
fd
,
0
,
SEEK_CUR
);
return
pos
-
sdslen
(
r
->
io
.
fd
.
buf
)
+
r
->
io
.
fd
.
pos
;
}
/* Flushes any buffer to target device if applicable. Returns 1 on success
* and 0 on failures. */
static
int
rioFdFlush
(
rio
*
r
)
{
/* Our flush is implemented by the write method, that recognizes a
* buffer set to NULL with a count of zero as a flush request. */
return
rioFdWrite
(
r
,
NULL
,
0
);
}
static
const
rio
rioFdIO
=
{
rioFdRead
,
rioFdWrite
,
rioFdTell
,
rioFdFlush
,
NULL
,
/* update_checksum */
0
,
/* current checksum */
0
,
/* bytes read or written */
0
,
/* read/write chunk size */
{
{
NULL
,
0
}
}
/* union for io-specific vars */
};
void
rioInitWithFd
(
rio
*
r
,
int
fd
)
{
*
r
=
rioFdIO
;
r
->
io
.
fd
.
fd
=
fd
;
r
->
io
.
fd
.
pos
=
0
;
r
->
io
.
fd
.
buf
=
sdsnewlen
(
NULL
,
REDIS_IOBUF_LEN
);
sdsclear
(
r
->
io
.
fd
.
buf
);
}
/* release the rio stream.
* optionally returns the unread buffered data. */
void
rioFreeFd
(
rio
*
r
,
sds
*
out_remainingBufferedData
)
{
if
(
out_remainingBufferedData
&&
(
size_t
)
r
->
io
.
fd
.
pos
<
sdslen
(
r
->
io
.
fd
.
buf
))
{
if
(
r
->
io
.
fd
.
pos
>
0
)
sdsrange
(
r
->
io
.
fd
.
buf
,
r
->
io
.
fd
.
pos
,
-
1
);
*
out_remainingBufferedData
=
r
->
io
.
fd
.
buf
;
}
else
{
sdsfree
(
r
->
io
.
fd
.
buf
);
if
(
out_remainingBufferedData
)
*
out_remainingBufferedData
=
NULL
;
}
r
->
io
.
fd
.
buf
=
NULL
;
}
/* ------------------- File descriptors set implementation ------------------- */
/* Returns 1 or 0 for success/failure.
* The function returns success as long as we are able to correctly write
* to at least one file descriptor.
*
* When buf is NULL a
d
n len is 0, the function performs a flush operation
* When buf is NULL an
d
len is 0, the function performs a flush operation
* if there is some pending buffer, so this function is also used in order
* to implement rioFdsetFlush(). */
static
size_t
rioFdsetWrite
(
rio
*
r
,
const
void
*
buf
,
size_t
len
)
{
...
...
@@ -176,7 +269,7 @@ static size_t rioFdsetWrite(rio *r, const void *buf, size_t len) {
* a given size, we actually write to the sockets. */
if
(
len
)
{
r
->
io
.
fdset
.
buf
=
sdscatlen
(
r
->
io
.
fdset
.
buf
,
buf
,
len
);
len
=
0
;
/* Prevent entering the while belo
ve
if we don't flush. */
len
=
0
;
/* Prevent entering the while belo
w
if we don't flush. */
if
(
sdslen
(
r
->
io
.
fdset
.
buf
)
>
REDIS_IOBUF_LEN
)
doflush
=
1
;
}
...
...
@@ -276,6 +369,7 @@ void rioInitWithFdset(rio *r, int *fds, int numfds) {
r
->
io
.
fdset
.
buf
=
sdsempty
();
}
/* release the rio stream. */
void
rioFreeFdset
(
rio
*
r
)
{
zfree
(
r
->
io
.
fdset
.
fds
);
zfree
(
r
->
io
.
fdset
.
state
);
...
...
src/rio.h
View file @
fe63046e
...
...
@@ -73,6 +73,12 @@ struct _rio {
off_t
buffered
;
/* Bytes written since last fsync. */
off_t
autosync
;
/* fsync after 'autosync' bytes written. */
}
file
;
/* file descriptor */
struct
{
int
fd
;
/* File descriptor. */
off_t
pos
;
sds
buf
;
}
fd
;
/* Multiple FDs target (used to write to N sockets). */
struct
{
int
*
fds
;
/* File descriptors. */
...
...
@@ -126,8 +132,12 @@ static inline int rioFlush(rio *r) {
void
rioInitWithFile
(
rio
*
r
,
FILE
*
fp
);
void
rioInitWithBuffer
(
rio
*
r
,
sds
s
);
void
rioInitWithFd
(
rio
*
r
,
int
fd
);
void
rioInitWithFdset
(
rio
*
r
,
int
*
fds
,
int
numfds
);
void
rioFreeFdset
(
rio
*
r
);
void
rioFreeFd
(
rio
*
r
,
sds
*
out_remainingBufferedData
);
size_t
rioWriteBulkCount
(
rio
*
r
,
char
prefix
,
int
count
);
size_t
rioWriteBulkString
(
rio
*
r
,
const
char
*
buf
,
size_t
len
);
size_t
rioWriteBulkLongLong
(
rio
*
r
,
long
long
l
);
...
...
tests/integration/replication-4.tcl
View file @
fe63046e
proc start_bg_complex_data
{
host port db ops
}
{
set tclsh
[
info nameofexecutable
]
exec $tclsh tests/helpers/bg_complex_data.tcl $host $port $db $ops &
}
proc stop_bg_complex_data
{
handle
}
{
catch
{
exec /bin/kill -9 $handle
}
}
start_server
{
tags
{
"repl"
}}
{
start_server
{}
{
...
...
tests/integration/replication-psync.tcl
View file @
fe63046e
proc start_bg_complex_data
{
host port db ops
}
{
set tclsh
[
info nameofexecutable
]
exec $tclsh tests/helpers/bg_complex_data.tcl $host $port $db $ops &
}
proc stop_bg_complex_data
{
handle
}
{
catch
{
exec /bin/kill -9 $handle
}
}
# Creates a master-slave pair and breaks the link continuously to force
# partial resyncs attempts, all this while flooding the master with
# write queries.
...
...
tests/integration/replication.tcl
View file @
fe63046e
...
...
@@ -94,85 +94,91 @@ start_server {tags {"repl"}} {
}
}
foreach dl
{
no yes
}
{
start_server
{
tags
{
"repl"
}
}
{
s
et master
[
srv 0 client
]
$master config set repl-diskless-sync $dl
set
master
_host
[
srv 0 host
]
set
master
_port
[
srv 0 port
]
set slaves
{}
set load_handle0
[
start_write_load $master_host $master_
port
3
]
set load_handle1
[
start_write_load $master_host $master_port 5
]
set load_handle
2
[
start_
write_load
$master_host $master_port
2
0
]
set load_handle
3
[
start_
write_load
$master_host $master_port
8
]
set load_handle
4
[
start_
write_load
$master_host $master_port
4
]
start_server
{}
{
lappend slaves
[
srv 0 client
]
foreach
m
dl
{
no yes
}
{
foreach sdl
{
no yes
}
{
s
tart_server
{
tags
{
"repl"
}}
{
set master
[
srv 0 client
]
$
master
config set repl-diskless-sync $mdl
$
master
config set repl-diskless-sync-delay 1
set master_host
[
srv 0 host
]
set master_port
[
srv 0
port
]
set slaves
{}
set load_handle
0
[
start_
bg_complex_data
$master_host $master_port
9 10000000
0
]
set load_handle
1
[
start_
bg_complex_data
$master_host $master_port
11 100000000
]
set load_handle
2
[
start_
bg_complex_data
$master_host $master_port
12 100000000
]
set load_handle3
[
start_write_load $master_host $master_port 8
]
set load_handle4
[
start_write_load $master_host $master_port 4
]
start_server
{}
{
lappend slaves
[
srv 0 client
]
start_server
{}
{
lappend slaves
[
srv 0 client
]
test
"Connect multiple slaves at the same time (issue #141), diskless=
$dl
"
{
# Send SALVEOF commands to slaves
[
lindex $slaves 0
]
slaveof $master_host $master_port
[
lindex $slaves 1
]
slaveof $master_host $master_port
[
lindex $slaves 2
]
slaveof $master_host $master_port
# Wait for all the three slaves to reach the
"online"
# state from the POV of the master.
set retry 500
while
{
$retry
}
{
set info
[
r -3 info
]
if
{[
string match
{
*slave0:*state=online*slave1:*state=online*slave2:*state=online*
}
$info
]}
{
break
start_server
{}
{
lappend slaves
[
srv 0 client
]
test
"Connect multiple slaves at the same time (issue #141), master diskless=
$mdl
, slave diskless=
$sdl
"
{
# Send SALVEOF commands to slaves
[
lindex $slaves 0
]
config set repl-diskless-sync $sdl
[
lindex $slaves 1
]
config set repl-diskless-sync $sdl
[
lindex $slaves 2
]
config set repl-diskless-sync $sdl
[
lindex $slaves 0
]
slaveof $master_host $master_port
[
lindex $slaves 1
]
slaveof $master_host $master_port
[
lindex $slaves 2
]
slaveof $master_host $master_port
# Wait for all the three slaves to reach the
"online"
# state from the POV of the master.
set retry 500
while
{
$retry
}
{
set info
[
r -3 info
]
if
{[
string match
{
*slave0:*state=online*slave1:*state=online*slave2:*state=online*
}
$info
]}
{
break
}
else
{
incr retry -1
after 100
}
}
if
{
$retry
== 0
}
{
error
"assertion:Slaves not correctly synchronized"
}
# Wait that slaves acknowledge they are online so
# we are sure that DBSIZE and DEBUG DIGEST will not
# fail because of timing issues.
wait_for_condition 500 100
{
[
lindex
[[
lindex $slaves 0
]
role
]
3
]
eq
{
connected
}
&&
[
lindex
[[
lindex $slaves 1
]
role
]
3
]
eq
{
connected
}
&&
[
lindex
[[
lindex $slaves 2
]
role
]
3
]
eq
{
connected
}
}
else
{
incr retry -1
after 100
fail
"Slaves still not connected after some time"
}
}
if
{
$retry
== 0
}
{
error
"assertion:Slaves not correctly synchronized"
}
# Wait that slaves acknowledge they are online so
# we are sure that DBSIZE and DEBUG DIGEST will not
# fail because of timing issues.
wait_for_condition 500 100
{
[
lindex
[[
lindex $slaves 0
]
role
]
3
]
eq
{
connected
}
&&
[
lindex
[[
lindex $slaves 1
]
role
]
3
]
eq
{
connected
}
&&
[
lindex
[[
lindex $slaves 2
]
role
]
3
]
eq
{
connected
}
}
else
{
fail
"Slaves still not connected after some time"
}
# Stop the write load
stop_bg_complex_data $load_handle0
stop_bg_complex_data $load_handle1
stop_bg_complex_data $load_handle2
stop_write_load $load_handle3
stop_write_load $load_handle4
# Make sure that slaves and master have same
# number of keys
wait_for_condition 500 100
{
[
$master
dbsize
]
==
[[
lindex $slaves 0
]
dbsize
]
&&
[
$master
dbsize
]
==
[[
lindex $slaves 1
]
dbsize
]
&&
[
$master
dbsize
]
==
[[
lindex $slaves 2
]
dbsize
]
}
else
{
fail
"Different number of keys between masted and slave after too long time."
}
# Stop the write load
stop_write_load $load_handle0
stop_write_load $load_handle1
stop_write_load $load_handle2
stop_write_load $load_handle3
stop_write_load $load_handle4
# Make sure that slaves and master have same
# number of keys
wait_for_condition 500 100
{
[
$master
dbsize
]
==
[[
lindex $slaves 0
]
dbsize
]
&&
[
$master
dbsize
]
==
[[
lindex $slaves 1
]
dbsize
]
&&
[
$master
dbsize
]
==
[[
lindex $slaves 2
]
dbsize
]
}
else
{
fail
"Different number of keys between masted and slave after too long time."
# Check digests
set digest
[
$master
debug digest
]
set digest0
[[
lindex $slaves 0
]
debug digest
]
set digest1
[[
lindex $slaves 1
]
debug digest
]
set digest2
[[
lindex $slaves 2
]
debug digest
]
assert
{
$digest
ne 0000000000000000000000000000000000000000
}
assert
{
$digest
eq $digest0
}
assert
{
$digest
eq $digest1
}
assert
{
$digest
eq $digest2
}
}
# Check digests
set digest
[
$master
debug digest
]
set digest0
[[
lindex $slaves 0
]
debug digest
]
set digest1
[[
lindex $slaves 1
]
debug digest
]
set digest2
[[
lindex $slaves 2
]
debug digest
]
assert
{
$digest
ne 0000000000000000000000000000000000000000
}
assert
{
$digest
eq $digest0
}
assert
{
$digest
eq $digest1
}
assert
{
$digest
eq $digest2
}
}
}
}
}
}
}
}
...
...
tests/support/util.tcl
View file @
fe63046e
...
...
@@ -371,3 +371,15 @@ proc start_write_load {host port seconds} {
proc stop_write_load
{
handle
}
{
catch
{
exec /bin/kill -9 $handle
}
}
# Execute a background process writing complex data for the specified number
# of ops to the specified Redis instance.
proc start_bg_complex_data
{
host port db ops
}
{
set tclsh
[
info nameofexecutable
]
exec $tclsh tests/helpers/bg_complex_data.tcl $host $port $db $ops &
}
# Stop a process generating write load executed with start_bg_complex_data.
proc stop_bg_complex_data
{
handle
}
{
catch
{
exec /bin/kill -9 $handle
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment