Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
f80dff62
Commit
f80dff62
authored
Nov 01, 2009
by
antirez
Browse files
first version of append only file loading -- STILL BROKEN don't use it
parent
412a8bce
Changes
1
Hide whitespace changes
Inline
Side-by-side
redis.c
View file @
f80dff62
...
@@ -1666,86 +1666,6 @@ static void replicationFeedSlaves(list *slaves, struct redisCommand *cmd, int di
...
@@ -1666,86 +1666,6 @@ static void replicationFeedSlaves(list *slaves, struct redisCommand *cmd, int di
if
(
outv
!=
static_outv
)
zfree
(
outv
);
if
(
outv
!=
static_outv
)
zfree
(
outv
);
}
}
/* TODO: translate EXPIREs into EXPIRETOs */
static
void
feedAppendOnlyFile
(
struct
redisCommand
*
cmd
,
int
dictid
,
robj
**
argv
,
int
argc
)
{
sds
buf
=
sdsempty
();
int
j
;
ssize_t
nwritten
;
time_t
now
;
robj
*
tmpargv
[
3
];
/* The DB this command was targetting is not the same as the last command
* we appendend. To issue a SELECT command is needed. */
if
(
dictid
!=
server
.
appendseldb
)
{
char
seldb
[
64
];
snprintf
(
seldb
,
sizeof
(
seldb
),
"%d"
,
dictid
);
buf
=
sdscatprintf
(
buf
,
"*2
\r\n
$6
\r\n
SELECT
\r\n
$%d
\r\n
%s
\r\n
"
,
strlen
(
seldb
),
seldb
);
server
.
appendseldb
=
dictid
;
}
/* "Fix" the argv vector if the command is EXPIRE. We want to translate
* EXPIREs into EXPIREATs calls */
if
(
cmd
->
proc
==
expireCommand
)
{
long
when
;
tmpargv
[
0
]
=
createStringObject
(
"EXPIREAT"
,
8
);
tmpargv
[
1
]
=
argv
[
1
];
incrRefCount
(
argv
[
1
]);
when
=
time
(
NULL
)
+
strtol
(
argv
[
2
]
->
ptr
,
NULL
,
10
);
tmpargv
[
2
]
=
createObject
(
REDIS_STRING
,
sdscatprintf
(
sdsempty
(),
"%ld"
,
when
));
argv
=
tmpargv
;
}
/* Append the actual command */
buf
=
sdscatprintf
(
buf
,
"*%d
\r\n
"
,
argc
);
for
(
j
=
0
;
j
<
argc
;
j
++
)
{
robj
*
o
=
argv
[
j
];
if
(
o
->
encoding
!=
REDIS_ENCODING_RAW
)
o
=
getDecodedObject
(
o
);
buf
=
sdscatprintf
(
buf
,
"$%d
\r\n
"
,
sdslen
(
o
->
ptr
));
buf
=
sdscatlen
(
buf
,
o
->
ptr
,
sdslen
(
o
->
ptr
));
buf
=
sdscatlen
(
buf
,
"
\r\n
"
,
2
);
if
(
o
!=
argv
[
j
])
decrRefCount
(
o
);
}
/* Free the objects from the modified argv for EXPIREAT */
if
(
cmd
->
proc
==
expireCommand
)
{
for
(
j
=
0
;
j
<
3
;
j
++
)
decrRefCount
(
argv
[
j
]);
}
/* We want to perform a single write. This should be guaranteed atomic
* at least if the filesystem we are writing is a real physical one.
* While this will save us against the server being killed I don't think
* there is much to do about the whole server stopping for power problems
* or alike */
nwritten
=
write
(
server
.
appendfd
,
buf
,
sdslen
(
buf
));
if
(
nwritten
!=
(
signed
)
sdslen
(
buf
))
{
/* Ooops, we are in troubles. The best thing to do for now is
* to simply exit instead to give the illusion that everything is
* working as expected. */
if
(
nwritten
==
-
1
)
{
redisLog
(
REDIS_WARNING
,
"Aborting on error writing to the append-only file: %s"
,
strerror
(
errno
));
}
else
{
redisLog
(
REDIS_WARNING
,
"Aborting on short write while writing to the append-only file: %s"
,
strerror
(
errno
));
}
abort
();
}
now
=
time
(
NULL
);
if
(
server
.
appendfsync
==
APPENDFSYNC_ALWAYS
||
(
server
.
appendfsync
==
APPENDFSYNC_EVERYSEC
&&
now
-
server
.
lastfsync
>
1
))
{
fsync
(
server
.
appendfd
);
/* Let's try to get this data on the disk */
server
.
lastfsync
=
now
;
}
}
static
void
processInputBuffer
(
redisClient
*
c
)
{
static
void
processInputBuffer
(
redisClient
*
c
)
{
again:
again:
if
(
c
->
bulklen
==
-
1
)
{
if
(
c
->
bulklen
==
-
1
)
{
...
@@ -2834,7 +2754,7 @@ static int rdbLoad(char *filename) {
...
@@ -2834,7 +2754,7 @@ static int rdbLoad(char *filename) {
eoferr:
/* unexpected end of file is handled here with a fatal exit */
eoferr:
/* unexpected end of file is handled here with a fatal exit */
if
(
keyobj
)
decrRefCount
(
keyobj
);
if
(
keyobj
)
decrRefCount
(
keyobj
);
redisLog
(
REDIS_WARNING
,
"Short read or OOM loading DB. Unrecoverable error,
exi
ting now."
);
redisLog
(
REDIS_WARNING
,
"Short read or OOM loading DB. Unrecoverable error,
abor
ting now."
);
exit
(
1
);
exit
(
1
);
return
REDIS_ERR
;
/* Just to avoid warning */
return
REDIS_ERR
;
/* Just to avoid warning */
}
}
...
@@ -5334,6 +5254,195 @@ static void freeMemoryIfNeeded(void) {
...
@@ -5334,6 +5254,195 @@ static void freeMemoryIfNeeded(void) {
}
}
}
}
/* ============================== Append Only file ========================== */
static
void
feedAppendOnlyFile
(
struct
redisCommand
*
cmd
,
int
dictid
,
robj
**
argv
,
int
argc
)
{
sds
buf
=
sdsempty
();
int
j
;
ssize_t
nwritten
;
time_t
now
;
robj
*
tmpargv
[
3
];
/* The DB this command was targetting is not the same as the last command
* we appendend. To issue a SELECT command is needed. */
if
(
dictid
!=
server
.
appendseldb
)
{
char
seldb
[
64
];
snprintf
(
seldb
,
sizeof
(
seldb
),
"%d"
,
dictid
);
buf
=
sdscatprintf
(
buf
,
"*2
\r\n
$6
\r\n
SELECT
\r\n
$%d
\r\n
%s
\r\n
"
,
strlen
(
seldb
),
seldb
);
server
.
appendseldb
=
dictid
;
}
/* "Fix" the argv vector if the command is EXPIRE. We want to translate
* EXPIREs into EXPIREATs calls */
if
(
cmd
->
proc
==
expireCommand
)
{
long
when
;
tmpargv
[
0
]
=
createStringObject
(
"EXPIREAT"
,
8
);
tmpargv
[
1
]
=
argv
[
1
];
incrRefCount
(
argv
[
1
]);
when
=
time
(
NULL
)
+
strtol
(
argv
[
2
]
->
ptr
,
NULL
,
10
);
tmpargv
[
2
]
=
createObject
(
REDIS_STRING
,
sdscatprintf
(
sdsempty
(),
"%ld"
,
when
));
argv
=
tmpargv
;
}
/* Append the actual command */
buf
=
sdscatprintf
(
buf
,
"*%d
\r\n
"
,
argc
);
for
(
j
=
0
;
j
<
argc
;
j
++
)
{
robj
*
o
=
argv
[
j
];
if
(
o
->
encoding
!=
REDIS_ENCODING_RAW
)
o
=
getDecodedObject
(
o
);
buf
=
sdscatprintf
(
buf
,
"$%d
\r\n
"
,
sdslen
(
o
->
ptr
));
buf
=
sdscatlen
(
buf
,
o
->
ptr
,
sdslen
(
o
->
ptr
));
buf
=
sdscatlen
(
buf
,
"
\r\n
"
,
2
);
if
(
o
!=
argv
[
j
])
decrRefCount
(
o
);
}
/* Free the objects from the modified argv for EXPIREAT */
if
(
cmd
->
proc
==
expireCommand
)
{
for
(
j
=
0
;
j
<
3
;
j
++
)
decrRefCount
(
argv
[
j
]);
}
/* We want to perform a single write. This should be guaranteed atomic
* at least if the filesystem we are writing is a real physical one.
* While this will save us against the server being killed I don't think
* there is much to do about the whole server stopping for power problems
* or alike */
nwritten
=
write
(
server
.
appendfd
,
buf
,
sdslen
(
buf
));
if
(
nwritten
!=
(
signed
)
sdslen
(
buf
))
{
/* Ooops, we are in troubles. The best thing to do for now is
* to simply exit instead to give the illusion that everything is
* working as expected. */
if
(
nwritten
==
-
1
)
{
redisLog
(
REDIS_WARNING
,
"Exiting on error writing to the append-only file: %s"
,
strerror
(
errno
));
}
else
{
redisLog
(
REDIS_WARNING
,
"Exiting on short write while writing to the append-only file: %s"
,
strerror
(
errno
));
}
exit
(
1
);
}
now
=
time
(
NULL
);
if
(
server
.
appendfsync
==
APPENDFSYNC_ALWAYS
||
(
server
.
appendfsync
==
APPENDFSYNC_EVERYSEC
&&
now
-
server
.
lastfsync
>
1
))
{
fsync
(
server
.
appendfd
);
/* Let's try to get this data on the disk */
server
.
lastfsync
=
now
;
}
}
/* In Redis commands are always executed in the context of a client, so in
* order to load the append only file we need to create a fake client. */
static
struct
redisClient
*
createFakeClient
(
void
)
{
struct
redisClient
*
c
=
zmalloc
(
sizeof
(
*
c
));
selectDb
(
c
,
0
);
c
->
fd
=
-
1
;
c
->
querybuf
=
sdsempty
();
c
->
argc
=
0
;
c
->
argv
=
NULL
;
c
->
flags
=
0
;
c
->
reply
=
listCreate
();
listSetFreeMethod
(
c
->
reply
,
decrRefCount
);
listSetDupMethod
(
c
->
reply
,
dupClientReplyValue
);
return
c
;
}
static
void
freeFakeClient
(
struct
redisClient
*
c
)
{
sdsfree
(
c
->
querybuf
);
listRelease
(
c
->
reply
);
zfree
(
c
);
}
/* Replay the append log file. On error REDIS_OK is returned. On non fatal
* error (the append only file is zero-length) REDIS_ERR is returned. On
* fatal error an error message is logged and the program exists. */
int
loadAppendOnlyFile
(
char
*
filename
)
{
struct
redisClient
*
fakeClient
;
FILE
*
fp
=
fopen
(
filename
,
"r"
);
struct
redis_stat
sb
;
if
(
redis_fstat
(
fileno
(
fp
),
&
sb
)
!=
-
1
&&
sb
.
st_size
==
0
)
return
REDIS_ERR
;
if
(
fp
==
NULL
)
{
redisLog
(
REDIS_WARNING
,
"Fatal error: can't open the append log file for reading: %s"
,
strerror
(
errno
));
exit
(
1
);
}
fakeClient
=
createFakeClient
();
while
(
1
)
{
int
argc
,
j
;
unsigned
long
len
;
robj
**
argv
;
char
buf
[
128
];
sds
argsds
;
struct
redisCommand
*
cmd
;
if
(
fgets
(
buf
,
sizeof
(
buf
),
fp
)
==
NULL
)
{
if
(
feof
(
fp
))
break
;
else
goto
readerr
;
}
if
(
buf
[
0
]
!=
'*'
)
goto
fmterr
;
argc
=
atoi
(
buf
+
1
);
argv
=
zmalloc
(
sizeof
(
robj
*
)
*
argc
);
for
(
j
=
0
;
j
<
argc
;
j
++
)
{
if
(
fgets
(
buf
,
sizeof
(
buf
),
fp
)
==
NULL
)
goto
readerr
;
if
(
buf
[
0
]
!=
'$'
)
goto
fmterr
;
len
=
strtol
(
buf
+
1
,
NULL
,
10
);
argsds
=
sdsnewlen
(
NULL
,
len
);
if
(
fread
(
argsds
,
len
,
1
,
fp
)
==
0
)
goto
fmterr
;
argv
[
j
]
=
createObject
(
REDIS_STRING
,
argsds
);
if
(
fread
(
buf
,
2
,
1
,
fp
)
==
0
)
goto
fmterr
;
/* discard CRLF */
}
/* Command lookup */
cmd
=
lookupCommand
(
argv
[
0
]
->
ptr
);
if
(
!
cmd
)
{
redisLog
(
REDIS_WARNING
,
"Unknown command '%s' reading the append only file"
,
argv
[
0
]
->
ptr
);
exit
(
1
);
}
/* Try object sharing and encoding */
if
(
server
.
shareobjects
)
{
int
j
;
for
(
j
=
1
;
j
<
argc
;
j
++
)
argv
[
j
]
=
tryObjectSharing
(
argv
[
j
]);
}
if
(
cmd
->
flags
&
REDIS_CMD_BULK
)
tryObjectEncoding
(
argv
[
argc
-
1
]);
/* Run the command in the context of a fake client */
fakeClient
->
argc
=
argc
;
fakeClient
->
argv
=
argv
;
cmd
->
proc
(
fakeClient
);
/* Discard the reply objects list from the fake client */
while
(
listLength
(
fakeClient
->
reply
))
listDelNode
(
fakeClient
->
reply
,
listFirst
(
fakeClient
->
reply
));
/* Clean up, ready for the next command */
for
(
j
=
0
;
j
<
argc
;
j
++
)
decrRefCount
(
argv
[
j
]);
zfree
(
argv
);
}
fclose
(
fp
);
freeFakeClient
(
fakeClient
);
return
REDIS_OK
;
readerr:
if
(
feof
(
fp
))
{
redisLog
(
REDIS_WARNING
,
"Unexpected end of file reading the append only file"
);
}
else
{
redisLog
(
REDIS_WARNING
,
"Unrecoverable error reading the append only file: %s"
,
strerror
(
errno
));
}
exit
(
1
);
fmterr:
redisLog
(
REDIS_WARNING
,
"Bad file format reading the append only file"
);
exit
(
1
);
}
/* ================================= Debugging ============================== */
/* ================================= Debugging ============================== */
static
void
debugCommand
(
redisClient
*
c
)
{
static
void
debugCommand
(
redisClient
*
c
)
{
...
@@ -5666,8 +5775,13 @@ int main(int argc, char **argv) {
...
@@ -5666,8 +5775,13 @@ int main(int argc, char **argv) {
#ifdef __linux__
#ifdef __linux__
linuxOvercommitMemoryWarning
();
linuxOvercommitMemoryWarning
();
#endif
#endif
if
(
rdbLoad
(
server
.
dbfilename
)
==
REDIS_OK
)
if
(
server
.
appendonly
)
{
redisLog
(
REDIS_NOTICE
,
"DB loaded from disk"
);
if
(
loadAppendOnlyFile
(
server
.
appendfilename
)
==
REDIS_OK
)
redisLog
(
REDIS_NOTICE
,
"DB loaded from append only file"
);
}
else
{
if
(
rdbLoad
(
server
.
dbfilename
)
==
REDIS_OK
)
redisLog
(
REDIS_NOTICE
,
"DB loaded from disk"
);
}
if
(
aeCreateFileEvent
(
server
.
el
,
server
.
fd
,
AE_READABLE
,
if
(
aeCreateFileEvent
(
server
.
el
,
server
.
fd
,
AE_READABLE
,
acceptHandler
,
NULL
,
NULL
)
==
AE_ERR
)
oom
(
"creating file event"
);
acceptHandler
,
NULL
,
NULL
)
==
AE_ERR
)
oom
(
"creating file event"
);
redisLog
(
REDIS_NOTICE
,
"The server is now ready to accept connections on port %d"
,
server
.
port
);
redisLog
(
REDIS_NOTICE
,
"The server is now ready to accept connections on port %d"
,
server
.
port
);
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment