Unverified Commit e9bff797 authored by chenyang8094's avatar chenyang8094 Committed by GitHub
Browse files

Always create base AOF file when redis start from empty. (#10102)



Force create a BASE file (use a foreground `rewriteAppendOnlyFile`) when redis starts from an
empty data set and  `appendonly` is  yes.

The reasoning is that normally, after redis is running for some time, and the AOF has gone though
a few rewrites, there's always a base rdb file. and the scenario where the base file is missing, is
kinda rare (happens only at empty startup), so this change normalizes it.
But more importantly, there are or could be some complex modules that are started with some
configuration, when they create persistence they write that configuration to RDB AUX fields, so
that can can always know with which configuration the persistence file they're loading was
created (could be critical). there is (was) one scenario in which they could load their persisted data,
and that configuration was missing, and this change fixes it.

Add a new module event: REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_AOF_START, similar to
REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START which is async.
Co-authored-by: default avatarOran Agra <oran@redislabs.com>
parent 20c33fe6
...@@ -46,6 +46,7 @@ off_t getAppendOnlyFileSize(sds filename); ...@@ -46,6 +46,7 @@ off_t getAppendOnlyFileSize(sds filename);
off_t getBaseAndIncrAppendOnlyFilesSize(aofManifest *am); off_t getBaseAndIncrAppendOnlyFilesSize(aofManifest *am);
int getBaseAndIncrAppendOnlyFilesNum(aofManifest *am); int getBaseAndIncrAppendOnlyFilesNum(aofManifest *am);
int aofFileExist(char *filename); int aofFileExist(char *filename);
int rewriteAppendOnlyFile(char *filename);
/* ---------------------------------------------------------------------------- /* ----------------------------------------------------------------------------
* AOF Manifest file implementation. * AOF Manifest file implementation.
...@@ -667,11 +668,12 @@ int aofDelHistoryFiles(void) { ...@@ -667,11 +668,12 @@ int aofDelHistoryFiles(void) {
} }
/* Called after `loadDataFromDisk` when redis start. If `server.aof_state` is /* Called after `loadDataFromDisk` when redis start. If `server.aof_state` is
* 'AOF_ON', It will do two things: * 'AOF_ON', It will do three things:
* 1. Open the last opened INCR type AOF for writing, If not, create a new one * 1. Force create a BASE file when redis starts with an empty dataset
* 2. Synchronously update the manifest file to the disk * 2. Open the last opened INCR type AOF for writing, If not, create a new one
* 3. Synchronously update the manifest file to the disk
* *
* If any of the above two steps fails, the redis process will exit. * If any of the above steps fails, the redis process will exit.
*/ */
void aofOpenIfNeededOnServerStart(void) { void aofOpenIfNeededOnServerStart(void) {
if (server.aof_state != AOF_ON) { if (server.aof_state != AOF_ON) {
...@@ -687,6 +689,18 @@ void aofOpenIfNeededOnServerStart(void) { ...@@ -687,6 +689,18 @@ void aofOpenIfNeededOnServerStart(void) {
exit(1); exit(1);
} }
/* If we start with an empty dataset, we will force create a BASE file. */
if (!server.aof_manifest->base_aof_info &&
!listLength(server.aof_manifest->incr_aof_list))
{
sds base_name = getNewBaseFileNameAndMarkPreAsHistory(server.aof_manifest);
sds base_filepath = makePath(server.aof_dirname, base_name);
if (rewriteAppendOnlyFile(base_filepath) != C_OK) {
exit(1);
}
sdsfree(base_filepath);
}
/* Because we will 'exit(1)' if open AOF or persistent manifest fails, so /* Because we will 'exit(1)' if open AOF or persistent manifest fails, so
* we don't need atomic modification here. */ * we don't need atomic modification here. */
sds aof_name = getLastIncrAofName(server.aof_manifest); sds aof_name = getLastIncrAofName(server.aof_manifest);
...@@ -701,6 +715,7 @@ void aofOpenIfNeededOnServerStart(void) { ...@@ -701,6 +715,7 @@ void aofOpenIfNeededOnServerStart(void) {
exit(1); exit(1);
} }
/* Persist our changes. */
int ret = persistAofManifest(server.aof_manifest); int ret = persistAofManifest(server.aof_manifest);
if (ret != C_OK) { if (ret != C_OK) {
exit(1); exit(1);
......
...@@ -9046,6 +9046,7 @@ static uint64_t moduleEventVersions[] = { ...@@ -9046,6 +9046,7 @@ static uint64_t moduleEventVersions[] = {
* * `REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START` * * `REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START`
* * `REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START` * * `REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START`
* * `REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START` * * `REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START`
* * `REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_AOF_START`
* * `REDISMODULE_SUBEVENT_PERSISTENCE_ENDED` * * `REDISMODULE_SUBEVENT_PERSISTENCE_ENDED`
* * `REDISMODULE_SUBEVENT_PERSISTENCE_FAILED` * * `REDISMODULE_SUBEVENT_PERSISTENCE_FAILED`
* *
......
...@@ -2702,14 +2702,16 @@ void stopLoading(int success) { ...@@ -2702,14 +2702,16 @@ void stopLoading(int success) {
success? success?
REDISMODULE_SUBEVENT_LOADING_ENDED: REDISMODULE_SUBEVENT_LOADING_ENDED:
REDISMODULE_SUBEVENT_LOADING_FAILED, REDISMODULE_SUBEVENT_LOADING_FAILED,
NULL); NULL);
} }
void startSaving(int rdbflags) { void startSaving(int rdbflags) {
/* Fire the persistence modules end event. */ /* Fire the persistence modules end event. */
int subevent; int subevent;
if (rdbflags & RDBFLAGS_AOF_PREAMBLE) if (rdbflags & RDBFLAGS_AOF_PREAMBLE && getpid() != server.pid)
subevent = REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START; subevent = REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START;
else if (rdbflags & RDBFLAGS_AOF_PREAMBLE)
subevent = REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_AOF_START;
else if (getpid()!=server.pid) else if (getpid()!=server.pid)
subevent = REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START; subevent = REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START;
else else
......
...@@ -383,7 +383,8 @@ static const RedisModuleEvent ...@@ -383,7 +383,8 @@ static const RedisModuleEvent
#define REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START 2 #define REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START 2
#define REDISMODULE_SUBEVENT_PERSISTENCE_ENDED 3 #define REDISMODULE_SUBEVENT_PERSISTENCE_ENDED 3
#define REDISMODULE_SUBEVENT_PERSISTENCE_FAILED 4 #define REDISMODULE_SUBEVENT_PERSISTENCE_FAILED 4
#define _REDISMODULE_SUBEVENT_PERSISTENCE_NEXT 5 #define REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_AOF_START 5
#define _REDISMODULE_SUBEVENT_PERSISTENCE_NEXT 6
#define REDISMODULE_SUBEVENT_LOADING_RDB_START 0 #define REDISMODULE_SUBEVENT_LOADING_RDB_START 0
#define REDISMODULE_SUBEVENT_LOADING_AOF_START 1 #define REDISMODULE_SUBEVENT_LOADING_AOF_START 1
......
...@@ -687,7 +687,7 @@ tags {"external:skip"} { ...@@ -687,7 +687,7 @@ tags {"external:skip"} {
waitForBgrewriteaof $redis waitForBgrewriteaof $redis
assert_aof_manifest_content $aof_manifest_name { assert_aof_manifest_content $aof_manifest_name {
{file " file seq .aof .1.base.rdb" seq 1 type b} {file " file seq .aof .2.base.rdb" seq 2 type b}
{file " file seq .aof .2.incr.aof" seq 2 type i} {file " file seq .aof .2.incr.aof" seq 2 type i}
} }
...@@ -696,6 +696,54 @@ tags {"external:skip"} { ...@@ -696,6 +696,54 @@ tags {"external:skip"} {
set d2 [$redis debug digest] set d2 [$redis debug digest]
assert {$d1 eq $d2} assert {$d1 eq $d2}
} }
clean_aof_persistence $aof_dirpath
}
test {Multi Part AOF can create BASE (RDB format) when redis starts from empty} {
start_server_aof [list dir $server_path] {
set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
wait_done_loading $client
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.1${::base_aof_sufix}${::rdb_format_suffix}"]
assert_aof_manifest_content $aof_manifest_file {
{file appendonly.aof.1.base.rdb seq 1 type b}
{file appendonly.aof.1.incr.aof seq 1 type i}
}
$client set foo behavior
set d1 [$client debug digest]
$client debug loadaof
set d2 [$client debug digest]
assert {$d1 eq $d2}
}
clean_aof_persistence $aof_dirpath
}
test {Multi Part AOF can create BASE (AOF format) when redis starts from empty} {
start_server_aof [list dir $server_path aof-use-rdb-preamble no] {
set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
wait_done_loading $client
assert_equal 1 [check_file_exist $aof_dirpath "${aof_basename}.1${::base_aof_sufix}${::aof_format_suffix}"]
assert_aof_manifest_content $aof_manifest_file {
{file appendonly.aof.1.base.aof seq 1 type b}
{file appendonly.aof.1.incr.aof seq 1 type i}
}
$client set foo behavior
set d1 [$client debug digest]
$client debug loadaof
set d2 [$client debug digest]
assert {$d1 eq $d2}
}
clean_aof_persistence $aof_dirpath
} }
# Test Part 2 # Test Part 2
......
...@@ -189,14 +189,18 @@ void persistenceCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, ...@@ -189,14 +189,18 @@ void persistenceCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub,
switch (sub) { switch (sub) {
case REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START: keyname = "persistence-rdb-start"; break; case REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START: keyname = "persistence-rdb-start"; break;
case REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START: keyname = "persistence-aof-start"; break; case REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START: keyname = "persistence-aof-start"; break;
case REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_AOF_START: keyname = "persistence-syncaof-start"; break;
case REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START: keyname = "persistence-syncrdb-start"; break; case REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START: keyname = "persistence-syncrdb-start"; break;
case REDISMODULE_SUBEVENT_PERSISTENCE_ENDED: keyname = "persistence-end"; break; case REDISMODULE_SUBEVENT_PERSISTENCE_ENDED: keyname = "persistence-end"; break;
case REDISMODULE_SUBEVENT_PERSISTENCE_FAILED: keyname = "persistence-failed"; break; case REDISMODULE_SUBEVENT_PERSISTENCE_FAILED: keyname = "persistence-failed"; break;
} }
/* modifying the keyspace from the fork child is not an option, using log instead */ /* modifying the keyspace from the fork child is not an option, using log instead */
RedisModule_Log(ctx, "warning", "module-event-%s", keyname); RedisModule_Log(ctx, "warning", "module-event-%s", keyname);
if (sub == REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START) if (sub == REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START ||
sub == REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_AOF_START)
{
LogNumericEvent(ctx, keyname, 0); LogNumericEvent(ctx, keyname, 0);
}
} }
void loadingCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) void loadingCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data)
......
...@@ -2,6 +2,9 @@ set testmodule [file normalize tests/modules/hooks.so] ...@@ -2,6 +2,9 @@ set testmodule [file normalize tests/modules/hooks.so]
tags "modules" { tags "modules" {
start_server [list overrides [list loadmodule "$testmodule" appendonly yes]] { start_server [list overrides [list loadmodule "$testmodule" appendonly yes]] {
test {Test module aof save on server start from empty} {
assert {[r hooks.event_count persistence-syncaof-start] == 1}
}
test {Test clients connection / disconnection hooks} { test {Test clients connection / disconnection hooks} {
for {set j 0} {$j < 2} {incr j} { for {set j 0} {$j < 2} {incr j} {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment