Unverified Commit 070453ee authored by Wen Hui's avatar Wen Hui Committed by GitHub
Browse files

Cluster human readable nodename feature (#9564)



This PR adds a human readable name to a node in clusters that are visible as part of error logs. This is useful so that admins and operators of Redis cluster have better visibility into failures without having to cross-reference the generated ID with some logical identifier (such as pod-ID or EC2 instance ID). This is mentioned in #8948. Specific nodenames can be set by using the variable cluster-announce-human-nodename. The nodename is gossiped using the clusterbus extension in #9530.
Co-authored-by: default avatarMadelyn Olson <madelyneolson@gmail.com>
parent b00a2351
...@@ -1743,6 +1743,11 @@ aof-timestamp-enabled no ...@@ -1743,6 +1743,11 @@ aof-timestamp-enabled no
# #
# cluster-announce-hostname "" # cluster-announce-hostname ""
# Clusters can configure an optional nodename to be used in addition to the node ID for
# debugging and admin information. This name is broadcasted between nodes, so will be used
# in addition to the node ID when reporting cross node events such as node failures.
# cluster-announce-human-nodename ""
# Clusters can advertise how clients should connect to them using either their IP address, # Clusters can advertise how clients should connect to them using either their IP address,
# a user defined hostname, or by declaring they have no endpoint. Which endpoint is # a user defined hostname, or by declaring they have no endpoint. Which endpoint is
# shown as the preferred endpoint is set by using the cluster-preferred-endpoint-type # shown as the preferred endpoint is set by using the cluster-preferred-endpoint-type
......
...@@ -89,6 +89,9 @@ void clusterRemoveNodeFromShard(clusterNode *node); ...@@ -89,6 +89,9 @@ void clusterRemoveNodeFromShard(clusterNode *node);
int auxShardIdSetter(clusterNode *n, void *value, int length); int auxShardIdSetter(clusterNode *n, void *value, int length);
sds auxShardIdGetter(clusterNode *n, sds s); sds auxShardIdGetter(clusterNode *n, sds s);
int auxShardIdPresent(clusterNode *n); int auxShardIdPresent(clusterNode *n);
int auxHumanNodenameSetter(clusterNode *n, void *value, int length);
sds auxHumanNodenameGetter(clusterNode *n, sds s);
int auxHumanNodenamePresent(clusterNode *n);
static void clusterBuildMessageHdr(clusterMsg *hdr, int type, size_t msglen); static void clusterBuildMessageHdr(clusterMsg *hdr, int type, size_t msglen);
/* Links to the next and previous entries for keys in the same slot are stored /* Links to the next and previous entries for keys in the same slot are stored
...@@ -171,8 +174,8 @@ typedef struct { ...@@ -171,8 +174,8 @@ typedef struct {
/* Assign index to each aux field */ /* Assign index to each aux field */
typedef enum { typedef enum {
af_start, af_shard_id,
af_shard_id = af_start, af_human_nodename,
af_count, af_count,
} auxFieldIndex; } auxFieldIndex;
...@@ -182,14 +185,15 @@ typedef enum { ...@@ -182,14 +185,15 @@ typedef enum {
* 2. aux name can contain characters that pass the isValidAuxChar check only */ * 2. aux name can contain characters that pass the isValidAuxChar check only */
auxFieldHandler auxFieldHandlers[] = { auxFieldHandler auxFieldHandlers[] = {
{"shard-id", auxShardIdSetter, auxShardIdGetter, auxShardIdPresent}, {"shard-id", auxShardIdSetter, auxShardIdGetter, auxShardIdPresent},
{"nodename", auxHumanNodenameSetter, auxHumanNodenameGetter, auxHumanNodenamePresent},
}; };
int isValidAuxChar(int c) { int isValidAuxChar(int c) {
return isalnum(c) || (strchr("!#$%&()*+-.:;<>?@[]^_{|}~", c) != NULL); return isalnum(c) || (strchr("!#$%&()*+:;<>?@[]^{|}~", c) == NULL);
} }
int isValidAuxString(sds s) { int isValidAuxString(char *s, unsigned int length) {
for (unsigned i = 0; i < sdslen(s); i++) { for (unsigned i = 0; i < length; i++) {
if (!isValidAuxChar(s[i])) return 0; if (!isValidAuxChar(s[i])) return 0;
} }
return 1; return 1;
...@@ -219,6 +223,30 @@ int auxShardIdPresent(clusterNode *n) { ...@@ -219,6 +223,30 @@ int auxShardIdPresent(clusterNode *n) {
return strlen(n->shard_id); return strlen(n->shard_id);
} }
int auxHumanNodenameSetter(clusterNode *n, void *value, int length) {
if (n && !strncmp(value, n->human_nodename, length)) {
return C_OK;
} else if (!n && (length == 0)) {
return C_OK;
}
if (n) {
n->human_nodename = sdscpylen(n->human_nodename, value, length);
} else if (sdslen(n->human_nodename) != 0) {
sdsclear(n->human_nodename);
} else {
return C_ERR;
}
return C_OK;
}
sds auxHumanNodenameGetter(clusterNode *n, sds s) {
return sdscatprintf(s, "%.40s", n->human_nodename);
}
int auxHumanNodenamePresent(clusterNode *n) {
return strlen(n->human_nodename);
}
/* clusterLink send queue blocks */ /* clusterLink send queue blocks */
typedef struct { typedef struct {
size_t totlen; /* Total length of this block including the message */ size_t totlen; /* Total length of this block including the message */
...@@ -328,7 +356,7 @@ int clusterLoadConfig(char *filename) { ...@@ -328,7 +356,7 @@ int clusterLoadConfig(char *filename) {
clusterAddNode(n); clusterAddNode(n);
} }
/* Format for the node address and auxiliary argument information: /* Format for the node address and auxiliary argument information:
* ip:port[@cport][,hostname[,aux=val]*] */ * ip:port[@cport][,hostname][,aux=val]*] */
aux_argv = sdssplitlen(argv[1], sdslen(argv[1]), ",", 1, &aux_argc); aux_argv = sdssplitlen(argv[1], sdslen(argv[1]), ",", 1, &aux_argc);
if (aux_argv == NULL) { if (aux_argv == NULL) {
...@@ -362,7 +390,7 @@ int clusterLoadConfig(char *filename) { ...@@ -362,7 +390,7 @@ int clusterLoadConfig(char *filename) {
/* Validate that both aux and value contain valid characters only */ /* Validate that both aux and value contain valid characters only */
for (unsigned j = 0; j < 2; j++) { for (unsigned j = 0; j < 2; j++) {
if (!isValidAuxString(field_argv[j])) { if (!isValidAuxString(field_argv[j],sdslen(field_argv[j]))){
/* Invalid aux field format */ /* Invalid aux field format */
sdsfreesplitres(field_argv, field_argc); sdsfreesplitres(field_argv, field_argc);
sdsfreesplitres(argv,argc); sdsfreesplitres(argv,argc);
...@@ -396,7 +424,6 @@ int clusterLoadConfig(char *filename) { ...@@ -396,7 +424,6 @@ int clusterLoadConfig(char *filename) {
sdsfreesplitres(field_argv, field_argc); sdsfreesplitres(field_argv, field_argc);
} }
/* Address and port */ /* Address and port */
if ((p = strrchr(aux_argv[0],':')) == NULL) { if ((p = strrchr(aux_argv[0],':')) == NULL) {
sdsfreesplitres(aux_argv, aux_argc); sdsfreesplitres(aux_argv, aux_argc);
...@@ -811,6 +838,22 @@ static void updateAnnouncedHostname(clusterNode *node, char *new) { ...@@ -811,6 +838,22 @@ static void updateAnnouncedHostname(clusterNode *node, char *new) {
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG); clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG);
} }
static void updateAnnouncedHumanNodename(clusterNode *node, char *new) {
if (new && !strcmp(new, node->human_nodename)) {
return;
} else if (!new && (sdslen(node->human_nodename) == 0)) {
return;
}
if (new) {
node->human_nodename = sdscpy(node->human_nodename, new);
} else if (sdslen(node->human_nodename) != 0) {
sdsclear(node->human_nodename);
}
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG);
}
static void updateShardId(clusterNode *node, const char *shard_id) { static void updateShardId(clusterNode *node, const char *shard_id) {
if (memcmp(node->shard_id, shard_id, CLUSTER_NAMELEN) != 0) { if (memcmp(node->shard_id, shard_id, CLUSTER_NAMELEN) != 0) {
clusterRemoveNodeFromShard(node); clusterRemoveNodeFromShard(node);
...@@ -836,6 +879,11 @@ void clusterUpdateMyselfHostname(void) { ...@@ -836,6 +879,11 @@ void clusterUpdateMyselfHostname(void) {
updateAnnouncedHostname(myself, server.cluster_announce_hostname); updateAnnouncedHostname(myself, server.cluster_announce_hostname);
} }
void clusterUpdateMyselfHumanNodename(void) {
if (!myself) return;
updateAnnouncedHumanNodename(myself, server.cluster_announce_human_nodename);
}
void clusterInit(void) { void clusterInit(void) {
int saveconf = 0; int saveconf = 0;
...@@ -919,6 +967,7 @@ void clusterInit(void) { ...@@ -919,6 +967,7 @@ void clusterInit(void) {
clusterUpdateMyselfFlags(); clusterUpdateMyselfFlags();
clusterUpdateMyselfIp(); clusterUpdateMyselfIp();
clusterUpdateMyselfHostname(); clusterUpdateMyselfHostname();
clusterUpdateMyselfHumanNodename();
} }
void clusterInitListeners(void) { void clusterInitListeners(void) {
...@@ -1256,6 +1305,7 @@ clusterNode *createClusterNode(char *nodename, int flags) { ...@@ -1256,6 +1305,7 @@ clusterNode *createClusterNode(char *nodename, int flags) {
node->inbound_link = NULL; node->inbound_link = NULL;
memset(node->ip,0,sizeof(node->ip)); memset(node->ip,0,sizeof(node->ip));
node->hostname = sdsempty(); node->hostname = sdsempty();
node->human_nodename = sdsempty();
node->port = 0; node->port = 0;
node->cport = 0; node->cport = 0;
node->pport = 0; node->pport = 0;
...@@ -1422,6 +1472,7 @@ void freeClusterNode(clusterNode *n) { ...@@ -1422,6 +1472,7 @@ void freeClusterNode(clusterNode *n) {
serverAssert(dictDelete(server.cluster->nodes,nodename) == DICT_OK); serverAssert(dictDelete(server.cluster->nodes,nodename) == DICT_OK);
sdsfree(nodename); sdsfree(nodename);
sdsfree(n->hostname); sdsfree(n->hostname);
sdsfree(n->human_nodename);
/* Release links and associated data structures. */ /* Release links and associated data structures. */
if (n->link) freeClusterLink(n->link); if (n->link) freeClusterLink(n->link);
...@@ -1694,9 +1745,9 @@ void clusterHandleConfigEpochCollision(clusterNode *sender) { ...@@ -1694,9 +1745,9 @@ void clusterHandleConfigEpochCollision(clusterNode *sender) {
myself->configEpoch = server.cluster->currentEpoch; myself->configEpoch = server.cluster->currentEpoch;
clusterSaveConfigOrDie(1); clusterSaveConfigOrDie(1);
serverLog(LL_VERBOSE, serverLog(LL_VERBOSE,
"WARNING: configEpoch collision with node %.40s." "WARNING: configEpoch collision with node %.40s (%s)."
" configEpoch set to %llu", " configEpoch set to %llu",
sender->name, sender->name,sender->human_nodename,
(unsigned long long) myself->configEpoch); (unsigned long long) myself->configEpoch);
} }
...@@ -1812,7 +1863,7 @@ void markNodeAsFailingIfNeeded(clusterNode *node) { ...@@ -1812,7 +1863,7 @@ void markNodeAsFailingIfNeeded(clusterNode *node) {
if (failures < needed_quorum) return; /* No weak agreement from masters. */ if (failures < needed_quorum) return; /* No weak agreement from masters. */
serverLog(LL_NOTICE, serverLog(LL_NOTICE,
"Marking node %.40s as failing (quorum reached).", node->name); "Marking node %.40s (%s) as failing (quorum reached).", node->name, node->human_nodename);
/* Mark the node as failing. */ /* Mark the node as failing. */
node->flags &= ~CLUSTER_NODE_PFAIL; node->flags &= ~CLUSTER_NODE_PFAIL;
...@@ -1840,8 +1891,8 @@ void clearNodeFailureIfNeeded(clusterNode *node) { ...@@ -1840,8 +1891,8 @@ void clearNodeFailureIfNeeded(clusterNode *node) {
* node again. */ * node again. */
if (nodeIsSlave(node) || node->numslots == 0) { if (nodeIsSlave(node) || node->numslots == 0) {
serverLog(LL_NOTICE, serverLog(LL_NOTICE,
"Clear FAIL state for node %.40s: %s is reachable again.", "Clear FAIL state for node %.40s (%s):%s is reachable again.",
node->name, node->name,node->human_nodename,
nodeIsSlave(node) ? "replica" : "master without slots"); nodeIsSlave(node) ? "replica" : "master without slots");
node->flags &= ~CLUSTER_NODE_FAIL; node->flags &= ~CLUSTER_NODE_FAIL;
clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG); clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG);
...@@ -1856,8 +1907,8 @@ void clearNodeFailureIfNeeded(clusterNode *node) { ...@@ -1856,8 +1907,8 @@ void clearNodeFailureIfNeeded(clusterNode *node) {
(server.cluster_node_timeout * CLUSTER_FAIL_UNDO_TIME_MULT)) (server.cluster_node_timeout * CLUSTER_FAIL_UNDO_TIME_MULT))
{ {
serverLog(LL_NOTICE, serverLog(LL_NOTICE,
"Clear FAIL state for node %.40s: is reachable again and nobody is serving its slots after some time.", "Clear FAIL state for node %.40s (%s): is reachable again and nobody is serving its slots after some time.",
node->name); node->name, node->human_nodename);
node->flags &= ~CLUSTER_NODE_FAIL; node->flags &= ~CLUSTER_NODE_FAIL;
clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG); clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG);
} }
...@@ -1977,15 +2028,15 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) { ...@@ -1977,15 +2028,15 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) {
if (flags & (CLUSTER_NODE_FAIL|CLUSTER_NODE_PFAIL)) { if (flags & (CLUSTER_NODE_FAIL|CLUSTER_NODE_PFAIL)) {
if (clusterNodeAddFailureReport(node,sender)) { if (clusterNodeAddFailureReport(node,sender)) {
serverLog(LL_VERBOSE, serverLog(LL_VERBOSE,
"Node %.40s reported node %.40s as not reachable.", "Node %.40s (%s) reported node %.40s (%s) as not reachable.",
sender->name, node->name); sender->name, sender->human_nodename, node->name, node->human_nodename);
} }
markNodeAsFailingIfNeeded(node); markNodeAsFailingIfNeeded(node);
} else { } else {
if (clusterNodeDelFailureReport(node,sender)) { if (clusterNodeDelFailureReport(node,sender)) {
serverLog(LL_VERBOSE, serverLog(LL_VERBOSE,
"Node %.40s reported node %.40s is back online.", "Node %.40s (%s) reported node %.40s (%s) is back online.",
sender->name, node->name); sender->name, sender->human_nodename, node->name, node->human_nodename);
} }
} }
} }
...@@ -2121,8 +2172,8 @@ int nodeUpdateAddressIfNeeded(clusterNode *node, clusterLink *link, ...@@ -2121,8 +2172,8 @@ int nodeUpdateAddressIfNeeded(clusterNode *node, clusterLink *link,
node->cport = cport; node->cport = cport;
if (node->link) freeClusterLink(node->link); if (node->link) freeClusterLink(node->link);
node->flags &= ~CLUSTER_NODE_NOADDR; node->flags &= ~CLUSTER_NODE_NOADDR;
serverLog(LL_NOTICE,"Address updated for node %.40s, now %s:%d", serverLog(LL_NOTICE,"Address updated for node %.40s (%s), now %s:%d",
node->name, node->ip, node->port); node->name, node->human_nodename, node->ip, node->port);
/* Check if this is our master and we have to change the /* Check if this is our master and we have to change the
* replication target as well. */ * replication target as well. */
...@@ -2251,7 +2302,7 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc ...@@ -2251,7 +2302,7 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc
sender_slots == migrated_our_slots)) { sender_slots == migrated_our_slots)) {
serverLog(LL_NOTICE, serverLog(LL_NOTICE,
"Configuration change detected. Reconfiguring myself " "Configuration change detected. Reconfiguring myself "
"as a replica of %.40s", sender->name); "as a replica of %.40s (%s)", sender->name, sender->human_nodename);
clusterSetMaster(sender); clusterSetMaster(sender);
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG| clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|
CLUSTER_TODO_UPDATE_STATE| CLUSTER_TODO_UPDATE_STATE|
...@@ -2267,8 +2318,8 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc ...@@ -2267,8 +2318,8 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc
* into a replica if its last slot is removed. If no other node takes * into a replica if its last slot is removed. If no other node takes
* over the slot, there is nothing else to trigger replica migration. */ * over the slot, there is nothing else to trigger replica migration. */
serverLog(LL_NOTICE, serverLog(LL_NOTICE,
"I'm a sub-replica! Reconfiguring myself as a replica of grandmaster %.40s", "I'm a sub-replica! Reconfiguring myself as a replica of grandmaster %.40s (%s)",
myself->slaveof->slaveof->name); myself->slaveof->slaveof->name, myself->slaveof->slaveof->human_nodename);
clusterSetMaster(myself->slaveof->slaveof); clusterSetMaster(myself->slaveof->slaveof);
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG| clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|
CLUSTER_TODO_UPDATE_STATE| CLUSTER_TODO_UPDATE_STATE|
...@@ -2328,6 +2379,13 @@ uint32_t getHostnamePingExtSize(void) { ...@@ -2328,6 +2379,13 @@ uint32_t getHostnamePingExtSize(void) {
return getAlignedPingExtSize(sdslen(myself->hostname) + 1); return getAlignedPingExtSize(sdslen(myself->hostname) + 1);
} }
uint32_t getHumanNodenamePingExtSize(void) {
if (sdslen(myself->human_nodename) == 0) {
return 0;
}
return getAlignedPingExtSize(sdslen(myself->human_nodename) + 1);
}
uint32_t getShardIdPingExtSize(void) { uint32_t getShardIdPingExtSize(void) {
return getAlignedPingExtSize(sizeof(clusterMsgPingExtShardId)); return getAlignedPingExtSize(sizeof(clusterMsgPingExtShardId));
} }
...@@ -2376,6 +2434,20 @@ uint32_t writePingExt(clusterMsg *hdr, int gossipcount) { ...@@ -2376,6 +2434,20 @@ uint32_t writePingExt(clusterMsg *hdr, int gossipcount) {
extensions++; extensions++;
} }
if (sdslen(myself->human_nodename) != 0) {
if (cursor != NULL) {
/* Populate human_nodename */
clusterMsgPingExtHumanNodename *ext = preparePingExt(cursor, CLUSTERMSG_EXT_TYPE_HUMAN_NODENAME, getHumanNodenamePingExtSize());
memcpy(ext->human_nodename, myself->human_nodename, sdslen(myself->human_nodename));
/* Move the write cursor */
cursor = nextPingExt(cursor);
}
totlen += getHumanNodenamePingExtSize();
extensions++;
}
/* Gossip forgotten nodes */ /* Gossip forgotten nodes */
if (dictSize(server.cluster->nodes_black_list) > 0) { if (dictSize(server.cluster->nodes_black_list) > 0) {
dictIterator *di = dictGetIterator(server.cluster->nodes_black_list); dictIterator *di = dictGetIterator(server.cluster->nodes_black_list);
...@@ -2424,6 +2496,7 @@ uint32_t writePingExt(clusterMsg *hdr, int gossipcount) { ...@@ -2424,6 +2496,7 @@ uint32_t writePingExt(clusterMsg *hdr, int gossipcount) {
void clusterProcessPingExtensions(clusterMsg *hdr, clusterLink *link) { void clusterProcessPingExtensions(clusterMsg *hdr, clusterLink *link) {
clusterNode *sender = link->node ? link->node : clusterLookupNode(hdr->sender, CLUSTER_NAMELEN); clusterNode *sender = link->node ? link->node : clusterLookupNode(hdr->sender, CLUSTER_NAMELEN);
char *ext_hostname = NULL; char *ext_hostname = NULL;
char *ext_humannodename = NULL;
char *ext_shardid = NULL; char *ext_shardid = NULL;
uint16_t extensions = ntohs(hdr->extensions); uint16_t extensions = ntohs(hdr->extensions);
/* Loop through all the extensions and process them */ /* Loop through all the extensions and process them */
...@@ -2433,6 +2506,9 @@ void clusterProcessPingExtensions(clusterMsg *hdr, clusterLink *link) { ...@@ -2433,6 +2506,9 @@ void clusterProcessPingExtensions(clusterMsg *hdr, clusterLink *link) {
if (type == CLUSTERMSG_EXT_TYPE_HOSTNAME) { if (type == CLUSTERMSG_EXT_TYPE_HOSTNAME) {
clusterMsgPingExtHostname *hostname_ext = (clusterMsgPingExtHostname *) &(ext->ext[0].hostname); clusterMsgPingExtHostname *hostname_ext = (clusterMsgPingExtHostname *) &(ext->ext[0].hostname);
ext_hostname = hostname_ext->hostname; ext_hostname = hostname_ext->hostname;
} else if (type == CLUSTERMSG_EXT_TYPE_HUMAN_NODENAME) {
clusterMsgPingExtHumanNodename *humannodename_ext = (clusterMsgPingExtHumanNodename *) &(ext->ext[0].human_nodename);
ext_humannodename = humannodename_ext->human_nodename;
} else if (type == CLUSTERMSG_EXT_TYPE_FORGOTTEN_NODE) { } else if (type == CLUSTERMSG_EXT_TYPE_FORGOTTEN_NODE) {
clusterMsgPingExtForgottenNode *forgotten_node_ext = &(ext->ext[0].forgotten_node); clusterMsgPingExtForgottenNode *forgotten_node_ext = &(ext->ext[0].forgotten_node);
clusterNode *n = clusterLookupNode(forgotten_node_ext->name, CLUSTER_NAMELEN); clusterNode *n = clusterLookupNode(forgotten_node_ext->name, CLUSTER_NAMELEN);
...@@ -2461,6 +2537,7 @@ void clusterProcessPingExtensions(clusterMsg *hdr, clusterLink *link) { ...@@ -2461,6 +2537,7 @@ void clusterProcessPingExtensions(clusterMsg *hdr, clusterLink *link) {
* they don't have an announced hostname. Otherwise, we'll * they don't have an announced hostname. Otherwise, we'll
* set it now. */ * set it now. */
updateAnnouncedHostname(sender, ext_hostname); updateAnnouncedHostname(sender, ext_hostname);
updateAnnouncedHumanNodename(sender, ext_humannodename);
updateShardId(sender, ext_shardid); updateShardId(sender, ext_shardid);
} }
...@@ -2692,8 +2769,8 @@ int clusterProcessPacket(clusterLink *link) { ...@@ -2692,8 +2769,8 @@ int clusterProcessPacket(clusterLink *link) {
* IP/port of the node with the new one. */ * IP/port of the node with the new one. */
if (sender) { if (sender) {
serverLog(LL_VERBOSE, serverLog(LL_VERBOSE,
"Handshake: we already know node %.40s, " "Handshake: we already know node %.40s (%s), "
"updating the address if needed.", sender->name); "updating the address if needed.", sender->name, sender->human_nodename);
if (nodeUpdateAddressIfNeeded(sender,link,hdr)) if (nodeUpdateAddressIfNeeded(sender,link,hdr))
{ {
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG| clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|
...@@ -2904,8 +2981,8 @@ int clusterProcessPacket(clusterLink *link) { ...@@ -2904,8 +2981,8 @@ int clusterProcessPacket(clusterLink *link) {
!(failing->flags & (CLUSTER_NODE_FAIL|CLUSTER_NODE_MYSELF))) !(failing->flags & (CLUSTER_NODE_FAIL|CLUSTER_NODE_MYSELF)))
{ {
serverLog(LL_NOTICE, serverLog(LL_NOTICE,
"FAIL message received from %.40s about %.40s", "FAIL message received from %.40s (%s) about %.40s (%s)",
hdr->sender, hdr->data.fail.about.nodename); hdr->sender, sender->human_nodename, hdr->data.fail.about.nodename, failing->human_nodename);
failing->flags |= CLUSTER_NODE_FAIL; failing->flags |= CLUSTER_NODE_FAIL;
failing->fail_time = now; failing->fail_time = now;
failing->flags &= ~CLUSTER_NODE_PFAIL; failing->flags &= ~CLUSTER_NODE_PFAIL;
...@@ -2969,8 +3046,8 @@ int clusterProcessPacket(clusterLink *link) { ...@@ -2969,8 +3046,8 @@ int clusterProcessPacket(clusterLink *link) {
pauseActions(PAUSE_DURING_FAILOVER, pauseActions(PAUSE_DURING_FAILOVER,
now + (CLUSTER_MF_TIMEOUT * CLUSTER_MF_PAUSE_MULT), now + (CLUSTER_MF_TIMEOUT * CLUSTER_MF_PAUSE_MULT),
PAUSE_ACTIONS_CLIENT_WRITE_SET); PAUSE_ACTIONS_CLIENT_WRITE_SET);
serverLog(LL_NOTICE,"Manual failover requested by replica %.40s.", serverLog(LL_NOTICE,"Manual failover requested by replica %.40s (%s).",
sender->name); sender->name, sender->human_nodename);
/* We need to send a ping message to the replica, as it would carry /* We need to send a ping message to the replica, as it would carry
* `server.cluster->mf_master_offset`, which means the master paused clients * `server.cluster->mf_master_offset`, which means the master paused clients
* at offset `server.cluster->mf_master_offset`, so that the replica would * at offset `server.cluster->mf_master_offset`, so that the replica would
...@@ -3375,7 +3452,6 @@ void clusterSendPing(clusterLink *link, int type) { ...@@ -3375,7 +3452,6 @@ void clusterSendPing(clusterLink *link, int type) {
estlen = sizeof(clusterMsg) - sizeof(union clusterMsgData); estlen = sizeof(clusterMsg) - sizeof(union clusterMsgData);
estlen += (sizeof(clusterMsgDataGossip)*(wanted + pfail_wanted)); estlen += (sizeof(clusterMsgDataGossip)*(wanted + pfail_wanted));
estlen += writePingExt(NULL, 0); estlen += writePingExt(NULL, 0);
/* Note: clusterBuildMessageHdr() expects the buffer to be always at least /* Note: clusterBuildMessageHdr() expects the buffer to be always at least
* sizeof(clusterMsg) or more. */ * sizeof(clusterMsg) or more. */
if (estlen < (int)sizeof(clusterMsg)) estlen = sizeof(clusterMsg); if (estlen < (int)sizeof(clusterMsg)) estlen = sizeof(clusterMsg);
...@@ -3703,8 +3779,8 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) { ...@@ -3703,8 +3779,8 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) {
* request, if the request epoch was greater. */ * request, if the request epoch was greater. */
if (requestCurrentEpoch < server.cluster->currentEpoch) { if (requestCurrentEpoch < server.cluster->currentEpoch) {
serverLog(LL_WARNING, serverLog(LL_WARNING,
"Failover auth denied to %.40s: reqEpoch (%llu) < curEpoch(%llu)", "Failover auth denied to %.40s (%s): reqEpoch (%llu) < curEpoch(%llu)",
node->name, node->name, node->human_nodename,
(unsigned long long) requestCurrentEpoch, (unsigned long long) requestCurrentEpoch,
(unsigned long long) server.cluster->currentEpoch); (unsigned long long) server.cluster->currentEpoch);
return; return;
...@@ -3713,8 +3789,8 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) { ...@@ -3713,8 +3789,8 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) {
/* I already voted for this epoch? Return ASAP. */ /* I already voted for this epoch? Return ASAP. */
if (server.cluster->lastVoteEpoch == server.cluster->currentEpoch) { if (server.cluster->lastVoteEpoch == server.cluster->currentEpoch) {
serverLog(LL_WARNING, serverLog(LL_WARNING,
"Failover auth denied to %.40s: already voted for epoch %llu", "Failover auth denied to %.40s (%s): already voted for epoch %llu",
node->name, node->name, node->human_nodename,
(unsigned long long) server.cluster->currentEpoch); (unsigned long long) server.cluster->currentEpoch);
return; return;
} }
...@@ -3727,16 +3803,16 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) { ...@@ -3727,16 +3803,16 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) {
{ {
if (nodeIsMaster(node)) { if (nodeIsMaster(node)) {
serverLog(LL_WARNING, serverLog(LL_WARNING,
"Failover auth denied to %.40s: it is a master node", "Failover auth denied to %.40s (%s): it is a master node",
node->name); node->name, node->human_nodename);
} else if (master == NULL) { } else if (master == NULL) {
serverLog(LL_WARNING, serverLog(LL_WARNING,
"Failover auth denied to %.40s: I don't know its master", "Failover auth denied to %.40s (%s): I don't know its master",
node->name); node->name, node->human_nodename);
} else if (!nodeFailed(master)) { } else if (!nodeFailed(master)) {
serverLog(LL_WARNING, serverLog(LL_WARNING,
"Failover auth denied to %.40s: its master is up", "Failover auth denied to %.40s (%s): its master is up",
node->name); node->name, node->human_nodename);
} }
return; return;
} }
...@@ -3747,9 +3823,9 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) { ...@@ -3747,9 +3823,9 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) {
if (mstime() - node->slaveof->voted_time < server.cluster_node_timeout * 2) if (mstime() - node->slaveof->voted_time < server.cluster_node_timeout * 2)
{ {
serverLog(LL_WARNING, serverLog(LL_WARNING,
"Failover auth denied to %.40s: " "Failover auth denied to %.40s %s: "
"can't vote about this master before %lld milliseconds", "can't vote about this master before %lld milliseconds",
node->name, node->name, node->human_nodename,
(long long) ((server.cluster_node_timeout*2)- (long long) ((server.cluster_node_timeout*2)-
(mstime() - node->slaveof->voted_time))); (mstime() - node->slaveof->voted_time)));
return; return;
...@@ -3769,9 +3845,9 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) { ...@@ -3769,9 +3845,9 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) {
* is served by a master with a greater configEpoch than the one claimed * is served by a master with a greater configEpoch than the one claimed
* by the slave requesting our vote. Refuse to vote for this slave. */ * by the slave requesting our vote. Refuse to vote for this slave. */
serverLog(LL_WARNING, serverLog(LL_WARNING,
"Failover auth denied to %.40s: " "Failover auth denied to %.40s (%s): "
"slot %d epoch (%llu) > reqEpoch (%llu)", "slot %d epoch (%llu) > reqEpoch (%llu)",
node->name, j, node->name, node->human_nodename, j,
(unsigned long long) server.cluster->slots[j]->configEpoch, (unsigned long long) server.cluster->slots[j]->configEpoch,
(unsigned long long) requestConfigEpoch); (unsigned long long) requestConfigEpoch);
return; return;
...@@ -3782,8 +3858,8 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) { ...@@ -3782,8 +3858,8 @@ void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request) {
node->slaveof->voted_time = mstime(); node->slaveof->voted_time = mstime();
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|CLUSTER_TODO_FSYNC_CONFIG); clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|CLUSTER_TODO_FSYNC_CONFIG);
clusterSendFailoverAuth(node); clusterSendFailoverAuth(node);
serverLog(LL_NOTICE, "Failover auth granted to %.40s for epoch %llu", serverLog(LL_NOTICE, "Failover auth granted to %.40s (%s) for epoch %llu",
node->name, (unsigned long long) server.cluster->currentEpoch); node->name, node->human_nodename, (unsigned long long) server.cluster->currentEpoch);
} }
/* This function returns the "rank" of this instance, a slave, in the context /* This function returns the "rank" of this instance, a slave, in the context
...@@ -5007,24 +5083,20 @@ sds clusterGenNodeDescription(client *c, clusterNode *node, int use_pport) { ...@@ -5007,24 +5083,20 @@ sds clusterGenNodeDescription(client *c, clusterNode *node, int use_pport) {
/* Node coordinates */ /* Node coordinates */
ci = sdscatlen(sdsempty(),node->name,CLUSTER_NAMELEN); ci = sdscatlen(sdsempty(),node->name,CLUSTER_NAMELEN);
/* Node's ip/port and optional announced hostname */ ci = sdscatfmt(ci," %s:%i@%i",
node->ip,
port,
node->cport);
if (sdslen(node->hostname) != 0) { if (sdslen(node->hostname) != 0) {
ci = sdscatprintf(ci," %s:%i@%i,%s", ci = sdscatfmt(ci,",%s", node->hostname);
node->ip, }
port, if (sdslen(node->hostname) == 0) {
node->cport, ci = sdscatfmt(ci,",", 1);
node->hostname);
} else {
ci = sdscatprintf(ci," %s:%i@%i,",
node->ip,
port,
node->cport);
} }
/* Don't expose aux fields to any clients yet but do allow them /* Don't expose aux fields to any clients yet but do allow them
* to be persisted to nodes.conf */ * to be persisted to nodes.conf */
if (c == NULL) { if (c == NULL) {
for (int i = af_start; i < af_count; i++) { for (int i = af_count-1; i >=0; i--) {
if (auxFieldHandlers[i].isPresent(node)) { if (auxFieldHandlers[i].isPresent(node)) {
ci = sdscatprintf(ci, ",%s=", auxFieldHandlers[i].field); ci = sdscatprintf(ci, ",%s=", auxFieldHandlers[i].field);
ci = auxFieldHandlers[i].getter(node, ci); ci = auxFieldHandlers[i].getter(node, ci);
...@@ -5926,7 +5998,7 @@ NULL ...@@ -5926,7 +5998,7 @@ NULL
{ {
serverLog(LL_NOTICE, serverLog(LL_NOTICE,
"Configuration change detected. Reconfiguring myself " "Configuration change detected. Reconfiguring myself "
"as a replica of %.40s", n->name); "as a replica of %.40s (%s)", n->name, n->human_nodename);
clusterSetMaster(n); clusterSetMaster(n);
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG | clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG |
CLUSTER_TODO_UPDATE_STATE | CLUSTER_TODO_UPDATE_STATE |
......
...@@ -141,6 +141,7 @@ typedef struct clusterNode { ...@@ -141,6 +141,7 @@ typedef struct clusterNode {
long long repl_offset; /* Last known repl offset for this node. */ long long repl_offset; /* Last known repl offset for this node. */
char ip[NET_IP_STR_LEN]; /* Latest known IP address of this node */ char ip[NET_IP_STR_LEN]; /* Latest known IP address of this node */
sds hostname; /* The known hostname for this node */ sds hostname; /* The known hostname for this node */
sds human_nodename; /* The known human readable nodename for this node */
int port; /* Latest known clients port (TLS or plain). */ int port; /* Latest known clients port (TLS or plain). */
int pport; /* Latest known clients plaintext port. Only used int pport; /* Latest known clients plaintext port. Only used
if the main clients port is for TLS. */ if the main clients port is for TLS. */
...@@ -260,6 +261,7 @@ typedef struct { ...@@ -260,6 +261,7 @@ typedef struct {
* consistent manner. */ * consistent manner. */
typedef enum { typedef enum {
CLUSTERMSG_EXT_TYPE_HOSTNAME, CLUSTERMSG_EXT_TYPE_HOSTNAME,
CLUSTERMSG_EXT_TYPE_HUMAN_NODENAME,
CLUSTERMSG_EXT_TYPE_FORGOTTEN_NODE, CLUSTERMSG_EXT_TYPE_FORGOTTEN_NODE,
CLUSTERMSG_EXT_TYPE_SHARDID, CLUSTERMSG_EXT_TYPE_SHARDID,
} clusterMsgPingtypes; } clusterMsgPingtypes;
...@@ -271,6 +273,10 @@ typedef struct { ...@@ -271,6 +273,10 @@ typedef struct {
char hostname[1]; /* The announced hostname, ends with \0. */ char hostname[1]; /* The announced hostname, ends with \0. */
} clusterMsgPingExtHostname; } clusterMsgPingExtHostname;
typedef struct {
char human_nodename[1]; /* The announced nodename, ends with \0. */
} clusterMsgPingExtHumanNodename;
typedef struct { typedef struct {
char name[CLUSTER_NAMELEN]; /* Node name. */ char name[CLUSTER_NAMELEN]; /* Node name. */
uint64_t ttl; /* Remaining time to blacklist the node, in seconds. */ uint64_t ttl; /* Remaining time to blacklist the node, in seconds. */
...@@ -288,6 +294,7 @@ typedef struct { ...@@ -288,6 +294,7 @@ typedef struct {
uint16_t unused; /* 16 bits of padding to make this structure 8 byte aligned. */ uint16_t unused; /* 16 bits of padding to make this structure 8 byte aligned. */
union { union {
clusterMsgPingExtHostname hostname; clusterMsgPingExtHostname hostname;
clusterMsgPingExtHumanNodename human_nodename;
clusterMsgPingExtForgottenNode forgotten_node; clusterMsgPingExtForgottenNode forgotten_node;
clusterMsgPingExtShardId shard_id; clusterMsgPingExtShardId shard_id;
} ext[]; /* Actual extension information, formatted so that the data is 8 } ext[]; /* Actual extension information, formatted so that the data is 8
...@@ -425,5 +432,6 @@ void clusterUpdateMyselfAnnouncedPorts(void); ...@@ -425,5 +432,6 @@ void clusterUpdateMyselfAnnouncedPorts(void);
sds clusterGenNodesDescription(client *c, int filter, int use_pport); sds clusterGenNodesDescription(client *c, int filter, int use_pport);
sds genClusterInfoString(void); sds genClusterInfoString(void);
void freeClusterLink(clusterLink *link); void freeClusterLink(clusterLink *link);
void clusterUpdateMyselfHumanNodename(void);
int isValidAuxString(char *s, unsigned int length);
#endif /* __CLUSTER_H */ #endif /* __CLUSTER_H */
...@@ -38,6 +38,7 @@ ...@@ -38,6 +38,7 @@
#include <glob.h> #include <glob.h>
#include <string.h> #include <string.h>
#include <locale.h> #include <locale.h>
#include <ctype.h>
/*----------------------------------------------------------------------------- /*-----------------------------------------------------------------------------
* Config file name-value maps. * Config file name-value maps.
...@@ -2378,6 +2379,14 @@ static int isValidShutdownOnSigFlags(int val, const char **err) { ...@@ -2378,6 +2379,14 @@ static int isValidShutdownOnSigFlags(int val, const char **err) {
return 1; return 1;
} }
static int isValidAnnouncedNodename(char *val,const char **err) {
if (!(isValidAuxString(val,sdslen(val)))) {
*err = "Announced human node name contained invalid character";
return 0;
}
return 1;
}
static int isValidAnnouncedHostname(char *val, const char **err) { static int isValidAnnouncedHostname(char *val, const char **err) {
if (strlen(val) >= NET_HOST_STR_LEN) { if (strlen(val) >= NET_HOST_STR_LEN) {
*err = "Hostnames must be less than " *err = "Hostnames must be less than "
...@@ -2629,6 +2638,12 @@ int updateClusterHostname(const char **err) { ...@@ -2629,6 +2638,12 @@ int updateClusterHostname(const char **err) {
return 1; return 1;
} }
int updateClusterHumanNodename(const char **err) {
UNUSED(err);
clusterUpdateMyselfHumanNodename();
return 1;
}
static int applyTlsCfg(const char **err) { static int applyTlsCfg(const char **err) {
UNUSED(err); UNUSED(err);
...@@ -3085,6 +3100,7 @@ standardConfig static_configs[] = { ...@@ -3085,6 +3100,7 @@ standardConfig static_configs[] = {
createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL, NULL, updateClusterIp), createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL, NULL, updateClusterIp),
createStringConfig("cluster-config-file", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.cluster_configfile, "nodes.conf", NULL, NULL), createStringConfig("cluster-config-file", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.cluster_configfile, "nodes.conf", NULL, NULL),
createStringConfig("cluster-announce-hostname", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_hostname, NULL, isValidAnnouncedHostname, updateClusterHostname), createStringConfig("cluster-announce-hostname", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_hostname, NULL, isValidAnnouncedHostname, updateClusterHostname),
createStringConfig("cluster-announce-human-nodename", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_human_nodename, NULL, isValidAnnouncedNodename, updateClusterHumanNodename),
createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.syslog_ident, "redis", NULL, NULL), createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.syslog_ident, "redis", NULL, NULL),
createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG | PROTECTED_CONFIG, ALLOW_EMPTY_STRING, server.rdb_filename, "dump.rdb", isValidDBfilename, NULL), createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG | PROTECTED_CONFIG, ALLOW_EMPTY_STRING, server.rdb_filename, "dump.rdb", isValidDBfilename, NULL),
createStringConfig("appendfilename", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.aof_filename, "appendonly.aof", isValidAOFfilename, NULL), createStringConfig("appendfilename", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.aof_filename, "appendonly.aof", isValidAOFfilename, NULL),
......
...@@ -1979,6 +1979,7 @@ struct redisServer { ...@@ -1979,6 +1979,7 @@ struct redisServer {
if the master is in failure state. */ if the master is in failure state. */
char *cluster_announce_ip; /* IP address to announce on cluster bus. */ char *cluster_announce_ip; /* IP address to announce on cluster bus. */
char *cluster_announce_hostname; /* hostname to announce on cluster bus. */ char *cluster_announce_hostname; /* hostname to announce on cluster bus. */
char *cluster_announce_human_nodename; /* Human readable node name assigned to a node. */
int cluster_preferred_endpoint_type; /* Use the announced hostname when available. */ int cluster_preferred_endpoint_type; /* Use the announced hostname when available. */
int cluster_announce_port; /* base port to announce on cluster bus. */ int cluster_announce_port; /* base port to announce on cluster bus. */
int cluster_announce_tls_port; /* TLS port to announce on cluster bus. */ int cluster_announce_tls_port; /* TLS port to announce on cluster bus. */
......
...@@ -150,3 +150,28 @@ proc get_cluster_nodes id { ...@@ -150,3 +150,28 @@ proc get_cluster_nodes id {
} }
return $nodes return $nodes
} }
# Returns 1 if no node knows node_id, 0 if any node knows it.
proc node_is_forgotten {node_id} {
for {set j 0} {$j < [llength $::servers]} {incr j} {
set cluster_nodes [R $j CLUSTER NODES]
if { [string match "*$node_id*" $cluster_nodes] } {
return 0
}
}
return 1
}
# Isolate a node from the cluster and give it a new nodeid
proc isolate_node {id} {
set node_id [R $id CLUSTER MYID]
R $id CLUSTER RESET HARD
# Here we additionally test that CLUSTER FORGET propagates to all nodes.
set other_id [expr $id == 0 ? 1 : 0]
R $other_id CLUSTER FORGET $node_id
wait_for_condition 50 100 {
[node_is_forgotten $node_id]
} else {
fail "CLUSTER FORGET was not propagated to all nodes"
}
}
...@@ -98,6 +98,7 @@ set ::all_tests { ...@@ -98,6 +98,7 @@ set ::all_tests {
unit/cluster/cli unit/cluster/cli
unit/cluster/scripting unit/cluster/scripting
unit/cluster/hostnames unit/cluster/hostnames
unit/cluster/human-announced-nodename
unit/cluster/multi-slot-operations unit/cluster/multi-slot-operations
unit/cluster/slot-ownership unit/cluster/slot-ownership
unit/cluster/links unit/cluster/links
......
# Returns 1 if no node knows node_id, 0 if any node knows it.
proc node_is_forgotten {node_id} {
for {set j 0} {$j < [llength $::servers]} {incr j} {
set cluster_nodes [R $j CLUSTER NODES]
if { [string match "*$node_id*" $cluster_nodes] } {
return 0
}
}
return 1
}
# Isolate a node from the cluster and give it a new nodeid
proc isolate_node {id} {
set node_id [R $id CLUSTER MYID]
R $id CLUSTER RESET HARD
# Here we additionally test that CLUSTER FORGET propagates to all nodes.
set other_id [expr $id == 0 ? 1 : 0]
R $other_id CLUSTER FORGET $node_id
wait_for_condition 50 100 {
[node_is_forgotten $node_id]
} else {
fail "CLUSTER FORGET was not propagated to all nodes"
}
}
# Check if cluster's view of hostnames is consistent # Check if cluster's view of hostnames is consistent
proc are_hostnames_propagated {match_string} { proc are_hostnames_propagated {match_string} {
for {set j 0} {$j < [llength $::servers]} {incr j} { for {set j 0} {$j < [llength $::servers]} {incr j} {
......
# Check if cluster's view of human announced nodename is reported in logs
start_cluster 3 0 {tags {external:skip cluster}} {
test "Set cluster human announced nodename and let it propagate" {
for {set j 0} {$j < [llength $::servers]} {incr j} {
R $j config set cluster-announce-human-nodename "nodename-$j"
}
# We wait for everyone to agree on the epoch bump, which means everyone
# has exchanged messages so they know about the nodenames.
R 0 CLUSTER BUMPEPOCH
wait_for_condition 1000 50 {
[CI 0 cluster_current_epoch] == [CI 1 cluster_current_epoch] &&
[CI 0 cluster_current_epoch] == [CI 2 cluster_current_epoch]
} else {
fail "Cluster did not converge"
}
}
test "Human nodenames are visible in log messages" {
# Pause instance 0, so everyone thinks it is dead
pause_process [srv 0 pid]
# We're going to use a message we will know will be sent, node unreachable,
# since it includes the other node gossiping.
wait_for_log_messages -1 {"*Node * (nodename-2) reported node * (nodename-0) as not reachable*"} 0 10 500
wait_for_log_messages -2 {"*Node * (nodename-1) reported node * (nodename-0) as not reachable*"} 0 10 500
resume_process [srv 0 pid]
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment