Unverified Commit 070453ee authored by Wen Hui's avatar Wen Hui Committed by GitHub
Browse files

Cluster human readable nodename feature (#9564)



This PR adds a human readable name to a node in clusters that are visible as part of error logs. This is useful so that admins and operators of Redis cluster have better visibility into failures without having to cross-reference the generated ID with some logical identifier (such as pod-ID or EC2 instance ID). This is mentioned in #8948. Specific nodenames can be set by using the variable cluster-announce-human-nodename. The nodename is gossiped using the clusterbus extension in #9530.
Co-authored-by: default avatarMadelyn Olson <madelyneolson@gmail.com>
parent b00a2351
...@@ -1743,6 +1743,11 @@ aof-timestamp-enabled no ...@@ -1743,6 +1743,11 @@ aof-timestamp-enabled no
# #
# cluster-announce-hostname "" # cluster-announce-hostname ""
# Clusters can configure an optional nodename to be used in addition to the node ID for
# debugging and admin information. This name is broadcasted between nodes, so will be used
# in addition to the node ID when reporting cross node events such as node failures.
# cluster-announce-human-nodename ""
# Clusters can advertise how clients should connect to them using either their IP address, # Clusters can advertise how clients should connect to them using either their IP address,
# a user defined hostname, or by declaring they have no endpoint. Which endpoint is # a user defined hostname, or by declaring they have no endpoint. Which endpoint is
# shown as the preferred endpoint is set by using the cluster-preferred-endpoint-type # shown as the preferred endpoint is set by using the cluster-preferred-endpoint-type
......
This diff is collapsed.
...@@ -141,6 +141,7 @@ typedef struct clusterNode { ...@@ -141,6 +141,7 @@ typedef struct clusterNode {
long long repl_offset; /* Last known repl offset for this node. */ long long repl_offset; /* Last known repl offset for this node. */
char ip[NET_IP_STR_LEN]; /* Latest known IP address of this node */ char ip[NET_IP_STR_LEN]; /* Latest known IP address of this node */
sds hostname; /* The known hostname for this node */ sds hostname; /* The known hostname for this node */
sds human_nodename; /* The known human readable nodename for this node */
int port; /* Latest known clients port (TLS or plain). */ int port; /* Latest known clients port (TLS or plain). */
int pport; /* Latest known clients plaintext port. Only used int pport; /* Latest known clients plaintext port. Only used
if the main clients port is for TLS. */ if the main clients port is for TLS. */
...@@ -260,6 +261,7 @@ typedef struct { ...@@ -260,6 +261,7 @@ typedef struct {
* consistent manner. */ * consistent manner. */
typedef enum { typedef enum {
CLUSTERMSG_EXT_TYPE_HOSTNAME, CLUSTERMSG_EXT_TYPE_HOSTNAME,
CLUSTERMSG_EXT_TYPE_HUMAN_NODENAME,
CLUSTERMSG_EXT_TYPE_FORGOTTEN_NODE, CLUSTERMSG_EXT_TYPE_FORGOTTEN_NODE,
CLUSTERMSG_EXT_TYPE_SHARDID, CLUSTERMSG_EXT_TYPE_SHARDID,
} clusterMsgPingtypes; } clusterMsgPingtypes;
...@@ -271,6 +273,10 @@ typedef struct { ...@@ -271,6 +273,10 @@ typedef struct {
char hostname[1]; /* The announced hostname, ends with \0. */ char hostname[1]; /* The announced hostname, ends with \0. */
} clusterMsgPingExtHostname; } clusterMsgPingExtHostname;
typedef struct {
char human_nodename[1]; /* The announced nodename, ends with \0. */
} clusterMsgPingExtHumanNodename;
typedef struct { typedef struct {
char name[CLUSTER_NAMELEN]; /* Node name. */ char name[CLUSTER_NAMELEN]; /* Node name. */
uint64_t ttl; /* Remaining time to blacklist the node, in seconds. */ uint64_t ttl; /* Remaining time to blacklist the node, in seconds. */
...@@ -288,6 +294,7 @@ typedef struct { ...@@ -288,6 +294,7 @@ typedef struct {
uint16_t unused; /* 16 bits of padding to make this structure 8 byte aligned. */ uint16_t unused; /* 16 bits of padding to make this structure 8 byte aligned. */
union { union {
clusterMsgPingExtHostname hostname; clusterMsgPingExtHostname hostname;
clusterMsgPingExtHumanNodename human_nodename;
clusterMsgPingExtForgottenNode forgotten_node; clusterMsgPingExtForgottenNode forgotten_node;
clusterMsgPingExtShardId shard_id; clusterMsgPingExtShardId shard_id;
} ext[]; /* Actual extension information, formatted so that the data is 8 } ext[]; /* Actual extension information, formatted so that the data is 8
...@@ -425,5 +432,6 @@ void clusterUpdateMyselfAnnouncedPorts(void); ...@@ -425,5 +432,6 @@ void clusterUpdateMyselfAnnouncedPorts(void);
sds clusterGenNodesDescription(client *c, int filter, int use_pport); sds clusterGenNodesDescription(client *c, int filter, int use_pport);
sds genClusterInfoString(void); sds genClusterInfoString(void);
void freeClusterLink(clusterLink *link); void freeClusterLink(clusterLink *link);
void clusterUpdateMyselfHumanNodename(void);
int isValidAuxString(char *s, unsigned int length);
#endif /* __CLUSTER_H */ #endif /* __CLUSTER_H */
...@@ -38,6 +38,7 @@ ...@@ -38,6 +38,7 @@
#include <glob.h> #include <glob.h>
#include <string.h> #include <string.h>
#include <locale.h> #include <locale.h>
#include <ctype.h>
/*----------------------------------------------------------------------------- /*-----------------------------------------------------------------------------
* Config file name-value maps. * Config file name-value maps.
...@@ -2378,6 +2379,14 @@ static int isValidShutdownOnSigFlags(int val, const char **err) { ...@@ -2378,6 +2379,14 @@ static int isValidShutdownOnSigFlags(int val, const char **err) {
return 1; return 1;
} }
static int isValidAnnouncedNodename(char *val,const char **err) {
if (!(isValidAuxString(val,sdslen(val)))) {
*err = "Announced human node name contained invalid character";
return 0;
}
return 1;
}
static int isValidAnnouncedHostname(char *val, const char **err) { static int isValidAnnouncedHostname(char *val, const char **err) {
if (strlen(val) >= NET_HOST_STR_LEN) { if (strlen(val) >= NET_HOST_STR_LEN) {
*err = "Hostnames must be less than " *err = "Hostnames must be less than "
...@@ -2629,6 +2638,12 @@ int updateClusterHostname(const char **err) { ...@@ -2629,6 +2638,12 @@ int updateClusterHostname(const char **err) {
return 1; return 1;
} }
int updateClusterHumanNodename(const char **err) {
UNUSED(err);
clusterUpdateMyselfHumanNodename();
return 1;
}
static int applyTlsCfg(const char **err) { static int applyTlsCfg(const char **err) {
UNUSED(err); UNUSED(err);
...@@ -3085,6 +3100,7 @@ standardConfig static_configs[] = { ...@@ -3085,6 +3100,7 @@ standardConfig static_configs[] = {
createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL, NULL, updateClusterIp), createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL, NULL, updateClusterIp),
createStringConfig("cluster-config-file", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.cluster_configfile, "nodes.conf", NULL, NULL), createStringConfig("cluster-config-file", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.cluster_configfile, "nodes.conf", NULL, NULL),
createStringConfig("cluster-announce-hostname", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_hostname, NULL, isValidAnnouncedHostname, updateClusterHostname), createStringConfig("cluster-announce-hostname", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_hostname, NULL, isValidAnnouncedHostname, updateClusterHostname),
createStringConfig("cluster-announce-human-nodename", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_human_nodename, NULL, isValidAnnouncedNodename, updateClusterHumanNodename),
createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.syslog_ident, "redis", NULL, NULL), createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.syslog_ident, "redis", NULL, NULL),
createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG | PROTECTED_CONFIG, ALLOW_EMPTY_STRING, server.rdb_filename, "dump.rdb", isValidDBfilename, NULL), createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG | PROTECTED_CONFIG, ALLOW_EMPTY_STRING, server.rdb_filename, "dump.rdb", isValidDBfilename, NULL),
createStringConfig("appendfilename", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.aof_filename, "appendonly.aof", isValidAOFfilename, NULL), createStringConfig("appendfilename", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.aof_filename, "appendonly.aof", isValidAOFfilename, NULL),
......
...@@ -1979,6 +1979,7 @@ struct redisServer { ...@@ -1979,6 +1979,7 @@ struct redisServer {
if the master is in failure state. */ if the master is in failure state. */
char *cluster_announce_ip; /* IP address to announce on cluster bus. */ char *cluster_announce_ip; /* IP address to announce on cluster bus. */
char *cluster_announce_hostname; /* hostname to announce on cluster bus. */ char *cluster_announce_hostname; /* hostname to announce on cluster bus. */
char *cluster_announce_human_nodename; /* Human readable node name assigned to a node. */
int cluster_preferred_endpoint_type; /* Use the announced hostname when available. */ int cluster_preferred_endpoint_type; /* Use the announced hostname when available. */
int cluster_announce_port; /* base port to announce on cluster bus. */ int cluster_announce_port; /* base port to announce on cluster bus. */
int cluster_announce_tls_port; /* TLS port to announce on cluster bus. */ int cluster_announce_tls_port; /* TLS port to announce on cluster bus. */
......
...@@ -150,3 +150,28 @@ proc get_cluster_nodes id { ...@@ -150,3 +150,28 @@ proc get_cluster_nodes id {
} }
return $nodes return $nodes
} }
# Returns 1 if no node knows node_id, 0 if any node knows it.
proc node_is_forgotten {node_id} {
for {set j 0} {$j < [llength $::servers]} {incr j} {
set cluster_nodes [R $j CLUSTER NODES]
if { [string match "*$node_id*" $cluster_nodes] } {
return 0
}
}
return 1
}
# Isolate a node from the cluster and give it a new nodeid
proc isolate_node {id} {
set node_id [R $id CLUSTER MYID]
R $id CLUSTER RESET HARD
# Here we additionally test that CLUSTER FORGET propagates to all nodes.
set other_id [expr $id == 0 ? 1 : 0]
R $other_id CLUSTER FORGET $node_id
wait_for_condition 50 100 {
[node_is_forgotten $node_id]
} else {
fail "CLUSTER FORGET was not propagated to all nodes"
}
}
...@@ -98,6 +98,7 @@ set ::all_tests { ...@@ -98,6 +98,7 @@ set ::all_tests {
unit/cluster/cli unit/cluster/cli
unit/cluster/scripting unit/cluster/scripting
unit/cluster/hostnames unit/cluster/hostnames
unit/cluster/human-announced-nodename
unit/cluster/multi-slot-operations unit/cluster/multi-slot-operations
unit/cluster/slot-ownership unit/cluster/slot-ownership
unit/cluster/links unit/cluster/links
......
# Returns 1 if no node knows node_id, 0 if any node knows it.
proc node_is_forgotten {node_id} {
for {set j 0} {$j < [llength $::servers]} {incr j} {
set cluster_nodes [R $j CLUSTER NODES]
if { [string match "*$node_id*" $cluster_nodes] } {
return 0
}
}
return 1
}
# Isolate a node from the cluster and give it a new nodeid
proc isolate_node {id} {
set node_id [R $id CLUSTER MYID]
R $id CLUSTER RESET HARD
# Here we additionally test that CLUSTER FORGET propagates to all nodes.
set other_id [expr $id == 0 ? 1 : 0]
R $other_id CLUSTER FORGET $node_id
wait_for_condition 50 100 {
[node_is_forgotten $node_id]
} else {
fail "CLUSTER FORGET was not propagated to all nodes"
}
}
# Check if cluster's view of hostnames is consistent # Check if cluster's view of hostnames is consistent
proc are_hostnames_propagated {match_string} { proc are_hostnames_propagated {match_string} {
for {set j 0} {$j < [llength $::servers]} {incr j} { for {set j 0} {$j < [llength $::servers]} {incr j} {
......
# Check if cluster's view of human announced nodename is reported in logs
start_cluster 3 0 {tags {external:skip cluster}} {
test "Set cluster human announced nodename and let it propagate" {
for {set j 0} {$j < [llength $::servers]} {incr j} {
R $j config set cluster-announce-human-nodename "nodename-$j"
}
# We wait for everyone to agree on the epoch bump, which means everyone
# has exchanged messages so they know about the nodenames.
R 0 CLUSTER BUMPEPOCH
wait_for_condition 1000 50 {
[CI 0 cluster_current_epoch] == [CI 1 cluster_current_epoch] &&
[CI 0 cluster_current_epoch] == [CI 2 cluster_current_epoch]
} else {
fail "Cluster did not converge"
}
}
test "Human nodenames are visible in log messages" {
# Pause instance 0, so everyone thinks it is dead
pause_process [srv 0 pid]
# We're going to use a message we will know will be sent, node unreachable,
# since it includes the other node gossiping.
wait_for_log_messages -1 {"*Node * (nodename-2) reported node * (nodename-0) as not reachable*"} 0 10 500
wait_for_log_messages -2 {"*Node * (nodename-1) reported node * (nodename-0) as not reachable*"} 0 10 500
resume_process [srv 0 pid]
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment