Unverified Commit 5460c100 authored by Madelyn Olson's avatar Madelyn Olson Committed by GitHub
Browse files

Implement clusterbus message extensions and cluster hostname support (#9530)

Implement the ability for cluster nodes to advertise their location with extension messages.
parent 9f888576
......@@ -1632,6 +1632,32 @@ lua-time-limit 5000
# PubSub message by default. (client-query-buffer-limit default value is 1gb)
#
# cluster-link-sendbuf-limit 0
# Clusters can configure their announced hostname using this config. This is a common use case for
# applications that need to use TLS Server Name Indication (SNI) or dealing with DNS based
# routing. By default this value is only shown as additional metadata in the CLUSTER SLOTS
# command, but can be changed using 'cluster-preferred-endpoint-type' config. This value is
# communicated along the clusterbus to all nodes, setting it to an empty string will remove
# the hostname and also propgate the removal.
#
# cluster-announce-hostname ""
# Clusters can advertise how clients should connect to them using either their IP address,
# a user defined hostname, or by declaring they have no endpoint. Which endpoint is
# shown as the preferred endpoint is set by using the cluster-preferred-endpoint-type
# config with values 'ip', 'hostname', or 'unknown-endpoint'. This value controls how
# the endpoint returned for MOVED/ASKING requests as well as the first field of CLUSTER SLOTS.
# If the preferred endpoint type is set to hostname, but no announced hostname is set, a '?'
# will be returned instead.
#
# When a cluster advertises itself as having an unknown endpoint, it's indicating that
# the server doesn't know how clients can reach the cluster. This can happen in certain
# networking situations where there are multiple possible routes to the node, and the
# server doesn't know which one the client took. In this case, the server is expecting
# the client to reach out on the same endpoint it used for making the last request, but use
# the port provided in the response.
#
# cluster-preferred-endpoint-type ip
# In order to setup your cluster make sure to read the documentation
# available at https://redis.io web site.
......
This diff is collapsed.
......@@ -135,6 +135,7 @@ typedef struct clusterNode {
mstime_t orphaned_time; /* Starting time of orphaned master condition */
long long repl_offset; /* Last known repl offset for this node. */
char ip[NET_IP_STR_LEN]; /* Latest known IP address of this node */
char *hostname; /* The known hostname for this node */
int port; /* Latest known clients port (TLS or plain). */
int pport; /* Latest known clients plaintext port. Only used
if the main clients port is for TLS. */
......@@ -245,11 +246,38 @@ typedef struct {
unsigned char bulk_data[3]; /* 3 bytes just as placeholder. */
} clusterMsgModule;
/* The cluster supports optional extension messages that can be sent
* along with ping/pong/meet messages to give additional info in a
* consistent manner. */
typedef enum {
CLUSTERMSG_EXT_TYPE_HOSTNAME,
} clusterMsgPingtypes;
/* Helper function for making sure extensions are eight byte aligned. */
#define EIGHT_BYTE_ALIGN(size) ((((size) + 7) / 8) * 8)
typedef struct {
char hostname[1]; /* The announced hostname, ends with \0. */
} clusterMsgPingExtHostname;
typedef struct {
uint32_t length; /* Total length of this extension message (including this header) */
uint16_t type; /* Type of this extension message (see clusterMsgPingExtTypes) */
uint16_t unused; /* 16 bits of padding to make this structure 8 byte aligned. */
union {
clusterMsgPingExtHostname hostname;
} ext[]; /* Actual extension information, formatted so that the data is 8
* byte aligned, regardless of its content. */
} clusterMsgPingExt;
union clusterMsgData {
/* PING, MEET and PONG */
struct {
/* Array of N clusterMsgDataGossip structures */
clusterMsgDataGossip gossip[1];
/* Extension data that can optionally be sent for ping/meet/pong
* messages. We can't explicitly define them here though, since
* the gossip array isn't the real length of the gossip data. */
} ping;
/* FAIL */
......@@ -292,7 +320,8 @@ typedef struct {
unsigned char myslots[CLUSTER_SLOTS/8];
char slaveof[CLUSTER_NAMELEN];
char myip[NET_IP_STR_LEN]; /* Sender IP, if not all zeroed. */
char notused1[32]; /* 32 bytes reserved for future usage. */
uint16_t extensions; /* Number of extensions sent along with this packet. */
char notused1[16]; /* 16 bytes reserved for future usage. */
uint16_t pport; /* Sender TCP plaintext port, if base port is TLS */
uint16_t cport; /* Sender TCP cluster bus port */
uint16_t flags; /* Sender node flags */
......@@ -308,6 +337,7 @@ typedef struct {
#define CLUSTERMSG_FLAG0_PAUSED (1<<0) /* Master paused for manual failover. */
#define CLUSTERMSG_FLAG0_FORCEACK (1<<1) /* Give ACK to AUTH_REQUEST even if
master is up. */
#define CLUSTERMSG_FLAG0_EXT_DATA (1<<2) /* Message contains extension data */
/* ---------------------- API exported outside cluster.c -------------------- */
void clusterInit(void);
......@@ -334,5 +364,6 @@ void clusterUpdateMyselfFlags(void);
void clusterUpdateMyselfIp(void);
void slotToChannelAdd(sds channel);
void slotToChannelDel(sds channel);
void clusterUpdateMyselfHostname(void);
#endif /* __CLUSTER_H */
......@@ -141,6 +141,13 @@ configEnum protected_action_enum[] = {
{NULL, 0}
};
configEnum cluster_preferred_endpoint_type_enum[] = {
{"ip", CLUSTER_ENDPOINT_TYPE_IP},
{"hostname", CLUSTER_ENDPOINT_TYPE_HOSTNAME},
{"unknown-endpoint", CLUSTER_ENDPOINT_TYPE_UNKNOWN_ENDPOINT},
{NULL, 0}
};
/* Output buffer limits presets. */
clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = {
{0, 0, 0}, /* normal */
......@@ -2150,6 +2157,30 @@ static int isValidAOFfilename(char *val, const char **err) {
return 1;
}
static int isValidAnnouncedHostname(char *val, const char **err) {
if (strlen(val) >= NET_HOST_STR_LEN) {
*err = "Hostnames must be less than "
STRINGIFY(NET_HOST_STR_LEN) " characters";
return 0;
}
int i = 0;
char c;
while ((c = val[i])) {
/* We just validate the character set to make sure that everything
* is parsed and handled correctly. */
if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
|| (c >= '0' && c <= '9') || (c == '-') || (c == '.')))
{
*err = "Hostnames may only contain alphanumeric characters, "
"hyphens or dots";
return 0;
}
c = val[i++];
}
return 1;
}
/* Validate specified string is a valid proc-title-template */
static int isValidProcTitleTemplate(char *val, const char **err) {
if (!validateProcTitleTemplate(val)) {
......@@ -2305,6 +2336,12 @@ static int updateClusterIp(const char **err) {
return 1;
}
int updateClusterHostname(const char **err) {
UNUSED(err);
clusterUpdateMyselfHostname();
return 1;
}
#ifdef USE_OPENSSL
static int applyTlsCfg(const char **err) {
UNUSED(err);
......@@ -2652,6 +2689,7 @@ standardConfig configs[] = {
createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG | SENSITIVE_CONFIG, EMPTY_STRING_IS_NULL, server.masteruser, NULL, NULL, NULL),
createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL, NULL, updateClusterIp),
createStringConfig("cluster-config-file", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.cluster_configfile, "nodes.conf", NULL, NULL),
createStringConfig("cluster-announce-hostname", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_hostname, NULL, isValidAnnouncedHostname, updateClusterHostname),
createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.syslog_ident, "redis", NULL, NULL),
createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG | PROTECTED_CONFIG, ALLOW_EMPTY_STRING, server.rdb_filename, "dump.rdb", isValidDBfilename, NULL),
createStringConfig("appendfilename", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.aof_filename, "appendonly.aof", isValidAOFfilename, NULL),
......@@ -2681,6 +2719,7 @@ standardConfig configs[] = {
createEnumConfig("enable-protected-configs", NULL, IMMUTABLE_CONFIG, protected_action_enum, server.enable_protected_configs, PROTECTED_ACTION_ALLOWED_NO, NULL, NULL),
createEnumConfig("enable-debug-command", NULL, IMMUTABLE_CONFIG, protected_action_enum, server.enable_debug_cmd, PROTECTED_ACTION_ALLOWED_NO, NULL, NULL),
createEnumConfig("enable-module-command", NULL, IMMUTABLE_CONFIG, protected_action_enum, server.enable_module_cmd, PROTECTED_ACTION_ALLOWED_NO, NULL, NULL),
createEnumConfig("cluster-preferred-endpoint-type", NULL, MODIFIABLE_CONFIG, cluster_preferred_endpoint_type_enum, server.cluster_preferred_endpoint_type, CLUSTER_ENDPOINT_TYPE_IP, NULL, NULL),
/* Integer configs */
createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, 16, INTEGER_CONFIG, NULL, NULL),
......
......@@ -425,6 +425,8 @@ void debugCommand(client *c) {
#endif
"OBJECT <key>",
" Show low level info about `key` and associated value.",
"DROP-CLUSTER-PACKET-FILTER <packet-type>",
" Drop all packets that match the filtered type. Set to -1 allow all packets.",
"OOM",
" Crash the server simulating an out-of-memory error.",
"PANIC",
......@@ -575,6 +577,12 @@ NULL
server.dirty = 0; /* Prevent AOF / replication */
serverLog(LL_WARNING,"Append Only File loaded by DEBUG LOADAOF");
addReply(c,shared.ok);
} else if (!strcasecmp(c->argv[1]->ptr,"drop-cluster-packet-filter") && c->argc == 3) {
long packet_type;
if (getLongFromObjectOrReply(c, c->argv[2], &packet_type, NULL) != C_OK)
return;
server.cluster_drop_packet_filter = packet_type;
addReply(c,shared.ok);
} else if (!strcasecmp(c->argv[1]->ptr,"object") && c->argc == 3) {
dictEntry *de;
robj *val;
......
......@@ -2293,6 +2293,7 @@ void initServer(void) {
server.blocked_last_cron = 0;
server.blocking_op_nesting = 0;
server.thp_enabled = 0;
server.cluster_drop_packet_filter = -1;
resetReplicationBuffer();
if ((server.tls_port || server.tls_replication || server.tls_cluster)
......
......@@ -527,6 +527,13 @@ typedef struct {
mstime_t end;
} pause_event;
/* Ways that a clusters endpoint can be described */
typedef enum {
CLUSTER_ENDPOINT_TYPE_IP = 0, /* Show IP address */
CLUSTER_ENDPOINT_TYPE_HOSTNAME, /* Show hostname */
CLUSTER_ENDPOINT_TYPE_UNKNOWN_ENDPOINT /* Show NULL or empty */
} cluster_endpoint_type;
/* RDB active child save type. */
#define RDB_CHILD_TYPE_NONE 0
#define RDB_CHILD_TYPE_DISK 1 /* RDB is written to disk. */
......@@ -1771,6 +1778,8 @@ struct redisServer {
int cluster_slave_no_failover; /* Prevent slave from starting a failover
if the master is in failure state. */
char *cluster_announce_ip; /* IP address to announce on cluster bus. */
char *cluster_announce_hostname; /* IP address to announce on cluster bus. */
int cluster_preferred_endpoint_type; /* Use the announced hostname when available. */
int cluster_announce_port; /* base port to announce on cluster bus. */
int cluster_announce_tls_port; /* TLS port to announce on cluster bus. */
int cluster_announce_bus_port; /* bus port to announce on cluster bus. */
......@@ -1782,6 +1791,8 @@ struct redisServer {
is down? */
int cluster_config_file_lock_fd; /* cluster config fd, will be flock */
unsigned long long cluster_link_sendbuf_limit_bytes; /* Memory usage limit on individual link send buffers*/
int cluster_drop_packet_filter; /* Debug config that allows tactically
* dropping packets of a specific type */
/* Scripting */
client *script_caller; /* The client running script right now, or NULL */
mstime_t script_time_limit; /* Script timeout in milliseconds */
......@@ -3334,4 +3345,7 @@ int isTlsConfigured(void);
int iAmMaster(void);
#define STRINGIFY_(x) #x
#define STRINGIFY(x) STRINGIFY_(x)
#endif
......@@ -142,7 +142,8 @@ proc cluster_allocate_with_continuous_slots {n} {
}
}
# Create a cluster composed of the specified number of masters and slaves with continuous slots.
# Create a cluster composed of the specified number of masters and slaves,
# but with a continuous slot range.
proc cluster_create_with_continuous_slots {masters slaves} {
cluster_allocate_with_continuous_slots $masters
if {$slaves} {
......
source "../tests/includes/init-tests.tcl"
# Check if cluster's view of hostnames is consistent
proc are_hostnames_propagated {match_string} {
for {set j 0} {$j < $::cluster_master_nodes + $::cluster_replica_nodes} {incr j} {
set cfg [R $j cluster slots]
foreach node $cfg {
for {set i 2} {$i < [llength $node]} {incr i} {
if {! [string match $match_string [lindex [lindex [lindex $node $i] 3] 1]] } {
return 0
}
}
}
}
return 1
}
# Isolate a node from the cluster and give it a new nodeid
proc isolate_node {id} {
set node_id [R $id CLUSTER MYID]
R 6 CLUSTER RESET HARD
for {set j 0} {$j < 20} {incr j} {
if { $j eq $id } {
continue
}
R $j CLUSTER FORGET $node_id
}
}
proc get_slot_field {slot_output shard_id node_id attrib_id} {
return [lindex [lindex [lindex $slot_output $shard_id] $node_id] $attrib_id]
}
test "Create a 6 nodes cluster" {
cluster_create_with_continuous_slots 3 3
}
test "Cluster should start ok" {
assert_cluster_state ok
wait_for_cluster_propagation
}
test "Set cluster hostnames and verify they are propagated" {
for {set j 0} {$j < $::cluster_master_nodes + $::cluster_replica_nodes} {incr j} {
R $j config set cluster-announce-hostname "host-$j.com"
}
wait_for_condition 50 100 {
[are_hostnames_propagated "host-*.com"] eq 1
} else {
fail "cluster hostnames were not propagated"
}
# Now that everything is propagated, assert everyone agrees
wait_for_cluster_propagation
}
test "Update hostnames and make sure they are all eventually propagated" {
for {set j 0} {$j < $::cluster_master_nodes + $::cluster_replica_nodes} {incr j} {
R $j config set cluster-announce-hostname "host-updated-$j.com"
}
wait_for_condition 50 100 {
[are_hostnames_propagated "host-updated-*.com"] eq 1
} else {
fail "cluster hostnames were not propagated"
}
# Now that everything is propagated, assert everyone agrees
wait_for_cluster_propagation
}
test "Remove hostnames and make sure they are all eventually propagated" {
for {set j 0} {$j < $::cluster_master_nodes + $::cluster_replica_nodes} {incr j} {
R $j config set cluster-announce-hostname ""
}
wait_for_condition 50 100 {
[are_hostnames_propagated ""] eq 1
} else {
fail "cluster hostnames were not propagated"
}
# Now that everything is propagated, assert everyone agrees
wait_for_cluster_propagation
}
test "Verify cluster-preferred-endpoint-type behavior for redirects and info" {
R 0 config set cluster-announce-hostname "me.com"
R 1 config set cluster-announce-hostname ""
R 2 config set cluster-announce-hostname "them.com"
wait_for_cluster_propagation
# Verify default behavior
set slot_result [R 0 cluster slots]
assert_equal "" [lindex [get_slot_field $slot_result 0 2 0] 1]
assert_equal "" [lindex [get_slot_field $slot_result 2 2 0] 1]
assert_equal "hostname" [lindex [get_slot_field $slot_result 0 2 3] 0]
assert_equal "me.com" [lindex [get_slot_field $slot_result 0 2 3] 1]
assert_equal "hostname" [lindex [get_slot_field $slot_result 2 2 3] 0]
assert_equal "them.com" [lindex [get_slot_field $slot_result 2 2 3] 1]
# Redirect will use the IP address
catch {R 0 set foo foo} redir_err
assert_match "MOVED * 127.0.0.1:*" $redir_err
# Verify prefer hostname behavior
R 0 config set cluster-preferred-endpoint-type hostname
set slot_result [R 0 cluster slots]
assert_equal "me.com" [get_slot_field $slot_result 0 2 0]
assert_equal "them.com" [get_slot_field $slot_result 2 2 0]
# Redirect should use hostname
catch {R 0 set foo foo} redir_err
assert_match "MOVED * them.com:*" $redir_err
# Redirect to an unknown hostname returns ?
catch {R 0 set barfoo bar} redir_err
assert_match "MOVED * ?:*" $redir_err
# Verify unknown hostname behavior
R 0 config set cluster-preferred-endpoint-type unknown-endpoint
# Verify default behavior
set slot_result [R 0 cluster slots]
assert_equal "ip" [lindex [get_slot_field $slot_result 0 2 3] 0]
assert_equal "127.0.0.1" [lindex [get_slot_field $slot_result 0 2 3] 1]
assert_equal "ip" [lindex [get_slot_field $slot_result 2 2 3] 0]
assert_equal "127.0.0.1" [lindex [get_slot_field $slot_result 2 2 3] 1]
assert_equal "ip" [lindex [get_slot_field $slot_result 1 2 3] 0]
assert_equal "127.0.0.1" [lindex [get_slot_field $slot_result 1 2 3] 1]
# Not required by the protocol, but IP comes before hostname
assert_equal "hostname" [lindex [get_slot_field $slot_result 0 2 3] 2]
assert_equal "me.com" [lindex [get_slot_field $slot_result 0 2 3] 3]
assert_equal "hostname" [lindex [get_slot_field $slot_result 2 2 3] 2]
assert_equal "them.com" [lindex [get_slot_field $slot_result 2 2 3] 3]
# This node doesn't have a hostname
assert_equal 2 [llength [get_slot_field $slot_result 1 2 3]]
# Redirect should use empty string
catch {R 0 set foo foo} redir_err
assert_match "MOVED * :*" $redir_err
R 0 config set cluster-preferred-endpoint-type ip
}
test "Verify the nodes configured with prefer hostname only show hostname for new nodes" {
# Have everyone forget node 6 and isolate it from the cluster.
isolate_node 6
# Set hostnames for the primaries, now that the node is isolated
R 0 config set cluster-announce-hostname "shard-1.com"
R 1 config set cluster-announce-hostname "shard-2.com"
R 2 config set cluster-announce-hostname "shard-3.com"
# Prevent Node 0 and Node 6 from properly meeting,
# they'll hang in the handshake phase. This allows us to
# test the case where we "know" about it but haven't
# successfully retrieved information about it yet.
R 0 DEBUG DROP-CLUSTER-PACKET-FILTER 0
R 6 DEBUG DROP-CLUSTER-PACKET-FILTER 0
# Have a replica meet the isolated node
R 3 cluster meet 127.0.0.1 [get_instance_attrib redis 6 port]
# Now, we wait until the two nodes that aren't filtering packets
# to accept our isolated nodes connections. At this point they will
# start showing up in cluster slots.
wait_for_condition 50 100 {
[llength [R 6 CLUSTER SLOTS]] eq 2
} else {
fail "Node did not learn about the 2 shards it can talk to"
}
set slot_result [R 6 CLUSTER SLOTS]
assert_equal [lindex [get_slot_field $slot_result 0 2 3] 1] "shard-2.com"
assert_equal [lindex [get_slot_field $slot_result 1 2 3] 1] "shard-3.com"
# Also make sure we know about the isolated primary, we
# just can't reach it.
set primary_id [R 0 CLUSTER MYID]
assert_match "*$primary_id*" [R 6 CLUSTER NODES]
# Stop dropping cluster packets, and make sure everything
# stabilizes
R 0 DEBUG DROP-CLUSTER-PACKET-FILTER -1
R 6 DEBUG DROP-CLUSTER-PACKET-FILTER -1
wait_for_condition 50 100 {
[llength [R 6 CLUSTER SLOTS]] eq 3
} else {
fail "Node did not learn about the 2 shards it can talk to"
}
set slot_result [R 6 CLUSTER SLOTS]
assert_equal [lindex [get_slot_field $slot_result 0 2 3] 1] "shard-1.com"
assert_equal [lindex [get_slot_field $slot_result 1 2 3] 1] "shard-2.com"
assert_equal [lindex [get_slot_field $slot_result 2 2 3] 1] "shard-3.com"
}
test "Test restart will keep hostname information" {
# Set a new hostname, reboot and make sure it sticks
R 0 config set cluster-announce-hostname "restart-1.com"
restart_instance redis 0
set slot_result [R 0 CLUSTER SLOTS]
assert_equal [lindex [get_slot_field $slot_result 0 2 3] 1] "restart-1.com"
# As a sanity check, make sure everyone eventually agrees
wait_for_cluster_propagation
}
test "Test hostname validation" {
catch {R 0 config set cluster-announce-hostname [string repeat x 256]} err
assert_match "*Hostnames must be less than 256 characters*" $err
catch {R 0 config set cluster-announce-hostname "?.com"} err
assert_match "*Hostnames may only contain alphanumeric characters, hyphens or dots*" $err
# Note this isn't a valid hostname, but it passes our internal validation
R 0 config set cluster-announce-hostname "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-."
}
\ No newline at end of file
......@@ -42,6 +42,8 @@ test "Cluster nodes hard reset" {
R $id config set loading-process-events-interval-bytes 2097152
R $id config set key-load-delay 0
R $id config set repl-diskless-load disabled
R $id config set cluster-announce-hostname ""
R $id DEBUG DROP-CLUSTER-PACKET-FILTER -1
R $id config rewrite
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment