Unverified Commit 203b12e4 authored by Ping Xie's avatar Ping Xie Committed by GitHub
Browse files

Introduce Shard IDs to logically group nodes in cluster mode (#10536)

Introduce Shard IDs to logically group nodes in cluster mode.
1. Added a new "shard_id" field to "cluster nodes" output and nodes.conf after "hostname"
2. Added a new PING extension to propagate "shard_id"
3. Handled upgrade from pre-7.2 releases automatically
4. Refactored PING extension assembling/parsing logic

Behavior of Shard IDs:

Replicas will always follow the shards of their reported primaries. If a primary updates its shard ID, the replica will follow. (This need not follow for cluster v2) This is not an expected use case.
parent 2168ccc6
This diff is collapsed.
...@@ -117,6 +117,7 @@ typedef struct clusterNodeFailReport { ...@@ -117,6 +117,7 @@ typedef struct clusterNodeFailReport {
typedef struct clusterNode { typedef struct clusterNode {
mstime_t ctime; /* Node object creation time. */ mstime_t ctime; /* Node object creation time. */
char name[CLUSTER_NAMELEN]; /* Node name, hex string, sha1-size */ char name[CLUSTER_NAMELEN]; /* Node name, hex string, sha1-size */
char shard_id[CLUSTER_NAMELEN]; /* shard id, hex string, sha1-size */
int flags; /* CLUSTER_NODE_... */ int flags; /* CLUSTER_NODE_... */
uint64_t configEpoch; /* Last configEpoch observed for this node */ uint64_t configEpoch; /* Last configEpoch observed for this node */
unsigned char slots[CLUSTER_SLOTS/8]; /* slots handled by this node */ unsigned char slots[CLUSTER_SLOTS/8]; /* slots handled by this node */
...@@ -175,6 +176,7 @@ typedef struct clusterState { ...@@ -175,6 +176,7 @@ typedef struct clusterState {
int state; /* CLUSTER_OK, CLUSTER_FAIL, ... */ int state; /* CLUSTER_OK, CLUSTER_FAIL, ... */
int size; /* Num of master nodes with at least one slot */ int size; /* Num of master nodes with at least one slot */
dict *nodes; /* Hash table of name -> clusterNode structures */ dict *nodes; /* Hash table of name -> clusterNode structures */
dict *shards; /* Hash table of shard_id -> list (of nodes) structures */
dict *nodes_black_list; /* Nodes we don't re-add for a few seconds. */ dict *nodes_black_list; /* Nodes we don't re-add for a few seconds. */
clusterNode *migrating_slots_to[CLUSTER_SLOTS]; clusterNode *migrating_slots_to[CLUSTER_SLOTS];
clusterNode *importing_slots_from[CLUSTER_SLOTS]; clusterNode *importing_slots_from[CLUSTER_SLOTS];
...@@ -256,6 +258,7 @@ typedef struct { ...@@ -256,6 +258,7 @@ typedef struct {
typedef enum { typedef enum {
CLUSTERMSG_EXT_TYPE_HOSTNAME, CLUSTERMSG_EXT_TYPE_HOSTNAME,
CLUSTERMSG_EXT_TYPE_FORGOTTEN_NODE, CLUSTERMSG_EXT_TYPE_FORGOTTEN_NODE,
CLUSTERMSG_EXT_TYPE_SHARDID,
} clusterMsgPingtypes; } clusterMsgPingtypes;
/* Helper function for making sure extensions are eight byte aligned. */ /* Helper function for making sure extensions are eight byte aligned. */
...@@ -272,6 +275,10 @@ typedef struct { ...@@ -272,6 +275,10 @@ typedef struct {
static_assert(sizeof(clusterMsgPingExtForgottenNode) % 8 == 0, ""); static_assert(sizeof(clusterMsgPingExtForgottenNode) % 8 == 0, "");
typedef struct {
char shard_id[CLUSTER_NAMELEN]; /* The shard_id, 40 bytes fixed. */
} clusterMsgPingExtShardId;
typedef struct { typedef struct {
uint32_t length; /* Total length of this extension message (including this header) */ uint32_t length; /* Total length of this extension message (including this header) */
uint16_t type; /* Type of this extension message (see clusterMsgPingExtTypes) */ uint16_t type; /* Type of this extension message (see clusterMsgPingExtTypes) */
...@@ -279,6 +286,7 @@ typedef struct { ...@@ -279,6 +286,7 @@ typedef struct {
union { union {
clusterMsgPingExtHostname hostname; clusterMsgPingExtHostname hostname;
clusterMsgPingExtForgottenNode forgotten_node; clusterMsgPingExtForgottenNode forgotten_node;
clusterMsgPingExtShardId shard_id;
} ext[]; /* Actual extension information, formatted so that the data is 8 } ext[]; /* Actual extension information, formatted so that the data is 8
* byte aligned, regardless of its content. */ * byte aligned, regardless of its content. */
} clusterMsgPingExt; } clusterMsgPingExt;
......
...@@ -476,6 +476,17 @@ struct redisCommandArg CLUSTER_MEET_Args[] = { ...@@ -476,6 +476,17 @@ struct redisCommandArg CLUSTER_MEET_Args[] = {
/* CLUSTER MYID tips */ /* CLUSTER MYID tips */
#define CLUSTER_MYID_tips NULL #define CLUSTER_MYID_tips NULL
/********** CLUSTER MYSHARDID ********************/
/* CLUSTER MYSHARDID history */
#define CLUSTER_MYSHARDID_History NULL
/* CLUSTER MYSHARDID tips */
const char *CLUSTER_MYSHARDID_tips[] = {
"nondeterministic_output",
NULL
};
/********** CLUSTER NODES ********************/ /********** CLUSTER NODES ********************/
/* CLUSTER NODES history */ /* CLUSTER NODES history */
...@@ -647,6 +658,7 @@ struct redisCommand CLUSTER_Subcommands[] = { ...@@ -647,6 +658,7 @@ struct redisCommand CLUSTER_Subcommands[] = {
{"links","Returns a list of all TCP links to and from peer nodes in cluster","O(N) where N is the total number of Cluster nodes","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_CLUSTER,CLUSTER_LINKS_History,CLUSTER_LINKS_tips,clusterCommand,2,CMD_STALE,0}, {"links","Returns a list of all TCP links to and from peer nodes in cluster","O(N) where N is the total number of Cluster nodes","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_CLUSTER,CLUSTER_LINKS_History,CLUSTER_LINKS_tips,clusterCommand,2,CMD_STALE,0},
{"meet","Force a node cluster to handshake with another node","O(1)","3.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_CLUSTER,CLUSTER_MEET_History,CLUSTER_MEET_tips,clusterCommand,-4,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_STALE,0,.args=CLUSTER_MEET_Args}, {"meet","Force a node cluster to handshake with another node","O(1)","3.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_CLUSTER,CLUSTER_MEET_History,CLUSTER_MEET_tips,clusterCommand,-4,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_STALE,0,.args=CLUSTER_MEET_Args},
{"myid","Return the node id","O(1)","3.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_CLUSTER,CLUSTER_MYID_History,CLUSTER_MYID_tips,clusterCommand,2,CMD_STALE,0}, {"myid","Return the node id","O(1)","3.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_CLUSTER,CLUSTER_MYID_History,CLUSTER_MYID_tips,clusterCommand,2,CMD_STALE,0},
{"myshardid","Return the node shard id","O(1)","7.2.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_CLUSTER,CLUSTER_MYSHARDID_History,CLUSTER_MYSHARDID_tips,clusterCommand,2,CMD_STALE,0},
{"nodes","Get Cluster config for the node","O(N) where N is the total number of Cluster nodes","3.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_CLUSTER,CLUSTER_NODES_History,CLUSTER_NODES_tips,clusterCommand,2,CMD_STALE,0}, {"nodes","Get Cluster config for the node","O(N) where N is the total number of Cluster nodes","3.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_CLUSTER,CLUSTER_NODES_History,CLUSTER_NODES_tips,clusterCommand,2,CMD_STALE,0},
{"replicas","List replica nodes of the specified master node","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_CLUSTER,CLUSTER_REPLICAS_History,CLUSTER_REPLICAS_tips,clusterCommand,3,CMD_ADMIN|CMD_STALE,0,.args=CLUSTER_REPLICAS_Args}, {"replicas","List replica nodes of the specified master node","O(1)","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_CLUSTER,CLUSTER_REPLICAS_History,CLUSTER_REPLICAS_tips,clusterCommand,3,CMD_ADMIN|CMD_STALE,0,.args=CLUSTER_REPLICAS_Args},
{"replicate","Reconfigure a node as a replica of the specified master node","O(1)","3.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_CLUSTER,CLUSTER_REPLICATE_History,CLUSTER_REPLICATE_tips,clusterCommand,3,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_STALE,0,.args=CLUSTER_REPLICATE_Args}, {"replicate","Reconfigure a node as a replica of the specified master node","O(1)","3.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_CLUSTER,CLUSTER_REPLICATE_History,CLUSTER_REPLICATE_tips,clusterCommand,3,CMD_NO_ASYNC_LOADING|CMD_ADMIN|CMD_STALE,0,.args=CLUSTER_REPLICATE_Args},
......
{
"MYSHARDID": {
"summary": "Return the node shard id",
"complexity": "O(1)",
"group": "cluster",
"since": "7.2.0",
"arity": 2,
"container": "CLUSTER",
"function": "clusterCommand",
"history": [],
"command_flags": [
"STALE"
],
"command_tips": [
"NONDETERMINISTIC_OUTPUT"
]
}
}
...@@ -91,6 +91,9 @@ typedef struct redisObject robj; ...@@ -91,6 +91,9 @@ typedef struct redisObject robj;
#include "endianconv.h" #include "endianconv.h"
#include "crc64.h" #include "crc64.h"
/* helpers */
#define numElements(x) (sizeof(x)/sizeof((x)[0]))
/* min/max */ /* min/max */
#undef min #undef min
#undef max #undef max
...@@ -3301,6 +3304,7 @@ uint64_t dictSdsCaseHash(const void *key); ...@@ -3301,6 +3304,7 @@ uint64_t dictSdsCaseHash(const void *key);
int dictSdsKeyCompare(dict *d, const void *key1, const void *key2); int dictSdsKeyCompare(dict *d, const void *key1, const void *key2);
int dictSdsKeyCaseCompare(dict *d, const void *key1, const void *key2); int dictSdsKeyCaseCompare(dict *d, const void *key1, const void *key2);
void dictSdsDestructor(dict *d, void *val); void dictSdsDestructor(dict *d, void *val);
void dictListDestructor(dict *d, void *val);
void *dictSdsDup(dict *d, const void *key); void *dictSdsDup(dict *d, const void *key);
/* Git SHA1 */ /* Git SHA1 */
......
...@@ -18,7 +18,8 @@ proc get_cluster_nodes id { ...@@ -18,7 +18,8 @@ proc get_cluster_nodes id {
set args [split $l] set args [split $l]
set node [dict create \ set node [dict create \
id [lindex $args 0] \ id [lindex $args 0] \
addr [lindex $args 1] \ addr [lindex [split [lindex $args 1] ,] 0] \
shard-id [lindex [split [lindex [split [lindex $args 1] ,] 2] = ] 1]\
flags [split [lindex $args 2] ,] \ flags [split [lindex $args 2] ,] \
slaveof [lindex $args 3] \ slaveof [lindex $args 3] \
ping_sent [lindex $args 4] \ ping_sent [lindex $args 4] \
......
...@@ -39,6 +39,17 @@ proc get_node_info_from_shard {id reference {type node}} { ...@@ -39,6 +39,17 @@ proc get_node_info_from_shard {id reference {type node}} {
return {} return {}
} }
proc cluster_ensure_master {id} {
if { [regexp "master" [R $id role]] == 0 } {
assert_equal {OK} [R $id CLUSTER FAILOVER]
wait_for_condition 50 100 {
[regexp "master" [R $id role]] == 1
} else {
fail "instance $id is not master"
}
}
}
test "Create a 8 nodes cluster with 4 shards" { test "Create a 8 nodes cluster with 4 shards" {
cluster_create_with_split_slots 4 4 cluster_create_with_split_slots 4 4
} }
...@@ -87,7 +98,7 @@ test "Verify information about the shards" { ...@@ -87,7 +98,7 @@ test "Verify information about the shards" {
# Replica could be in online or loading # Replica could be in online or loading
} }
} }
} }
} }
test "Verify no slot shard" { test "Verify no slot shard" {
...@@ -180,7 +191,7 @@ test "Test the replica reports a loading state while it's loading" { ...@@ -180,7 +191,7 @@ test "Test the replica reports a loading state while it's loading" {
fail "Replica never transitioned to online" fail "Replica never transitioned to online"
} }
# Final sanity, the replica agrees it is online. # Final sanity, the replica agrees it is online.
assert_equal "online" [dict get [get_node_info_from_shard $replica_cluster_id $replica_id "node"] health] assert_equal "online" [dict get [get_node_info_from_shard $replica_cluster_id $replica_id "node"] health]
} }
...@@ -200,3 +211,93 @@ test "Regression test for a crash when calling SHARDS during handshake" { ...@@ -200,3 +211,93 @@ test "Regression test for a crash when calling SHARDS during handshake" {
test "Cluster is up" { test "Cluster is up" {
assert_cluster_state ok assert_cluster_state ok
} }
test "Shard ids are unique" {
set shard_ids {}
for {set i 0} {$i < 4} {incr i} {
set shard_id [R $i cluster myshardid]
assert_equal [dict exists $shard_ids $shard_id] 0
dict set shard_ids $shard_id 1
}
}
test "CLUSTER MYSHARDID reports same id for both primary and replica" {
for {set i 0} {$i < 4} {incr i} {
assert_equal [R $i cluster myshardid] [R [expr $i+4] cluster myshardid]
assert_equal [string length [R $i cluster myshardid]] 40
}
}
test "CLUSTER NODES reports correct shard id" {
for {set i 0} {$i < 8} {incr i} {
set nodes [get_cluster_nodes $i]
set node_id_to_shardid_mapping []
foreach n $nodes {
set node_shard_id [dict get $n shard-id]
set node_id [dict get $n id]
assert_equal [string length $node_shard_id] 40
if {[dict exists $node_id_to_shardid_mapping $node_id]} {
assert_equal [dict get $node_id_to_shardid_mapping $node_id] $node_shard_id
} else {
dict set node_id_to_shardid_mapping $node_id $node_shard_id
}
if {[lindex [dict get $n flags] 0] eq "myself"} {
assert_equal [R $i cluster myshardid] [dict get $n shard-id]
}
}
}
}
test "New replica receives primary's shard id" {
#find a primary
set id 0
for {} {$id < 8} {incr id} {
if {[regexp "master" [R $id role]]} {
break
}
}
assert_not_equal [R 8 cluster myshardid] [R $id cluster myshardid]
assert_equal {OK} [R 8 cluster replicate [R $id cluster myid]]
assert_equal [R 8 cluster myshardid] [R $id cluster myshardid]
}
test "CLUSTER MYSHARDID reports same shard id after shard restart" {
set node_ids {}
for {set i 0} {$i < 8} {incr i 4} {
dict set node_ids $i [R $i cluster myshardid]
kill_instance redis $i
wait_for_condition 50 100 {
[instance_is_killed redis $i]
} else {
fail "instance $i is not killed"
}
}
for {set i 0} {$i < 8} {incr i 4} {
restart_instance redis $i
}
assert_cluster_state ok
for {set i 0} {$i < 8} {incr i 4} {
assert_equal [dict get $node_ids $i] [R $i cluster myshardid]
}
}
test "CLUSTER MYSHARDID reports same shard id after cluster restart" {
set node_ids {}
for {set i 0} {$i < 8} {incr i} {
dict set node_ids $i [R $i cluster myshardid]
}
for {set i 0} {$i < 8} {incr i} {
kill_instance redis $i
wait_for_condition 50 100 {
[instance_is_killed redis $i]
} else {
fail "instance $i is not killed"
}
}
for {set i 0} {$i < 8} {incr i} {
restart_instance redis $i
}
assert_cluster_state ok
for {set i 0} {$i < 8} {incr i} {
assert_equal [dict get $node_ids $i] [R $i cluster myshardid]
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment