Skip to content

Commit 5c4f93c

Browse files
miguelaraujomiguelaraujo
committed
BUG#26001653: CLUSTER.DISSOLVE() FAILS IF ONE OR MORE INSTANCES HAVE THE
(MISSING) STATUS The Cluster.dissolve() function was trying to stop group-replication on all the instances registered in the Metadata which leads to connection errors if any of those instances have the "(MISSING)" status. This patch fixes the issue by making sure that group-replication is stopped only on the instances which have the "ONLINE" status. Added new tests to verify the changes above and also to fully test the dissolve() operation in single-primary and multi-primary scenarios.
1 parent e399b3e commit 5c4f93c

File tree

4 files changed

+243
-11
lines changed

4 files changed

+243
-11
lines changed

‎modules/adminapi/mod_dba_cluster.cc

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -977,6 +977,10 @@ shcore::Value Cluster::dissolve(const shcore::Argument_list &args) {
977977
// Point the metadata session to the cluster session
978978
_metadata_storage->set_session(_session);
979979

980+
// We need to check if the group has quorum and if not we must abort the
981+
// operation otherwise GR blocks the writes to preserve the consistency
982+
// of the group and we end up with a hang.
983+
// This check is done at check_preconditions()
980984
check_preconditions("dissolve");
981985

982986
try {
@@ -999,12 +1003,6 @@ shcore::Value Cluster::dissolve(const shcore::Argument_list &args) {
9991003
MetadataStorage::Transaction tx(_metadata_storage);
10001004
std::string cluster_name = get_name();
10011005

1002-
// We need to check if the group has quorum and if not we must abort the operation
1003-
// otherwise we GR blocks the writes to preserve the consistency of the group and we end up
1004-
// with a hang.
1005-
1006-
mysqlsh::mysql::ClassicSession *classic = dynamic_cast<mysqlsh::mysql::ClassicSession*>(_session.get());
1007-
10081006
// check if the Cluster is empty
10091007
if (_metadata_storage->is_cluster_empty(get_id())) {
10101008
_metadata_storage->drop_cluster(cluster_name);
@@ -1013,23 +1011,30 @@ shcore::Value Cluster::dissolve(const shcore::Argument_list &args) {
10131011
_dissolved = true;
10141012
} else {
10151013
if (force) {
1016-
// Gets the instances on the only available replica set
1017-
auto instances = _metadata_storage->get_replicaset_instances(_default_replica_set->get_id());
1014+
// We must stop GR on the online instances only, otherwise we'll
1015+
// get connection failures to the (MISSING) instances
1016+
// BUG#26001653.
1017+
// Get the online instances on the only available replica set
1018+
auto online_instances =
1019+
_metadata_storage->get_replicaset_online_instances(
1020+
_default_replica_set->get_id());
10181021

10191022
_metadata_storage->drop_replicaset(_default_replica_set->get_id());
10201023

1021-
// TODO: we only have the Default ReplicaSet, but will have more in the future
1024+
// TODO(miguel): we only have the Default ReplicaSet
1025+
// but will have more in the future
10221026
_metadata_storage->drop_cluster(cluster_name);
10231027

10241028
tx.commit();
10251029

10261030
// once the data changes are done, we proceed doing the remove from GR
1027-
_default_replica_set->remove_instances_from_gr(instances);
1031+
_default_replica_set->remove_instances_from_gr(online_instances);
10281032

10291033
// Set the flag, marking this cluster instance as invalid.
10301034
_dissolved = true;
10311035
} else {
1032-
throw Exception::logic_error("Cannot drop cluster: The cluster is not empty.");
1036+
throw Exception::logic_error(
1037+
"Cannot drop cluster: The cluster is not empty.");
10331038
}
10341039
}
10351040
}

‎unittest/scripts/js_devapi/scripts/dba_cluster_dissolve.js

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
// Assumptions: smart deployment rountines available
2+
//@ Initialization
3+
var deployed_here = reset_or_deploy_sandboxes();
4+
5+
//@ Create single-primary cluster
6+
shell.connect({scheme: 'mysql', host: localhost, port: __mysql_sandbox_port1, user: 'root', password: 'root'});
7+
var singleSession = session;
8+
9+
if (__have_ssl)
10+
var single = dba.createCluster('single', {memberSslMode:'REQUIRED'});
11+
else
12+
var single = dba.createCluster('single');
13+
14+
//@ Success adding instance 2
15+
add_instance_to_cluster(single, __mysql_sandbox_port2);
16+
17+
// Waiting for the added instance to become online
18+
wait_slave_state(single, uri2, "ONLINE");
19+
20+
// Wait for the second added instance to fetch all the replication data
21+
wait_sandbox_in_metadata(__mysql_sandbox_port2);
22+
23+
//@ Success adding instance 3
24+
add_instance_to_cluster(single, __mysql_sandbox_port3);
25+
26+
// Waiting for the added instance to become online
27+
wait_slave_state(single, uri3, "ONLINE");
28+
29+
// Wait for the third added instance to fetch all the replication data
30+
wait_sandbox_in_metadata(__mysql_sandbox_port3);
31+
32+
//@ Cluster.dissolve no force error
33+
single.dissolve();
34+
35+
//@ Success dissolving single-primary cluster
36+
single.dissolve({force: true});
37+
38+
//@ Cluster.dissolve already dissolved
39+
single.dissolve();
40+
41+
shell.connect({scheme: 'mysql', host: localhost, port: __mysql_sandbox_port1, user: 'root', password: 'root'});
42+
var multiSession = session;
43+
44+
//@ Create multi-primary cluster
45+
if (__have_ssl)
46+
var multi = dba.createCluster('multi', {multiMaster: true, memberSslMode:'REQUIRED', clearReadOnly: true, force: true});
47+
else
48+
var multi = dba.createCluster('multi', {multiMaster: true, clearReadOnly: true, force: true});
49+
50+
//@ Success adding instance 2 mp
51+
add_instance_to_cluster(multi, __mysql_sandbox_port2);
52+
53+
// Waiting for the added instance to become online
54+
wait_slave_state(multi, uri2, "ONLINE");
55+
56+
// Wait for the second added instance to fetch all the replication data
57+
wait_sandbox_in_metadata(__mysql_sandbox_port2);
58+
59+
//@ Success adding instance 3 mp
60+
add_instance_to_cluster(multi, __mysql_sandbox_port3);
61+
62+
// Waiting for the added instance to become online
63+
wait_slave_state(multi, uri3, "ONLINE");
64+
65+
// Wait for the third added instance to fetch all the replication data
66+
wait_sandbox_in_metadata(__mysql_sandbox_port3);
67+
68+
//@ Success dissolving multi-primary cluster
69+
multi.dissolve({force: true});
70+
71+
//@ Create single-primary cluster 2
72+
shell.connect({scheme: 'mysql', host: localhost, port: __mysql_sandbox_port1, user: 'root', password: 'root'});
73+
var singleSession2 = session;
74+
75+
if (__have_ssl)
76+
var single2 = dba.createCluster('single2', {memberSslMode:'REQUIRED', clearReadOnly: true});
77+
else
78+
var single2 = dba.createCluster('single2', {clearReadOnly: true});
79+
80+
//@ Success adding instance 2 2
81+
add_instance_to_cluster(single2, __mysql_sandbox_port2);
82+
83+
// Waiting for the added instance to become online
84+
wait_slave_state(single2, uri2, "ONLINE");
85+
86+
// Wait for the second added instance to fetch all the replication data
87+
wait_sandbox_in_metadata(__mysql_sandbox_port2);
88+
89+
//@ Success adding instance 3 2
90+
add_instance_to_cluster(single2, __mysql_sandbox_port3);
91+
92+
// Waiting for the added instance to become online
93+
wait_slave_state(single2, uri3, "ONLINE");
94+
95+
// Wait for the third added instance to fetch all the replication data
96+
wait_sandbox_in_metadata(__mysql_sandbox_port3);
97+
98+
// stop instance 3
99+
// Use stop sandbox instance to make sure the instance is gone before restarting it
100+
if (__sandbox_dir)
101+
dba.stopSandboxInstance(__mysql_sandbox_port3, {sandboxDir:__sandbox_dir, password: 'root'});
102+
else
103+
dba.stopSandboxInstance(__mysql_sandbox_port3, {password: 'root'});
104+
105+
wait_slave_state(single2, uri3, ["(MISSING)"]);
106+
107+
// Regression test for BUG#26001653
108+
//@ Success dissolving cluster 2
109+
single2.dissolve({force: true});
110+
111+
// start instance 3
112+
try_restart_sandbox(__mysql_sandbox_port3);
113+
114+
//@ Create multi-primary cluster 2
115+
shell.connect({scheme: 'mysql', host: localhost, port: __mysql_sandbox_port1, user: 'root', password: 'root'});
116+
var multiSession2 = session;
117+
118+
if (__have_ssl)
119+
var multi2 = dba.createCluster('multi2', {memberSslMode:'REQUIRED', clearReadOnly: true, multiMaster: true, force: true});
120+
else
121+
var multi2 = dba.createCluster('multi2', {clearReadOnly: true, multiMaster: true, force: true});
122+
123+
//@ Success adding instance 2 mp 2
124+
add_instance_to_cluster(multi2, __mysql_sandbox_port2);
125+
126+
// Waiting for the added instance to become online
127+
wait_slave_state(multi2, uri2, "ONLINE");
128+
129+
// Wait for the second added instance to fetch all the replication data
130+
wait_sandbox_in_metadata(__mysql_sandbox_port2);
131+
132+
//@ Success adding instance 3 mp 2
133+
add_instance_to_cluster(multi2, __mysql_sandbox_port3);
134+
135+
// Waiting for the added instance to become online
136+
wait_slave_state(multi2, uri3, "ONLINE");
137+
138+
// Wait for the third added instance to fetch all the replication data
139+
wait_sandbox_in_metadata(__mysql_sandbox_port3);
140+
141+
// stop instance 3
142+
// Use stop sandbox instance to make sure the instance is gone before restarting it
143+
if (__sandbox_dir)
144+
dba.stopSandboxInstance(__mysql_sandbox_port3, {sandboxDir:__sandbox_dir, password: 'root'});
145+
else
146+
dba.stopSandboxInstance(__mysql_sandbox_port3, {password: 'root'});
147+
148+
wait_slave_state(multi2, uri3, ["(MISSING)"]);
149+
150+
// Regression test for BUG#26001653
151+
//@ Success dissolving multi-primary cluster 2
152+
multi2.dissolve({force: true});
153+
154+
//@ Finalization
155+
// Will close opened sessions and delete the sandboxes ONLY if this test was executed standalone
156+
singleSession.close();
157+
multiSession.close();
158+
singleSession2.close();
159+
multiSession2.close();
160+
if (deployed_here)
161+
cleanup_sandboxes(true);

‎unittest/scripts/js_devapi/validation/dba_cluster_dissolve.js

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
//@ Initialization
2+
||
3+
4+
//@ Create single-primary cluster
5+
||
6+
7+
//@ Success adding instance 2
8+
||
9+
10+
//@ Success adding instance 3
11+
||
12+
13+
//@ Cluster.dissolve no force error
14+
||Cluster.dissolve: Cannot drop cluster: The cluster is not empty.
15+
16+
//@ Success dissolving single-primary cluster
17+
||
18+
19+
//@ Cluster.dissolve already dissolved
20+
||Cluster.dissolve: Can't call function 'dissolve' on a dissolved cluster (RuntimeError)
21+
22+
//@ Create multi-primary cluster
23+
||
24+
25+
//@ Success adding instance 2 mp
26+
||
27+
28+
//@ Success adding instance 3 mp
29+
||
30+
31+
//@ Success dissolving multi-primary cluster
32+
||
33+
34+
//@ Create single-primary cluster 2
35+
||
36+
37+
//@ Success adding instance 2 2
38+
||
39+
40+
//@ Success adding instance 3 2
41+
||
42+
43+
//@ Success dissolving cluster 2
44+
||
45+
46+
//@ Create multi-primary cluster 2
47+
||
48+
49+
//@ Success adding instance 2 mp 2
50+
||
51+
52+
//@ Success adding instance 3 mp 2
53+
||
54+
55+
//@ Success dissolving multi-primary cluster 2
56+
||
57+
58+
//@ Finalization
59+
||

‎unittest/shell_js_dba_t.cc

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -854,6 +854,13 @@ TEST_F(Shell_js_dba_tests, adopt_from_gr_interactive) {
854854
validate_interactive("dba_adopt_from_gr_interactive.js");
855855
}
856856

857+
TEST_F(Shell_js_dba_tests, dba_cluster_dissolve) {
858+
_options->wizards = false;
859+
reset_shell();
860+
861+
validate_interactive("dba_cluster_dissolve.js");
862+
}
863+
857864
TEST_F(Shell_js_dba_tests, no_interactive_delete_instances) {
858865
_options->wizards = false;
859866
reset_shell();

0 commit comments

Comments
 (0)