AdminAPI test fixes:

Added check_sandbox_in_metadata() in the available test functions to
wait for an instance which was recently added to a cluster to be
present in its own metedata schema: there may be some replication
delay since the instance is added to the metadata on the primary.

Included the usage of check_sandbox_in_metadata() for the
dba_cluster_add_instance and super_read_only_handling tests.
Those tests were failing due to the assumption that if an instance
has the "ONLINE" state it already has all the transactions applied
and therefore has the most recent version of the metadata schema.
This may not be always true due to the slowness of the system
running the tests.

Added connection attempt in try_restart_sandbox() to verify if the
instance is up-and-running and accepting connection before returning.

Replaced the use of killSandboxInstance() with stopSandboxInstance()
in several tests to make sure the instance is completely dead before
restarting it to make sure the restart does not fail. Made use of
try_restart_sandbox too instead of startSandboxInstance().

Author: miguelaraujo

unittest/scripts/js_devapi/scripts/dba_cluster_add_instance.js

@@ -17,6 +17,9 @@ add_instance_to_cluster(single, __mysql_sandbox_port2);
 // Waiting for the second added instance to become online
 wait_slave_state(single, uri2, "ONLINE");
 
+// Wait for the second added instance to fetch all the replication data
+wait_sandbox_in_metadata(__mysql_sandbox_port2);
+
 // Connect to the future new seed node
 shell.connect({scheme: 'mysql', host: localhost, port: __mysql_sandbox_port2, user: 'root', password: 'root'});
 var singleSession2 = session;
@@ -34,7 +37,7 @@ if (__sandbox_dir)
 else
   dba.killSandboxInstance(__mysql_sandbox_port1);
 
-wait_slave_state(single, uri1, ["UNREACHABLE", "OFFLINE"]);
+wait_slave_state(single, uri1, ["(MISSING)"]);
 
 //@ Restore the quorum
 single.forceQuorumUsingPartitionOf({host: localhost, port: __mysql_sandbox_port2, user: 'root', password:'root'});

unittest/scripts/js_devapi/scripts/dba_cluster_interactive.js

@@ -119,16 +119,14 @@ if (__sandbox_dir)
 else
   dba.killSandboxInstance(__mysql_sandbox_port3);
 
-wait_slave_state(Cluster, 'third_sandbox', ["UNREACHABLE", "OFFLINE"]);
+wait_slave_state(Cluster, 'third_sandbox', ["(MISSING)"]);
 
 //@# Dba: start instance 3
 if (__sandbox_dir)
   dba.startSandboxInstance(__mysql_sandbox_port3, {sandboxDir: __sandbox_dir});
 else
   dba.startSandboxInstance(__mysql_sandbox_port3);
 
-wait_slave_state(Cluster, 'third_sandbox', ["OFFLINE", "(MISSING)"]);
-
 //@: Cluster: rejoinInstance errors
 Cluster.rejoinInstance();
 Cluster.rejoinInstance(1,2,3);

unittest/scripts/js_devapi/scripts/dba_cluster_multimaster_interactive.js

@@ -72,22 +72,17 @@ Cluster.status()
 
 // Rejoin tests
 
-//@# Dba: kill instance 3
+//@# Dba: stop instance 3
+// Use stop sandbox instance to make sure the instance is gone before restarting it
 if (__sandbox_dir)
-  dba.killSandboxInstance(__mysql_sandbox_port3, {sandboxDir:__sandbox_dir});
+  dba.stopSandboxInstance(__mysql_sandbox_port3, {sandboxDir:__sandbox_dir, password: 'root'});
 else
-  dba.killSandboxInstance(__mysql_sandbox_port3);
+  dba.stopSandboxInstance(__mysql_sandbox_port3, {password: 'root'});
 
-// XCOM needs time to kick out the member of the group. The GR team has a patch to fix this
-// But won't be available for the GA release. So we need to wait until the instance is reported
-// as offline
-wait_slave_state(Cluster, uri3, ["OFFLINE", "UNREACHABLE"]);
+wait_slave_state(Cluster, uri3, ["(MISSING)"]);
 
-//@# Dba: start instance 3
-if (__sandbox_dir)
-  dba.startSandboxInstance(__mysql_sandbox_port3, {sandboxDir: __sandbox_dir});
-else
-  dba.startSandboxInstance(__mysql_sandbox_port3);
+// start instance 3
+try_restart_sandbox(__mysql_sandbox_port3);
 
 //@: Cluster: rejoinInstance errors
 Cluster.rejoinInstance();
@@ -112,4 +107,6 @@ Cluster.status();
 Cluster.dissolve({force: true})
 
 // Disable super-read-only (BUG#26422638)
-session.runSql("SET GLOBAL SUPER_READ_ONLY = 0;")
+shell.connect({scheme: 'mysql', host: localhost, port: __mysql_sandbox_port1, user: 'root', password: 'root'});
+session.runSql("SET GLOBAL SUPER_READ_ONLY = 0;");
+session.close();

unittest/scripts/js_devapi/scripts/dba_cluster_multimaster_no_interactive.js

@@ -66,22 +66,17 @@ Cluster.status();
 
 // Rejoin tests
 
-//@# Dba: kill instance 3
+//@# Dba: stop instance 3
+// Use stop sandbox instance to make sure the instance is gone before restarting it
 if (__sandbox_dir)
-  dba.killSandboxInstance(__mysql_sandbox_port3, {sandboxDir:__sandbox_dir});
+  dba.stopSandboxInstance(__mysql_sandbox_port3, {sandboxDir:__sandbox_dir, password: 'root'});
 else
-  dba.killSandboxInstance(__mysql_sandbox_port3);
+  dba.stopSandboxInstance(__mysql_sandbox_port3, {password: 'root'});
 
-// XCOM needs time to kick out the member of the group. The GR team has a patch to fix this
-// But won't be available for the GA release. So we need to wait until the instance is reported
-// as offline
-wait_slave_state(Cluster, uri3, ["OFFLINE", "UNREACHABLE"]);
+wait_slave_state(Cluster, uri3, ["(MISSING)"]);
 
-//@# Dba: start instance 3
-if (__sandbox_dir)
-  dba.startSandboxInstance(__mysql_sandbox_port3, {sandboxDir: __sandbox_dir});
-else
-  dba.startSandboxInstance(__mysql_sandbox_port3);
+// start instance 3
+try_restart_sandbox(__mysql_sandbox_port3);
 
 //@ Cluster: rejoinInstance errors
 Cluster.rejoinInstance();

unittest/scripts/js_devapi/scripts/dba_cluster_rejoin_instance.js

@@ -47,20 +47,18 @@ cluster.addInstance({dbUser: 'foo', host: 'localhost', port:__mysql_sandbox_port
 // Waiting for the instance 3 to become online
 wait_slave_state(cluster, uri3, "ONLINE");
 
-// kill instance 2
+// stop instance 2
+// Use stop sandbox instance to make sure the instance is gone before restarting it
 if (__sandbox_dir)
-  dba.killSandboxInstance(__mysql_sandbox_port2, {sandboxDir:__sandbox_dir});
+    dba.stopSandboxInstance(__mysql_sandbox_port2, {sandboxDir:__sandbox_dir, password: 'root'});
 else
-  dba.killSandboxInstance(__mysql_sandbox_port2);
+    dba.stopSandboxInstance(__mysql_sandbox_port2, {password: 'root'});
 
 // Waiting for instance 2 to become missing
 wait_slave_state(cluster, uri2, "(MISSING)");
 
 // Start instance 2
-if (__sandbox_dir)
-  dba.startSandboxInstance(__mysql_sandbox_port2, {sandboxDir:__sandbox_dir});
-else
-  dba.startSandboxInstance(__mysql_sandbox_port2);
+try_restart_sandbox(__mysql_sandbox_port2);
 
 //@<OUT> Cluster status
 cluster.status()

unittest/scripts/js_devapi/scripts/dba_super_read_only_handling.js

@@ -48,10 +48,16 @@ var cluster = dba.createCluster('sample');
 cluster.addInstance(connection2);
 wait_slave_state(cluster, uri2, "ONLINE");
 
+// Wait for the second added instance to fetch all the replication data
+wait_sandbox_in_metadata(__mysql_sandbox_port2);
+
 //@<OUT> Adds other instance
 cluster.addInstance(connection3);
 wait_slave_state(cluster, uri3, "ONLINE");
 
+// Wait for the third added instance to fetch all the replication data
+wait_sandbox_in_metadata(__mysql_sandbox_port3);
+
 //@<OUT> Rejoins an instance
 stop_sandbox(__mysql_sandbox_port3);
 wait_slave_state(cluster, uri3, "(MISSING)");

unittest/scripts/js_devapi/setup/setup.js

@@ -149,6 +149,34 @@ function wait_slave_state(cluster, slave_uri, states) {
   recov_cluster = null;
 }
 
+function wait_sandbox(timeout, wait_interval, condition, sandbox_port){
+  waiting = 0;
+  res = condition([sandbox_port]);
+  while(!res && waiting < timeout) {
+    os.sleep(wait_interval);
+    waiting = waiting + 1;
+    res = condition([sandbox_port]);
+  }
+  return res;
+}
+
+function check_sandbox_in_metadata(instance_port) {
+  var sandbox_count_metadata =
+    session.runSql("select count(*) from mysql_innodb_cluster_metadata.instances where instance_name = 'localhost:" + instance_port + "'").fetchOne()[0];
+
+    println("---> count(*) sandbox in metadata = " + sandbox_count_metadata);
+
+    return sandbox_count_metadata == "1";
+}
+
+function wait_sandbox_in_metadata(instance_port) {
+  var connected = connect_to_sandbox([instance_port]);
+  if (connected) {
+    wait_sandbox(60, 1, check_sandbox_in_metadata, instance_port);
+    session.close();
+  }
+}
+
 function connect_to_sandbox(params) {
   var port = params[0];
   var connected = false;
@@ -403,6 +431,11 @@ function try_restart_sandbox(port) {
       try {
           dba.startSandboxInstance(port, options);
 
+          // Try to establish a connection to the instance to make sure
+          // it's up and running.
+          shell.connect({scheme: 'mysql', host: localhost, port: port, user: 'root', password: 'root'});
+          session.close();
+
           println(' succeeded');
           return true;
       } catch (err) {

unittest/scripts/js_devapi/validation/dba_cluster_multimaster_interactive.js

@@ -235,10 +235,7 @@ The instance 'root@localhost:<<<__mysql_sandbox_port3>>>' was successfully added
     }
 }
 
-//@# Dba: kill instance 3
-||
-
-//@# Dba: start instance 3
+//@# Dba: stop instance 3
 ||
 
 //@: Cluster: rejoinInstance errors

unittest/scripts/js_devapi/validation/dba_cluster_multimaster_no_interactive.js

@@ -166,10 +166,7 @@
     }
 }
 
-//@# Dba: kill instance 3
-||
-
-//@# Dba: start instance 3
+//@# Dba: stop instance 3
 ||
 
 //@ Cluster: rejoinInstance errors

unittest/scripts/py_devapi/scripts/dba_cluster_add_instance.py

@@ -17,6 +17,9 @@
 # Waiting for the second added instance to become online
 wait_slave_state(single, uri2, "ONLINE")
 
+# Wait for the second added instance to fetch all the replication data
+wait_sandbox_in_metadata(__mysql_sandbox_port2)
+
 # Connect to the future new seed node
 shell.connect({'scheme': 'mysql', 'host': localhost, 'port': __mysql_sandbox_port2, 'user': 'root', 'password': 'root'})
 single_session2 = session
@@ -30,7 +33,7 @@
 else:
   dba.kill_sandbox_instance(__mysql_sandbox_port1)
 
-wait_slave_state(single, uri1, ["UNREACHABLE", "OFFLINE"])
+wait_slave_state(single, uri1, ["(MISSING)"])
 
 #@ Restore the quorum
 single.force_quorum_using_partition_of({'host': localhost, 'port': __mysql_sandbox_port2, 'user': 'root', 'password':'root'})

unittest/scripts/py_devapi/scripts/dba_cluster_interactive.py

@@ -123,7 +123,7 @@
 else:
   dba.kill_sandbox_instance(__mysql_sandbox_port3)
 
-wait_slave_state(cluster, 'third_sandbox', ["UNREACHABLE", "OFFLINE"])
+wait_slave_state(cluster, 'third_sandbox', ["(MISSING)"])
 
 #@# Dba: start instance 3
 if __sandbox_dir:

unittest/scripts/py_devapi/scripts/dba_cluster_multimaster_interactive.py

@@ -73,22 +73,16 @@
 
 # Rejoin tests
 
-#@# Dba: kill instance 3
+#@# Dba: stop instance 3
 if __sandbox_dir:
-  dba.kill_sandbox_instance(__mysql_sandbox_port3, {"sandboxDir":__sandbox_dir})
+  dba.stop_sandbox_instance(__mysql_sandbox_port3, {'sandboxDir': __sandbox_dir, 'password': 'root'})
 else:
-  dba.kill_sandbox_instance(__mysql_sandbox_port3)
+  dba.stop_sandbox_instance(__mysql_sandbox_port3, {'password': 'root'})
 
-# XCOM needs time to kick out the member of the group. The GR team has a patch to fix this
-# But won't be available for the GA release. So we need to wait until the instance is reported
-# as offline
-wait_slave_state(cluster, uri3, ["OFFLINE", "UNREACHABLE"])
+wait_slave_state(cluster, uri3, ["(MISSING)"])
 
-#@# Dba: start instance 3
-if __sandbox_dir:
-  dba.start_sandbox_instance(__mysql_sandbox_port3, {"sandboxDir": __sandbox_dir})
-else:
-  dba.start_sandbox_instance(__mysql_sandbox_port3)
+# start instance 3
+try_restart_sandbox(__mysql_sandbox_port3)
 
 #@: Cluster: rejoin_instance errors
 cluster.rejoin_instance()
@@ -111,3 +105,8 @@
 cluster.status()
 
 cluster.dissolve({'force': True})
+
+# Disable super-read-only (BUG#26422638)
+shell.connect({'scheme': 'mysql', 'host': localhost, 'port': __mysql_sandbox_port1, 'user': 'root', 'password': 'root'})
+session.run_sql("SET GLOBAL SUPER_READ_ONLY = 0;")
+session.close()

unittest/scripts/py_devapi/scripts/dba_cluster_multimaster_no_interactive.py

@@ -65,22 +65,16 @@
 
 # Rejoin tests
 
-#@# Dba: kill instance 3
+#@# Dba: stop instance 3
 if __sandbox_dir:
-  dba.kill_sandbox_instance(__mysql_sandbox_port3, {'sandboxDir': __sandbox_dir})
+  dba.stop_sandbox_instance(__mysql_sandbox_port3, {'sandboxDir': __sandbox_dir, 'password': 'root'})
 else:
-  dba.kill_sandbox_instance(__mysql_sandbox_port3)
+  dba.stop_sandbox_instance(__mysql_sandbox_port3, {'password': 'root'})
 
-# XCOM needs time to kick out the member of the group. The GR team has a patch to fix this
-# But won't be available for the GA release. So we need to wait until the instance is reported
-# as offline
-wait_slave_state(cluster, uri3, ["OFFLINE", "UNREACHABLE"])
+wait_slave_state(cluster, uri3, ["(MISSING)"])
 
-#@# Dba: start instance 3
-if __sandbox_dir:
-  dba.start_sandbox_instance(__mysql_sandbox_port3, {'sandboxDir': __sandbox_dir})
-else:
-  dba.start_sandbox_instance(__mysql_sandbox_port3)
+# start instance 3
+try_restart_sandbox(__mysql_sandbox_port3)
 
 #@ Cluster: rejoin_instance errors
 cluster.rejoin_instance()

unittest/scripts/py_devapi/scripts/dba_cluster_rejoin_instance.py

@@ -47,20 +47,18 @@
 # Waiting for the instance 3 to become online
 wait_slave_state(cluster, uri3, "ONLINE")
 
-# kill instance 2
+# stop instance 2
+# Use stop sandbox instance to make sure the instance is gone before restarting it
 if __sandbox_dir:
-  dba.kill_sandbox_instance(__mysql_sandbox_port2, {'sandboxDir':__sandbox_dir})
+  dba.stop_sandbox_instance(__mysql_sandbox_port2, {'sandboxDir':__sandbox_dir, 'password': 'root'})
 else:
-  dba.kill_sandbox_instance(__mysql_sandbox_port2)
+  dba.stop_sandbox_instance(__mysql_sandbox_port2, {'password': 'root'})
 
 # Waiting for instance 2 to become missing
 wait_slave_state(cluster, uri2, "(MISSING)")
 
 # Start instance 2
-if __sandbox_dir:
-  dba.start_sandbox_instance(__mysql_sandbox_port2, {'sandboxDir':__sandbox_dir})
-else:
-  dba.start_sandbox_instance(__mysql_sandbox_port2)
+try_restart_sandbox(__mysql_sandbox_port2)
 
 #@<OUT> Cluster status
 cluster.status()

unittest/scripts/py_devapi/setup/setup.py

@@ -148,6 +148,28 @@ def wait_slave_state(cluster, slave_uri, states):
 
   recov_cluster = None
 
+def wait_sandbox(timeout, wait_interval, condition, sandbox_port):
+  waiting = 0
+  res = condition([sandbox_port])
+  while not res and waiting < timeout:
+    time.sleep(wait_interval)
+    waiting = waiting + 1
+    res = condition([sandbox_port])
+  return res
+
+def check_sandbox_in_metadata(instance_port):
+  sandbox_count_metadata = session.run_sql("select count(*) from mysql_innodb_cluster_metadata.instances where instance_name = 'localhost:{0}'".format(instance_port)).fetch_one()[0]
+
+  print "---> count(*) sandbox in metadata = %s" % sandbox_count_metadata
+
+  return sandbox_count_metadata == "1"
+
+def wait_sandbox_in_metadata(instance_port):
+  connected = connect_to_sandbox([instance_port]);
+  if (connected):
+    wait_sandbox(60, 1, check_sandbox_in_metadata, instance_port)
+    session.close()
+
 # Smart deployment routines
 
 def connect_to_sandbox(params):
@@ -379,6 +401,12 @@ def try_restart_sandbox(port):
     def try_start():
         try:
             dba.start_sandbox_instance(port, options)
+
+            # Try to establish a connection to the instance to make sure
+            # it's up and running.
+            shell.connect({'host': localhost, 'port': port, 'user': 'root', 'password': 'root'})
+            session.close();
+
             print "succeeded"
             return True
         except Exception, err:

unittest/scripts/py_devapi/validation/dba_cluster_multimaster_interactive.py

@@ -235,10 +235,7 @@
     }
 }
 
-#@# Dba: kill instance 3
-||
-
-#@# Dba: start instance 3
+#@# Dba: stop instance 3
 ||
 
 #@: Cluster: rejoin_instance errors