Bug#25475132 GROUP REPLICATION: MYSQL CLIENT CONNECTIONS HANG DURING GROUP REPLICATION START

Bug#26037344 THE DELAYED PLUGIN INITIALIZATION THREAD IS LEFT RUNNING WHEN IT IS NOT NEEDED

Client complaints existed about how group replication would make
connections hang on server start if the plugin was also
starting. i.e., joining the group.
To solve this we reworked the delayed initialization mechanism used on
server starts, so it only blocks connections until the server is in
read mode, not waiting for the member to join the group as before.
This rework also solves the issue of the thread responsible for this
task being initialized and running even when not needed.

Author: pedrolgomes

mysql-test/include/rpl_reconnect.inc

@@ -62,6 +62,10 @@ if ($rpl_server_number == 1)
 
     if ($rpl_group_replication)
     {
+      --let $rpl_connection_name= server1
+      --source include/rpl_connection.inc
+      --enable_reconnect
+
       --let $rpl_connection_name= server_1
       --source include/rpl_connection.inc
       --enable_reconnect
@@ -83,6 +87,10 @@ if ($rpl_server_number == 2)
 
     if ($rpl_group_replication)
     {
+      --let $rpl_connection_name= server2
+      --source include/rpl_connection.inc
+      --enable_reconnect
+
       --let $rpl_connection_name= server_2
       --source include/rpl_connection.inc
       --enable_reconnect
@@ -131,6 +139,10 @@ if ($rpl_server_number == 1)
 
     if ($rpl_group_replication)
     {
+      --let $rpl_connection_name= server1
+      --source include/rpl_connection.inc
+      --enable_reconnect
+
       --let $rpl_connection_name= server_1
       --source include/rpl_connection.inc
       --source include/wait_until_connected_again.inc
@@ -155,6 +167,10 @@ if ($rpl_server_number == 2)
 
     if ($rpl_group_replication)
     {
+      --let $rpl_connection_name= server2
+      --source include/rpl_connection.inc
+      --enable_reconnect
+
       --let $rpl_connection_name= server_2
       --source include/rpl_connection.inc
       --source include/wait_until_connected_again.inc

rapid/plugin/group_replication/include/delayed_plugin_initialization.h

@@ -56,7 +56,17 @@ class Delayed_initialization_thread
   /**
     Wait for the initialization thread to do its job.
   */
-  void wait_for_initialization();
+  void wait_for_thread_end();
+
+  /**
+    Signal that the read mode is set on the server.
+  */
+  void signal_read_mode_ready();
+
+  /**
+    Wait for the read mode to be set by the thread process.
+  */
+  void wait_for_read_mode();
 
 private:
 
@@ -68,6 +78,9 @@ class Delayed_initialization_thread
   /* Is the server ready*/
   bool is_server_ready;
 
+  /* Is the read mode already set*/
+  bool is_super_read_only_set;
+
   /* Thread related structures */
 
   my_thread_handle delayed_init_pthd;

rapid/plugin/group_replication/include/plugin.h

@@ -82,26 +82,13 @@ extern Blocked_transaction_handler* blocked_transaction_handler;
 //Plugin global methods
 bool server_engine_initialized();
 void *get_plugin_pointer();
-int configure_and_start_applier_module();
-int configure_group_member_manager(char *hostname, char *uuid,
-                                   uint port, unsigned int server_version);
-int configure_compatibility_manager();
-int terminate_applier_module();
-int initialize_recovery_module();
-void initialize_group_partition_handler();
-int terminate_recovery_module();
-int configure_group_communication(st_server_ssl_variables *ssl_variables);
-int start_group_communication();
-void declare_plugin_running();
+mysql_mutex_t* get_plugin_running_lock();
+int initialize_plugin_and_join(enum_plugin_con_isolation sql_api_isolation,
+                               Delayed_initialization_thread *delayed_init_thd);
 void register_server_reset_master();
-int leave_group();
-int terminate_plugin_modules();
 bool get_allow_local_lower_version_join();
 bool get_allow_local_disjoint_gtids_join();
 ulong get_transaction_size_limit();
-void initialize_asynchronous_channels_observer();
-void terminate_asynchronous_channels_observer();
-bool check_async_channel_running_on_secondary();
 bool is_plugin_waiting_to_set_server_read_mode();
 
 //Plugin public methods

rapid/plugin/group_replication/src/delayed_plugin_initialization.cc

@@ -29,7 +29,7 @@ static void *launch_handler_thread(void* arg)
 }
 
 Delayed_initialization_thread::Delayed_initialization_thread()
-  : thread_running(false), is_server_ready(false)
+  : thread_running(false), is_server_ready(false), is_super_read_only_set(false)
 {
   mysql_mutex_init(key_GR_LOCK_delayed_init_run, &run_lock, MY_MUTEX_INIT_FAST);
   mysql_mutex_init(key_GR_LOCK_delayed_init_server_ready,
@@ -61,14 +61,14 @@ void Delayed_initialization_thread::signal_thread_ready()
   DBUG_VOID_RETURN;
 }
 
-void Delayed_initialization_thread::wait_for_initialization()
+void Delayed_initialization_thread::wait_for_thread_end()
 {
-  DBUG_ENTER("Delayed_initialization_thread::wait_for_initialization");
+  DBUG_ENTER("Delayed_initialization_thread::wait_for_thread_end");
 
   mysql_mutex_lock(&run_lock);
   while (thread_running)
   {
-    DBUG_PRINT("sleep",("Waiting for the Delayed initialization thread to end"));
+    DBUG_PRINT("sleep",("Waiting for the Delayed initialization thread to finish"));
     mysql_cond_wait(&run_cond, &run_lock);
   }
   mysql_mutex_unlock(&run_lock);
@@ -79,6 +79,33 @@ void Delayed_initialization_thread::wait_for_initialization()
   DBUG_VOID_RETURN;
 }
 
+void Delayed_initialization_thread::signal_read_mode_ready()
+{
+  DBUG_ENTER("Delayed_initialization_thread::signal_read_mode_ready");
+
+  mysql_mutex_lock(&run_lock);
+  is_super_read_only_set= true;
+  mysql_cond_broadcast(&run_cond);
+  mysql_mutex_unlock(&run_lock);
+
+  DBUG_VOID_RETURN;
+}
+
+void Delayed_initialization_thread::wait_for_read_mode()
+{
+  DBUG_ENTER("Delayed_initialization_thread::wait_for_read_mode");
+
+  mysql_mutex_lock(&run_lock);
+  while (!is_super_read_only_set)
+  {
+    DBUG_PRINT("sleep",("Waiting for the Delayed initialization thread to set super_read_only"));
+    mysql_cond_wait(&run_cond, &run_lock);
+  }
+  mysql_mutex_unlock(&run_lock);
+
+  DBUG_VOID_RETURN;
+}
+
 int Delayed_initialization_thread::launch_initialization_thread()
 {
   DBUG_ENTER("Delayed_initialization_thread::launch_initialization_thread");
@@ -128,158 +155,12 @@ int Delayed_initialization_thread::initialization_thread_handler()
   }
   mysql_mutex_unlock(&server_ready_lock);
 
-  //delayed initialization code starts here
-
-  int error= 0;
-  Sql_service_command_interface *sql_command_interface= NULL;
-
-  //Just terminate it
-  if (!wait_on_engine_initialization ||
-      get_plugin_pointer() == NULL)
-  {
-    goto end;
-  }
-
-  /*
-    The plugin was initialized on server start
-    so only now we can start the applier
-  */
-  if (wait_on_engine_initialization)
-  {
-    DBUG_ASSERT(server_engine_initialized());
-    wait_on_engine_initialization= false;
-
-    //Avoid unnecessary operations
-    bool enabled_super_read_only= false;
-    bool read_only_mode= false, super_read_only_mode=false;
-
-    char *hostname, *uuid;
-    uint port;
-    unsigned int server_version;
-    st_server_ssl_variables server_ssl_variables=
-      {false,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL};
-
-    get_server_parameters(&hostname, &port, &uuid, &server_version,
-                          &server_ssl_variables);
-
-    sql_command_interface= new Sql_service_command_interface();
-    if (sql_command_interface->
-            establish_session_connection(PSESSION_INIT_THREAD,
-                                         get_plugin_pointer()) ||
-        sql_command_interface->set_interface_user(GROUPREPL_USER))
-    {
-      /* purecov: begin inspected */
-      log_message(MY_ERROR_LEVEL,
-                  "It was not possible to establish a connection to "
-                    "server SQL service");
-      error= 1;
-      goto err;
-      /* purecov: end */
-    }
-
-    /*
-     At this point in the code, set the super_read_only mode here on the
-     server to protect recovery and version module of the Group Replication.
-
-     Save the current read mode state to restore it in case Group Replication
-     fail to start.
-    */
-
-    get_read_mode_state(sql_command_interface, &read_only_mode,
-                        &super_read_only_mode);
-
-    if (enable_super_read_only_mode(sql_command_interface))
-    {
-      error =1; /* purecov: inspected */
-      log_message(MY_ERROR_LEVEL,
-                  "Could not enable the server read only mode and guarantee a "
-                  "safe recovery execution"); /* purecov: inspected */
-      goto err; /* purecov: inspected */
-    }
-
-    enabled_super_read_only= true;
-
-    if ((error= configure_group_communication(&server_ssl_variables)))
-      goto err; /* purecov: inspected */
-
-    if ((error=  configure_group_member_manager(hostname, uuid, port,
-                                                server_version)))
-      goto err; /* purecov: inspected */
-
-    if (check_async_channel_running_on_secondary())
-    {
-      error= 1;
-      log_message(MY_ERROR_LEVEL, "Can't start group replication on secondary"
-                                  " member with single primary-mode while"
-                                  " asynchronous replication channels are"
-                                  " running.");
-      goto err; /* purecov: inspected */
-    }
-
-    configure_compatibility_manager();
-
-    // need to be initialized before applier, is called on kill_pending_transactions
-    blocked_transaction_handler= new Blocked_transaction_handler();
-
-    if ((error= initialize_recovery_module()))
-      goto err; /* purecov: inspected */
-
-    if (configure_and_start_applier_module())
-    {
-      error= GROUP_REPLICATION_REPLICATION_APPLIER_INIT_ERROR;
-      goto err;
-    }
-
-    initialize_asynchronous_channels_observer();
-    initialize_group_partition_handler();
-
-    if ((error= start_group_communication()))
-    {
-      //terminate the before created pipeline
-      log_message(MY_ERROR_LEVEL,
-                  "Error on group communication initialization methods, "
-                  "killing the Group Replication applier"); /* purecov: inspected */
-      applier_module->terminate_applier_thread(); /* purecov: inspected */
-      goto err; /* purecov: inspected */
-    }
-
-    if (view_change_notifier->wait_for_view_modification())
-    {
-      /* purecov: begin inspected */
-      if (!view_change_notifier->is_cancelled())
-      {
-        //Only log a error when a view modification was not canceled.
-        log_message(MY_ERROR_LEVEL,
-                    "Timeout on wait for view after joining group");
-      }
-      error= view_change_notifier->get_error();
-      goto err;
-      /* purecov: end */
-    }
-    declare_plugin_running(); //All is OK
-
-  err:
-    if (error)
-    {
-      leave_group();
-      terminate_plugin_modules();
-      if (!server_shutdown_status && server_engine_initialized()
-          && enabled_super_read_only)
-      {
-        set_read_mode_state(sql_command_interface, read_only_mode,
-                            super_read_only_mode);
-      }
-      if (certification_latch != NULL)
-      {
-        delete certification_latch; /* purecov: inspected */
-        certification_latch= NULL;  /* purecov: inspected */
-      }
-    }
-  }
+  DBUG_ASSERT(server_engine_initialized());
 
-end:
+  //Protect this delayed start against other start/stop requests
+  Mutex_autolock auto_lock_mutex(get_plugin_running_lock());
 
-  delete sql_command_interface;
+  int error= initialize_plugin_and_join(PSESSION_INIT_THREAD, this);
 
   mysql_mutex_lock(&run_lock);
   thread_running= false;

rapid/plugin/group_replication/src/observer_server_state.cc

@@ -26,9 +26,7 @@ int group_replication_before_handle_connection(Server_state_param *param)
   if (wait_on_engine_initialization)
   {
     delayed_initialization_thread->signal_thread_ready();
-    delayed_initialization_thread->wait_for_initialization();
-    delete delayed_initialization_thread;
-    delayed_initialization_thread= NULL;
+    delayed_initialization_thread->wait_for_read_mode();
   }
   return 0;
 }