Bug#26486160 : mysql server upgrade to 8.0.3 with 1 million tables fails

When mysql-5.7 data directory contains a large number of tables, in-place
upgrade to 8.0 takes a large amount of memory.

The large amount of memory consumption is due to usage of THD::mem_root and
THD::main_mem_root for space allocation in

- prepare_create_field() for table migration
- dd::create_dd_user_table() for table migration
- migrate_routine_to_dd()
- migrate_event_to_dd()
- add_sdi_info()

Fix:

- Allocate m_query_tables_list in THD::Transaction_state on heap rather than
  thd->main_mem_root.
- Add fetch_global_component_names() and fetch_global_component_ids() APIs to
  DD Client to replace usage of fetch_global_components() in add_sdi_info().
- Reduce scope of Auto_releaser in add_sdi_info() to each Tablespace and to
  each Table.
- Swap out THD::mem_root while adding SDI information and migrating tables,
  events and routines.
- Add trans_commit_stmt() and trans_commit() after each call of
  dd::sdi::store().
- Register InnoDB engine in the Transaction context when calling
  Handler::sdi_set() so that trans_commit_stmt() can release InnoDB locks on
  the tables.

Change-Id: Ib96259a5c991d48c90df6ea0735f6fbce4aabb23

Author: amitab

sql/dd/cache/dictionary_client.h

@@ -855,6 +855,34 @@ class Dictionary_client {
                                     std::vector<String_type> *names) const
       MY_ATTRIBUTE((warn_unused_result));
 
+  /**
+    Fetch all global component ids of the given type.
+
+    @tparam        T              Type of components to get.
+    @param   [out] ids            An std::vector containing all component ids.
+
+    @return      true   Failure (error is reported).
+    @return      false  Success.
+  */
+
+  template <typename T>
+  bool fetch_global_component_ids(std::vector<Object_id> *ids) const
+      MY_ATTRIBUTE((warn_unused_result));
+
+  /**
+    Fetch all global component names of the given type.
+
+    @tparam        T              Type of components to get.
+    @param   [out] names          An std::vector containing all component names.
+
+    @return      true   Failure (error is reported).
+    @return      false  Success.
+  */
+
+  template <typename T>
+  bool fetch_global_component_names(std::vector<String_type> *names) const
+      MY_ATTRIBUTE((warn_unused_result));
+
   /**
     Fetch all components in the schema.
 

sql/dd/impl/cache/dictionary_client.cc

@@ -1877,6 +1877,80 @@ bool Dictionary_client::check_foreign_key_exists(
   return false;
 }
 
+template <typename T>
+bool fetch_raw_record(THD *thd,
+                      std::function<bool(Raw_record *)> const &processor) {
+  Transaction_ro trx(thd, ISO_READ_COMMITTED);
+
+  trx.otx.register_tables<T>();
+  Raw_table *table = trx.otx.get_table<T>();
+  DBUG_ASSERT(table);
+
+  if (trx.otx.open_tables()) {
+    DBUG_ASSERT(thd->is_system_thread() || thd->killed || thd->is_error());
+    return true;
+  }
+
+  std::unique_ptr<Raw_record_set> rs;
+  if (table->open_record_set(nullptr, rs)) {
+    DBUG_ASSERT(thd->is_system_thread() || thd->killed || thd->is_error());
+    return true;
+  }
+
+  Raw_record *r = rs->current_record();
+  String_type s;
+  while (r) {
+    if (processor(r) || rs->next(r)) {
+      DBUG_ASSERT(thd->is_system_thread() || thd->killed || thd->is_error());
+      return true;
+    }
+  }
+
+  return false;
+}
+
+// Fetch the ids of all the components.
+template <typename T>
+bool Dictionary_client::fetch_global_component_ids(
+    std::vector<Object_id> *ids) const {
+  DBUG_ASSERT(ids);
+
+  auto processor = [&](Raw_record *r) -> bool {
+    ids->push_back(r->read_int(0));
+    return false;
+  };
+
+  if (fetch_raw_record<T>(m_thd, processor)) {
+    ids->clear();
+    DBUG_ASSERT(m_thd->is_system_thread() || m_thd->killed ||
+                m_thd->is_error());
+    return true;
+  }
+
+  return false;
+}
+
+// Fetch the names of all the components.
+template <typename T>
+bool Dictionary_client::fetch_global_component_names(
+    std::vector<String_type> *names) const {
+  DBUG_ASSERT(names);
+
+  auto processor = [&](Raw_record *r) -> bool {
+    names->push_back(r->read_str(T::DD_table::FIELD_NAME));
+    return false;
+  };
+
+  if (fetch_raw_record<T>(m_thd, processor)) {
+    names->clear();
+    DBUG_ASSERT(m_thd->is_system_thread() || m_thd->killed ||
+                m_thd->is_error());
+    return true;
+  }
+
+  return false;
+}
+
 // Fetch the names of all the components in the schema.
 template <typename T>
 bool Dictionary_client::fetch_schema_component_names(
@@ -2757,6 +2831,12 @@ template bool Dictionary_client::fetch_schema_component_names<Abstract_table>(
 template bool Dictionary_client::fetch_schema_component_names<Event>(
     const Schema *, std::vector<String_type> *) const;
 
+template bool Dictionary_client::fetch_global_component_ids<Table>(
+    std::vector<Object_id> *) const;
+
+template bool Dictionary_client::fetch_global_component_names<Tablespace>(
+    std::vector<String_type> *) const;
+
 template bool Dictionary_client::fetch_referencing_views_object_id<View_table>(
     const char *schema, const char *tbl_or_sf_name,
     std::vector<Object_id> *view_ids) const;

sql/dd/impl/sdi.cc

@@ -545,7 +545,7 @@ namespace sdi {
 
 bool store(THD *thd, const Table *tp) {
   const Table &t = ptr_as_cref(tp);
-  const handlerton &hton = ptr_as_cref(resolve_hton(thd, t));
+  handlerton *hton = resolve_hton(thd, t);
 
   return with_schema(thd, t.schema_id(), [&](const Schema &s) {
     dd::Sdi_type sdi = serialize(thd, t, s.name());
@@ -558,8 +558,8 @@ bool store(THD *thd, const Table *tp) {
       return true;
     });
     return checked_return(
-        hton.sdi_set ? sdi_tablespace::store_tbl_sdi(thd, hton, sdi, t, s)
-                     : sdi_file::store_tbl_sdi(sdi, t, s));
+        hton->sdi_set ? sdi_tablespace::store_tbl_sdi(thd, hton, sdi, t, s)
+                      : sdi_file::store_tbl_sdi(sdi, t, s));
   });
 }
 
@@ -572,7 +572,7 @@ bool store(THD *thd, const Tablespace *ts) {
   if (sdi.empty()) {
     return checked_return(true);
   }
-  return checked_return(sdi_tablespace::store_tsp_sdi(*hton, sdi, *ts));
+  return checked_return(sdi_tablespace::store_tsp_sdi(hton, sdi, *ts));
 }
 
 bool drop(THD *thd, const Table *tp) {

sql/dd/impl/sdi_tablespace.cc

@@ -1,4 +1,4 @@
-/* Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -163,7 +163,7 @@ dd::sdi_key_t get_sdi_key(const dd::Tablespace &tablespace) {
 
 namespace dd {
 namespace sdi_tablespace {
-bool store_tbl_sdi(THD *thd, const handlerton &hton, const dd::Sdi_type &sdi,
+bool store_tbl_sdi(THD *thd, handlerton *hton, const dd::Sdi_type &sdi,
                    const dd::Table &table,
                    const dd::Schema &schema MY_ATTRIBUTE((unused))) {
   const dd::sdi_key_t key = get_sdi_key(table);
@@ -172,7 +172,7 @@ bool store_tbl_sdi(THD *thd, const handlerton &hton, const dd::Sdi_type &sdi,
     DBUG_PRINT("ddsdi", ("store_sdi_with_schema[](Schema" ENTITY_FMT
                          ", Table" ENTITY_FMT ")",
                          ENTITY_VAL(schema), ENTITY_VAL(table)));
-    if (hton.sdi_set(tblspc, &table, &key, sdi.c_str(), sdi.size())) {
+    if (hton->sdi_set(hton, tblspc, &table, &key, sdi.c_str(), sdi.size())) {
       return checked_return(true);
     }
 
@@ -182,12 +182,12 @@ bool store_tbl_sdi(THD *thd, const handlerton &hton, const dd::Sdi_type &sdi,
   return apply_to_tablespaces(thd, table, store_sdi);
 }
 
-bool store_tsp_sdi(const handlerton &hton, const Sdi_type &sdi,
+bool store_tsp_sdi(handlerton *hton, const Sdi_type &sdi,
                    const Tablespace &tblspc) {
   dd::sdi_key_t key = get_sdi_key(tblspc);
 
   DBUG_PRINT("ddsdi", ("store_tsp_sdi(" ENTITY_FMT ")", ENTITY_VAL(tblspc)));
-  if (hton.sdi_set(tblspc, nullptr, &key, sdi.c_str(), sdi.size())) {
+  if (hton->sdi_set(hton, tblspc, nullptr, &key, sdi.c_str(), sdi.size())) {
     return checked_return(true);
   }
   return false;

sql/dd/impl/sdi_tablespace.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -61,7 +61,7 @@ namespace sdi_tablespace {
   @param table
   @param schema
  */
-bool store_tbl_sdi(THD *thd, const handlerton &hton, const Sdi_type &sdi,
+bool store_tbl_sdi(THD *thd, handlerton *hton, const Sdi_type &sdi,
                    const Table &table, const dd::Schema &schema);
 
 /**
@@ -71,7 +71,7 @@ bool store_tbl_sdi(THD *thd, const handlerton &hton, const Sdi_type &sdi,
   @param sdi
   @param tablespace
  */
-bool store_tsp_sdi(const handlerton &hton, const Sdi_type &sdi,
+bool store_tsp_sdi(handlerton *hton, const Sdi_type &sdi,
                    const Tablespace &tablespace);
 
 /**

sql/dd/upgrade/event.cc

@@ -551,12 +551,12 @@ bool migrate_events_to_dd(THD *thd) {
   uint flags = MYSQL_LOCK_IGNORE_TIMEOUT;
   DML_prelocking_strategy prelocking_strategy;
   MEM_ROOT records_mem_root;
+  Thd_mem_root_guard root_guard(thd, &records_mem_root);
 
   tables.init_one_table("mysql", 5, "event", 5, "event", TL_READ);
 
   table_list = &tables;
   if (open_and_lock_tables(thd, table_list, flags, &prelocking_strategy)) {
-    close_thread_tables(thd);
     LogErr(ERROR_LEVEL, ER_EVENT_CANT_OPEN_TABLE_MYSQL_EVENT);
     return true;
   }
@@ -593,10 +593,6 @@ bool migrate_events_to_dd(THD *thd) {
     goto err;
   }
 
-  init_sql_alloc(PSI_NOT_INSTRUMENTED, &records_mem_root, MEM_ROOT_BLOCK_SIZE,
-                 0);
-  thd->mem_root = &records_mem_root;
-
   if (migrate_event_to_dd(thd, event_table)) goto err;
 
   // Read the next row in 'event' table via index.
@@ -610,12 +606,10 @@ bool migrate_events_to_dd(THD *thd) {
   }
 
   my_tz_free();
-  free_root(&records_mem_root, MYF(0));
   return false;
 
 err:
   my_tz_free();
-  free_root(&records_mem_root, MYF(0));
   return true;
 }
 

sql/dd/upgrade/global.cc

@@ -1,4 +1,4 @@
-/* Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -40,14 +40,20 @@ namespace upgrade_57 {
 const char *TRN_EXT = ".TRN";
 const char *TRG_EXT = ".TRG";
 
-System_table_close_guard::System_table_close_guard(THD *thd, TABLE *table) {
+Thd_mem_root_guard::Thd_mem_root_guard(THD *thd, MEM_ROOT *mem_root) {
   m_thd = thd;
-  m_table = table;
-  m_mem_root = m_thd->mem_root;
+  m_thd_prev_mem_root = m_thd->mem_root;
+  m_thd->mem_root = mem_root;
 }
 
+Thd_mem_root_guard::~Thd_mem_root_guard() {
+  m_thd->mem_root = m_thd_prev_mem_root;
+}
+
+System_table_close_guard::System_table_close_guard(THD *thd, TABLE *table)
+    : m_thd(thd), m_table(table) {}
+
 System_table_close_guard::~System_table_close_guard() {
-  m_thd->mem_root = m_mem_root;
   if (m_table->file->inited) (void)m_table->file->ha_index_end();
   close_thread_tables(m_thd);
 }

sql/dd/upgrade/global.h

@@ -54,14 +54,26 @@ const String_type index_stats_backup = "innodb_index_stats_backup57";
 const String_type table_stats = "innodb_table_stats";
 const String_type table_stats_backup = "innodb_table_stats_backup57";
 
+/**
+  THD::mem_root is only switched with the given mem_root and switched back
+  on destruction. This does not free any mem_root.
+ */
+class Thd_mem_root_guard {
+  THD *m_thd;
+  MEM_ROOT *m_thd_prev_mem_root;
+
+ public:
+  Thd_mem_root_guard(THD *thd, MEM_ROOT *mem_root);
+  ~Thd_mem_root_guard();
+};
+
 /**
    RAII for handling open and close of event and proc tables.
 */
 
 class System_table_close_guard {
   THD *m_thd;
   TABLE *m_table;
-  MEM_ROOT *m_mem_root;
 
  public:
   System_table_close_guard(THD *thd, TABLE *table);

sql/dd/upgrade/routine.cc

@@ -442,12 +442,12 @@ bool migrate_routines_to_dd(THD *thd) {
   uint flags = MYSQL_LOCK_IGNORE_TIMEOUT;
   DML_prelocking_strategy prelocking_strategy;
   MEM_ROOT records_mem_root;
+  Thd_mem_root_guard root_guard(thd, &records_mem_root);
 
   tables.init_one_table("mysql", 5, "proc", 4, "proc", TL_READ);
 
   table_list = &tables;
   if (open_and_lock_tables(thd, table_list, flags, &prelocking_strategy)) {
-    close_thread_tables(thd);
     LogErr(ERROR_LEVEL, ER_CANT_OPEN_TABLE_MYSQL_PROC);
     return true;
   }
@@ -473,35 +473,25 @@ bool migrate_routines_to_dd(THD *thd) {
   // Read first record from mysql.proc table. Return if table is empty.
   if ((error = proc_table->file->ha_index_first(proc_table->record[0]))) {
     if (error == HA_ERR_END_OF_FILE) return false;
-
     LogErr(ERROR_LEVEL, ER_CANT_READ_TABLE_MYSQL_PROC);
     return true;
   }
 
-  init_sql_alloc(PSI_NOT_INSTRUMENTED, &records_mem_root, MEM_ROOT_BLOCK_SIZE,
-                 0);
-  thd->mem_root = &records_mem_root;
-
   // Migrate first record read to dd routines table.
-  if (migrate_routine_to_dd(thd, proc_table)) goto err;
+  if (migrate_routine_to_dd(thd, proc_table)) return true;
 
   // Read one record from mysql.proc table and
   // migrate it until all records are finished
   while (!(error = proc_table->file->ha_index_next(proc_table->record[0]))) {
-    if (migrate_routine_to_dd(thd, proc_table)) goto err;
+    if (migrate_routine_to_dd(thd, proc_table)) return true;
   }
 
   if (error != HA_ERR_END_OF_FILE) {
     LogErr(ERROR_LEVEL, ER_CANT_READ_TABLE_MYSQL_PROC);
-    goto err;
+    return true;
   }
 
-  free_root(&records_mem_root, MYF(0));
   return false;
-
-err:
-  free_root(&records_mem_root, MYF(0));
-  return true;
 }
 
 }  // namespace upgrade_57