WL#7167 - Change DDL to update rows for view columns in DD.COLUMNS and other dependent values.

With WL6599(Integration of IS with New Data Dictionary),
INFORMATION_SCHEMA.COLUMNS is implemented as a view on the new
data dictionary table mysql.columns. But only column information
of tables is stored in the mysql.columns. Hence only columns of
tables are listed by the INFORMATION_SCHEMA.COLUMNS table.
Even SHOW COLUMNS lists columns of only tables.

The main goal of this work log is to support INFORMATION_SCHEMA.COLUMNS
and SHOW COLUMNS implementation to list even view columns
information.

Functional changes introduced:
------------------------------

1. Now view columns information is stored in the data dictionary
   table mysql.columns and these rows are updated each time
   type of the column changes i.e each time when type of columns
   in the base table changes.

   With this change INFORMATION_SCHEMA.COLUMNS and SHOW COLUMNS
   lists columns from the view.

2. If view becomes invalid then correct error warning or
   error messages are reported.

   Section A.c of  the WL page lists reason for becoming
   valid or invalid.

3. This work log even takes care of updating mysql.tables.is_updatable,
   each time when view becomes updatable or non-updatable.

   With this change, INFORMATION_SCHEMA.VIEWS shows the correct IS_UPDATABLE
   state of view always.

Compatibility issues:
------------------------
The view can become valid or invalid on execution of account
management statements too.  The error/warning reporting in such
case is not handled as part of this WL. It will be handled
as part of WL9496.

Source files:
----------------
Most of code changes is placed in sql/dd_sql_view.* and
sql/dd/dd_view.*.

Related worklogs:
------------------
* WL#6599 - New Data Dictionary and I_S integration.

  This WL defines INFORMATION_SCHEMA (I_S) system view over DD
  tables, representing a I_S table. WL7167 implemented over the
  WL6599 changes. So WL#6599 and WL#7167 would be pushed together.
  They are QA'ed together.

Upcoming WLs:
------------------
WL#9496 - Extend view status(valid/invalid) handling to Account
          Management Statements.

Author: phulakun

client/dump/mysql_object_reader.cc

@@ -90,7 +90,8 @@ void Mysql_object_reader::read_table_rows_task(
     "SELECT `COLUMN_NAME`, `EXTRA` FROM " +
     this->get_quoted_object_full_name("INFORMATION_SCHEMA", "COLUMNS") +
     "WHERE TABLE_SCHEMA ='" + runner->escape_string(table->get_schema()) +
-    "' AND TABLE_NAME ='" + runner->escape_string(table->get_name()) + "'",
+    "' AND TABLE_NAME ='" + runner->escape_string(table->get_name()) + "'" +
+    " ORDER BY ORDINAL_POSITION ",
     &columns);
 
   std::string column_names;

client/mysqldump.cc

@@ -1709,6 +1709,40 @@ static int connect_to_db(char *host, char *user,char *passwd)
     if (mysql_query_with_error_report(mysql, 0, buff))
       DBUG_RETURN(1);
   }
+
+  /*
+    With the introduction of new information schema views on top
+    of new data dictionary, the way the SHOW command works is
+    changed. We now have two ways of SHOW command picking table
+    statistics.
+
+    One is to read it from DD table mysql.table_stats and
+    mysql.index_stats. For this to happen, we need to execute
+    ANALYZE TABLE prior to execution of mysqldump tool.  As the
+    tool can run on whole database, we would end-up running
+    ANALYZE TABLE for all tables in the database irrespective of
+    whether statistics are already present in the statistics
+    tables. This could be a time consuming additional step to
+    carry.
+
+    Second option is to read statistics from SE itself. This
+    options looks safe and execution of mysqldump tool need not
+    care if ANALYZE TABLE command was run on every table. We
+    always get the statistics, which match the behavior without
+    data dictionary.
+
+    The first option would be faster as we do not opening the
+    underlying tables during execution of SHOW command. However
+    the first option might read old statistics, so we feel second
+    option is preferred here to get statistics dynamically from
+    SE by setting information_schema_stats=latest for this
+    session.
+  */
+  my_snprintf(buff, sizeof(buff),
+              "/*!80000 SET SESSION INFORMATION_SCHEMA_STATS=latest */");
+  if (mysql_query_with_error_report(mysql, 0, buff))
+    DBUG_RETURN(1);
+
   DBUG_RETURN(0);
 } /* connect_to_db */
 
@@ -2619,7 +2653,8 @@ static uint get_table_structure(char *table, char *db, char *table_type,
                                 "`EXTRA` AS `Extra`, "
                                 "`COLUMN_COMMENT` AS `Comment` "
                                 "FROM `INFORMATION_SCHEMA`.`COLUMNS` WHERE "
-                                "TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s'";
+                                "TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s' "
+                                "ORDER BY ORDINAL_POSITION";
   FILE       *sql_file= md_result_file;
   size_t     len;
   my_bool    is_log_table;

mysql-test/include/commit.inc

@@ -600,7 +600,7 @@ begin
 end|
 delimiter ;|
 --disable_warnings
-call p_verify_status_increment(3, 0, 2, 0);
+call p_verify_status_increment(4, 0, 2, 0);
 --enable_warnings
 
 --echo # 16. A function changes non-trans-table.
@@ -653,7 +653,7 @@ drop table t2;
 set sql_mode=no_engine_substitution;
 create temporary table t2 (a int);
 --disable_warnings
-call p_verify_status_increment(5, 0, 0, 0);
+call p_verify_status_increment(6, 0, 0, 0);
 --enable_warnings
 set sql_mode=default;
 --echo # 19. A function changes temp-trans-table.
@@ -791,13 +791,13 @@ create table t2 (a int);
 # COM_PREPARE.
 if (`SELECT $PS_PROTOCOL = 0`)
 {
-  --replace_regex /(call p_verify_status_increment.*)6(.*6)/\1<commit_count>\2/
-  call p_verify_status_increment(6, 0, 6, 0);
+  --replace_regex /(call p_verify_status_increment.*)7(.*6)/\1<commit_count>\2/
+  call p_verify_status_increment(7, 0, 6, 0);
 }
 if (`SELECT $PS_PROTOCOL > 0`)
 {
-  --replace_regex /(call p_verify_status_increment.*)7(.*6)/\1<commit_count>\2/
-  call p_verify_status_increment(7, 0, 6, 0);
+  --replace_regex /(call p_verify_status_increment.*)8(.*6)/\1<commit_count>\2/
+  call p_verify_status_increment(8, 0, 6, 0);
 }
 do (select f1() from t1 where a=2);
 call p_verify_status_increment(2, 2, 3, 2);
@@ -850,24 +850,24 @@ create table t3 select a from t2;
 # COM_PREPARE.
 if (`SELECT $PS_PROTOCOL = 0`)
 {
-  --replace_regex /8/<commit_count>/
-  call p_verify_status_increment(8, 0, 4, 4);
+  --replace_regex /9/<commit_count>/
+  call p_verify_status_increment(9, 0, 4, 4);
 }
 if (`SELECT $PS_PROTOCOL > 0`)
 {
-  --replace_regex /9/<commit_count>/
-  call p_verify_status_increment(9, 0, 4, 4);
+  --replace_regex /10/<commit_count>/
+  call p_verify_status_increment(10, 0, 4, 4);
 }
 alter table t3 add column (b int);
 call p_verify_status_increment(10, 0, 2, 0);
 alter table t3 rename t4;
-call p_verify_status_increment(5, 0, 3, 0);
+call p_verify_status_increment(7, 0, 3, 0);
 rename table t4 to t3;
 call p_verify_status_increment(7, 0, 5, 0);
 truncate table t3;
 call p_verify_status_increment(2, 0, 4, 0);
 create view v1 as select * from t2;
-call p_verify_status_increment(4, 0, 3, 0);
+call p_verify_status_increment(5, 0, 3, 0);
 check table t1;
 call p_verify_status_increment(2, 0, 2, 0);
 --echo # Sic: after this bug is fixed, CHECK leaves no pending transaction
@@ -878,7 +878,7 @@ call p_verify_status_increment(6, 0, 6, 0);
 commit;
 call p_verify_status_increment(0, 0, 0, 0);
 drop view v1;
-call p_verify_status_increment(5, 0, 3, 0);
+call p_verify_status_increment(6, 0, 3, 0);
 --enable_warnings
 
 --echo #

mysql-test/include/view_alias.inc

@@ -3,7 +3,7 @@
 # The variable $after_select must be set before calling this routine.
 
 eval CREATE VIEW v1 AS SELECT $after_select;
-SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = 'test' AND TABLE_NAME = 'v1';
+SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = 'test' AND TABLE_NAME = 'v1' ORDER BY COLUMN_NAME;
 #
 # Extract the VIEW's SELECT from INFORMATION_SCHEMA.VIEWS
 let $query1 = `SELECT VIEW_DEFINITION FROM INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA = 'test' AND TABLE_NAME = 'v1'`;

mysql-test/r/commit_1innodb.result

@@ -570,7 +570,7 @@ begin
 insert t2 set a=2;
 return 2;
 end|
-call p_verify_status_increment(3, 0, 2, 0);
+call p_verify_status_increment(4, 0, 2, 0);
 SUCCESS
 
 # 16. A function changes non-trans-table.
@@ -633,7 +633,7 @@ set sql_mode=no_engine_substitution;
 Warnings:
 Warning	3090	Changing sql mode 'NO_AUTO_CREATE_USER' is deprecated. It will be removed in a future release.
 create temporary table t2 (a int);
-call p_verify_status_increment(5, 0, 0, 0);
+call p_verify_status_increment(6, 0, 0, 0);
 SUCCESS
 
 set sql_mode=default;
@@ -844,22 +844,22 @@ SUCCESS
 
 alter table t3 add column (b int);
 call p_verify_status_increment(10, 0, 2, 0);
-SUCCESS
-
+ERROR
+Expected commit increment: 10 actual: 11
 alter table t3 rename t4;
-call p_verify_status_increment(5, 0, 3, 0);
+call p_verify_status_increment(7, 0, 3, 0);
 SUCCESS
 
 rename table t4 to t3;
 call p_verify_status_increment(7, 0, 5, 0);
-SUCCESS
-
+ERROR
+Expected commit increment: 7 actual: 9
 truncate table t3;
 call p_verify_status_increment(2, 0, 4, 0);
 SUCCESS
 
 create view v1 as select * from t2;
-call p_verify_status_increment(4, 0, 3, 0);
+call p_verify_status_increment(5, 0, 3, 0);
 SUCCESS
 
 check table t1;
@@ -886,7 +886,7 @@ call p_verify_status_increment(0, 0, 0, 0);
 SUCCESS
 
 drop view v1;
-call p_verify_status_increment(5, 0, 3, 0);
+call p_verify_status_increment(6, 0, 3, 0);
 SUCCESS
 
 #

mysql-test/r/create.result

@@ -936,7 +936,7 @@ create table t2 select sql_big_result f1,count(f2) from t1 group by f1;
 show status like 'handler_read%';
 Variable_name	Value
 Handler_read_first	1
-Handler_read_key	10
+Handler_read_key	11
 Handler_read_last	0
 Handler_read_next	2
 Handler_read_prev	0
@@ -1056,6 +1056,7 @@ select COLUMN_NAME from information_schema.columns where
 table_schema='test' order by COLUMN_NAME;
 COLUMN_NAME
 имя_поля_в_кодировке_утф8_длиной_больше_чем_45
+имя_поля_в_кодировке_утф8_длиной_больше_чем_45
 select INDEX_NAME from information_schema.statistics where
 table_schema='test' order by INDEX_NAME;
 INDEX_NAME

mysql-test/r/create_ps_protocol.result

@@ -5,7 +5,7 @@ CREATE TABLE t2 SELECT SQL_BIG_RESULT f1, COUNT(f2) FROM t1 GROUP BY f1;
 SHOW STATUS LIKE 'handler_read%';
 Variable_name	Value
 Handler_read_first	1
-Handler_read_key	11
+Handler_read_key	12
 Handler_read_last	0
 Handler_read_next	2
 Handler_read_prev	0

mysql-test/r/ctype_uca.result

@@ -1,3 +1,4 @@
+call mtr.add_suppression("Plugin \'InnoDB\'");
 DROP TABLE IF EXISTS t1;
 set names utf8;
 set collation_connection=utf8_unicode_ci;

mysql-test/r/ctype_ucs.result

@@ -1156,7 +1156,7 @@ set names latin1;
 create table t1(a blob, b text charset utf8, c text charset ucs2);
 select data_type, character_octet_length, character_maximum_length
 from information_schema.columns where table_name='t1';
-data_type	character_octet_length	character_maximum_length
+DATA_TYPE	CHARACTER_OCTET_LENGTH	CHARACTER_MAXIMUM_LENGTH
 blob	65535	65535
 text	65535	65535
 text	65535	32767

mysql-test/r/dd_is_compatibility.result

@@ -67,7 +67,7 @@ tables_priv
 # I_S view definitions.
 SHOW CREATE TABLE information_schema.tables;
 View	Create View	character_set_client	collation_connection
-TABLES	CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `information_schema`.`TABLES` AS select `cat`.`name` AS `TABLE_CATALOG`,`sch`.`name` AS `TABLE_SCHEMA`,`tbl`.`name` AS `TABLE_NAME`,`tbl`.`type` AS `TABLE_TYPE`,if((`tbl`.`type` = 'VIEW'),NULL,`tbl`.`engine`) AS `ENGINE`,if((`tbl`.`type` = 'VIEW'),NULL,10) AS `VERSION`,`tbl`.`row_format` AS `ROW_FORMAT`,`stat`.`table_rows` AS `TABLE_ROWS`,`stat`.`avg_row_length` AS `AVG_ROW_LENGTH`,`stat`.`data_length` AS `DATA_LENGTH`,`stat`.`max_data_length` AS `MAX_DATA_LENGTH`,`stat`.`index_length` AS `INDEX_LENGTH`,`stat`.`data_free` AS `DATA_FREE`,`stat`.`auto_increment` AS `AUTO_INCREMENT`,`tbl`.`created` AS `CREATE_TIME`,`stat`.`update_time` AS `UPDATE_TIME`,`stat`.`check_time` AS `CHECK_TIME`,`col`.`name` AS `TABLE_COLLATION`,`stat`.`checksum` AS `CHECKSUM`,if((`tbl`.`type` = 'VIEW'),NULL,get_dd_create_options(`tbl`.`options`,if((ifnull(`tbl`.`partition_expression`,'NOT_PART_TBL') = 'NOT_PART_TBL'),0,1))) AS `CREATE_OPTIONS`,if((`tbl`.`type` = 'VIEW'),'VIEW',`tbl`.`comment`) AS `TABLE_COMMENT` from ((((`mysql`.`tables` `tbl` join `mysql`.`schemata` `sch` on((`tbl`.`schema_id` = `sch`.`id`))) join `mysql`.`catalogs` `cat` on((`cat`.`id` = `sch`.`catalog_id`))) left join `mysql`.`collations` `col` on((`tbl`.`collation_id` = `col`.`id`))) left join `mysql`.`table_stats` `stat` on(((`tbl`.`name` = `stat`.`table_name`) and (`sch`.`name` = `stat`.`schema_name`)))) where (can_access_table(`sch`.`name`,`tbl`.`name`) and (not(`tbl`.`hidden`)))	utf8	utf8_general_ci
+TABLES	CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `information_schema`.`TABLES` AS select `cat`.`name` AS `TABLE_CATALOG`,`sch`.`name` AS `TABLE_SCHEMA`,`tbl`.`name` AS `TABLE_NAME`,`tbl`.`type` AS `TABLE_TYPE`,if((`tbl`.`type` = 'BASE TABLE'),`tbl`.`engine`,NULL) AS `ENGINE`,if((`tbl`.`type` = 'VIEW'),NULL,10) AS `VERSION`,`tbl`.`row_format` AS `ROW_FORMAT`,`stat`.`table_rows` AS `TABLE_ROWS`,`stat`.`avg_row_length` AS `AVG_ROW_LENGTH`,`stat`.`data_length` AS `DATA_LENGTH`,`stat`.`max_data_length` AS `MAX_DATA_LENGTH`,`stat`.`index_length` AS `INDEX_LENGTH`,`stat`.`data_free` AS `DATA_FREE`,`stat`.`auto_increment` AS `AUTO_INCREMENT`,`tbl`.`created` AS `CREATE_TIME`,`stat`.`update_time` AS `UPDATE_TIME`,`stat`.`check_time` AS `CHECK_TIME`,`col`.`name` AS `TABLE_COLLATION`,`stat`.`checksum` AS `CHECKSUM`,if((`tbl`.`type` = 'VIEW'),NULL,get_dd_create_options(`tbl`.`options`,if((ifnull(`tbl`.`partition_expression`,'NOT_PART_TBL') = 'NOT_PART_TBL'),0,1))) AS `CREATE_OPTIONS`,internal_get_comment_or_error(`sch`.`name`,`tbl`.`name`,`tbl`.`type`,`tbl`.`options`,`tbl`.`comment`) AS `TABLE_COMMENT` from ((((`mysql`.`tables` `tbl` join `mysql`.`schemata` `sch` on((`tbl`.`schema_id` = `sch`.`id`))) join `mysql`.`catalogs` `cat` on((`cat`.`id` = `sch`.`catalog_id`))) left join `mysql`.`collations` `col` on((`tbl`.`collation_id` = `col`.`id`))) left join `mysql`.`table_stats` `stat` on(((`tbl`.`name` = `stat`.`table_name`) and (`sch`.`name` = `stat`.`schema_name`)))) where (can_access_table(`sch`.`name`,`tbl`.`name`) and (not(`tbl`.`hidden`)))	utf8	utf8_general_ci
 #########################################################
 # Issue WL#6599/HLS/6c): Capital cased I_S table column names.
 #########################################################
@@ -222,7 +222,8 @@ count(*) = IF(@@lower_case_table_names = 0, 5, 10)
 # lctn=0 will have table_name as utf8_bin, so we do not match
 # capital information_schema name.
 SELECT COUNT(*)+IF(@@lower_case_table_names=0, 1, 0) FROM
-INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA='INFORMATION_SCHEMA';
+INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA='INFORMATION_SCHEMA' AND
+TABLE_NAME='TABLES';
 COUNT(*)+IF(@@lower_case_table_names=0, 1, 0)
 1
 #########################################################
@@ -280,7 +281,6 @@ DROP TABLE t1;
 #########################################################
 CREATE DATABASE test1;
 # Following statements should pass as INFORMATION_SCHEMA db is not used.
-# Following statements should pass as INFORMATION_SCHEMA db is not used.
 DROP DATABASE test1;
 ########################################################################
 # In 5.7 code, MDL lock on the table being created in another session is
@@ -348,3 +348,46 @@ DROP TABLE t1;
 DROP TABLE t2;
 DROP TABLE t3;
 DROP TABLE t4;
+#
+# 6. Change in view IS_UDPATABLE value in I_S.views.
+#
+CREATE TABLE t1 (c1 INT(11) DEFAULT NULL, c2 INT(11) DEFAULT NULL);
+INSERT INTO t1 VALUES(5, 5);
+CREATE VIEW v1 AS SELECT A.c1 AS c1 FROM t1 A
+WHERE EXISTS(SELECT B.c2 FROM t1 B WHERE (B.c2 = A.c1));
+# View "v1" is non-updatable but "IS_UPDATABLE" column of I_S.views used
+# show view as "updatable". Now we get correct value for is_updatable
+# column for view "v1".
+SELECT table_name, is_updatable FROM INFORMATION_SCHEMA.VIEWS
+WHERE table_name = 'v1';
+TABLE_NAME	IS_UPDATABLE
+v1	NO
+INSERT INTO v1 VALUES (10);
+UPDATE v1 SET c1=25;
+ERROR HY000: The target table v1 of the UPDATE is not updatable
+DELETE FROM v1;
+ERROR HY000: The target table v1 of the DELETE is not updatable
+# Cleanup
+DROP TABLE t1;
+DROP VIEW v1;
+########################################################################
+# In the 5.7 code while filling schema table "VIEWS", is_updatable column
+# of view is evaluated to YES/NO depending on view is mergable and view
+# has at least one updatable field in the view. 
+# Even while creating view and storing values in new DD tables, is_updatable
+# value is evaluated but there was no check to find view has at least
+# one updatable field. Without this check v1 below was evaluated to
+# updatable view instead of non-updatable.
+# Added check to find minimum one updatable field in create view code to
+# keep the behavior similar to 5.7.
+########################################################################
+CREATE TABLE t1(f1 int);
+CREATE VIEW v1 AS SELECT f1+1 AS a FROM t1;
+# With out check for minimum one updatable field, is_updatable
+# field for view v1 was evaluated to YES instead of NO here.
+SELECT table_name, is_updatable FROM INFORMATION_SCHEMA.VIEWS
+WHERE table_schema != 'sys' ORDER BY table_name;
+TABLE_NAME	IS_UPDATABLE
+v1	NO
+DROP TABLE t1;
+DROP VIEW v1;