WL#10928 - MTR: Improvements to --check-testcases option

Description:
===========
This WL focuses on improving check-testcases logic in MTR. MTR check-testcases
perform checks just before and after the test execution to ensure there are no
changes in the internal state of the Database.

Multiple scenarios were seen where the test does not perform cleanup after itself
eg - (check for persisted variables, check for leftover files in var/tmp dir) etc

Solution:
========
Below are the new checks introduced as part of WL#10928:

1. check for persisted variables.
2. check for any left behind tablespaces.
3. check for any active/open connection.
4. check for any left-over files.
5. check for installed plugins/components.
6. check for new plugin/component variables.
7. check for new UDFs.

Reviewed by:
===========
Pavan Naik <[email protected]>
Srikanth B.R <[email protected]>
Narendra Chauhan <[email protected]>

Author: Mohit Joshi

mysql-test/extra/binlog_tests/mix_innodb_myisam_binlog.test

@@ -662,6 +662,9 @@ CREATE TABLE t5 (a int, PRIMARY KEY (a)) ENGINE=InnoDB;
 drop trigger trg_del_t2;
 drop table t1,t2,t3,t4,t5;
 drop function bug27417;
+if (`SELECT @@binlog_format = 'STATEMENT' || @@binlog_format = 'MIXED'`) {
+  --remove_file $MYSQLTEST_VARDIR/tmp/mix_innodb_myisam_binlog.output
+}
 
 
 --echo end of tests

mysql-test/extra/rpl_tests/rpl_mixing_engines.test

@@ -1813,5 +1813,7 @@ if (`select @@session.binlog_direct_non_transactional_updates = 0 || @@session.b
 --echo ###################################################################################
 --echo #                                        CLEAN
 --echo ###################################################################################
+--remove_file $MYSQLTEST_VARDIR/tmp/test-nmt-master.sql
+--remove_file $MYSQLTEST_VARDIR/tmp/test-nmt-slave.sql
 --let $commands= clean
 --source extra/rpl_tests/rpl_mixing_engines.inc

mysql-test/extra/rpl_tests/rpl_replication_observers_example_plugin_server_startup.inc

@@ -71,3 +71,8 @@ DROP TABLE t1;
 
 --let $include_filename= rpl_replication_observers_example_plugin_server_startup.inc
 --source include/end_include_file.inc
+
+--echo #
+--echo # Re-installing the Plugin
+--echo #
+INSTALL PLUGIN replication_observers_example SONAME 'replication_observers_example_plugin.so';

mysql-test/extra/rpl_tests/rpl_row_UUID.test

@@ -74,4 +74,5 @@ diff_files $MYSQLTEST_VARDIR/tmp/rpl_row_UUID_master.sql $MYSQLTEST_VARDIR/tmp/r
 # Long-term "system rm" is not portable; we could live without
 # this cleanup as no other test will use these files and they'll
 # be removed at next testsuite run.
-
+--remove_file $MYSQLTEST_VARDIR/tmp/rpl_row_UUID_master.sql
+--remove_file $MYSQLTEST_VARDIR/tmp/rpl_row_UUID_slave.sql

mysql-test/extra/rpl_tests/rpl_row_blob.test

@@ -160,6 +160,8 @@ connection master;
 --exec $MYSQL_DUMP_SLAVE --compact --order-by-primary --skip-extended-insert --no-create-info test > $MYSQLTEST_VARDIR/tmp/rpl_row_blob_slave.sql
 
 diff_files $MYSQLTEST_VARDIR/tmp/rpl_row_blob_master.sql $MYSQLTEST_VARDIR/tmp/rpl_row_blob_slave.sql;
+--remove_file $MYSQLTEST_VARDIR/tmp/rpl_row_blob_master.sql
+--remove_file $MYSQLTEST_VARDIR/tmp/rpl_row_blob_slave.sql
 
 DROP TABLE IF EXISTS test.t1;
 DROP TABLE IF EXISTS test.t2;

mysql-test/extra/rpl_tests/rpl_row_sp006.test

@@ -79,6 +79,9 @@ DROP PROCEDURE p2;
 
 diff_files $MYSQLTEST_VARDIR/tmp/sp006_master.sql $MYSQLTEST_VARDIR/tmp/sp006_slave.sql;
 
+# Cleanup
+--remove_file $MYSQLTEST_VARDIR/tmp/sp006_master.sql
+--remove_file $MYSQLTEST_VARDIR/tmp/sp006_slave.sql
 --source include/sync_slave_sql_with_master.inc
 
 # End of 5.1 test case

mysql-test/include/assert_grep.inc

@@ -153,6 +153,6 @@ if ($_ag_outcome != 'assert_grep.inc ok')
   --die assert_grep.inc failed.
 }
 
-
+--remove_file $_AG_OUT
 --let $include_filename= include/assert_grep.inc [$assert_text]
 --source include/end_include_file.inc

mysql-test/include/check-testcase.test

@@ -20,13 +20,12 @@
 # no error, and a few other invariants.  So we issue SHOW SLAVE
 # STATUS, and if it returns no rows we fake the result.
 --let $tmp= 0
-if (`SELECT @@server_id <> 0`)
-{
+if (`SELECT @@server_id <> 0`) {
   --let $tmp= query_get_value(SHOW SLAVE STATUS, Slave_IO_Running, 1)
   --let $tmp= `SELECT '$tmp' = 'No such row'`
 }
-if ($tmp)
-{
+
+if ($tmp) {
   # Note: after WL#5177, fields 13-18 shall not be filtered-out.
   --echo Slave_IO_State	
   --echo Master_Host	127.0.0.1
@@ -95,6 +94,128 @@ if (!$tmp) {
   SHOW SLAVE STATUS;
 }
 
-call mtr.check_testcase();
+# There might be more than one server used by a test and check-testcase.test
+# will be run for each server. The file lists for each server needs to be
+# uniquely identified. This can be achieved by appending the @@server_id with
+# the filename.
+--let $infile = `SELECT CONCAT('$MYSQLTEST_VARDIR_ABS/file_list_', @@server_id,'.flist')`
+
+# There may be restrictions on paths usable for import/export operations
+# on a server due to the --secure-file-priv setting.
+#
+# Example 1: sys_vars.secure_file_priv2.test
+#    - The server gets started with --secure_file_priv=$SECURE_LOAD_PATH
+#      --> LOAD DATA INFILE '$MYSQLTEST_VARDIR_ABS/...' gets rejected in this case
+#          if $SECURE_LOAD_PATH <> $MYSQLTEST_VARDIR_ABS
+# Hence, we do not run check-testcase in this case.
+#
+# Example 2: auth_sec.secure_file_priv_null.test
+#    - The server gets started with --secure_file_priv=null
+# Hence, we do not run check-testcase in this case.
+#
+# Example 3: sys_vars.secure_file_priv.test
+#    - The server gets started with --secure_file_priv=''
+# Default location for import/export operation is $MYSQLTEST_VARDIR_ABS
+# Note: $MYSQLTEST_VARDIR_ABS returns the absolute path of the vardir in-case
+# symbolic links are used (e.g --mem) option.
+
+# There might be a problem with "--innodb-force-recovery > 0"
+# When --innodb-force-recovery ranges between [1,3], it prevents all
+# INSERT, UPDATE or DELETE
+# When --innodb-force-recovery=4 (or greater), it places InnoDB in
+# READONLY mode
+# Example: innodb-force-recovery-4.test
+# Hence, check-testcase will not work in these cases.
+
+--let $secure_file_priv = `SELECT @@global.secure_file_priv`
+--let $proceed_with_check_of_files = `SELECT @@global.innodb_force_recovery = 0 AND '$secure_file_priv' = '$MYSQLTEST_VARDIR_ABS/'`
+if ($proceed_with_check_of_files) {
+  # Ensure that we have no impact on the content of the binary log.
+  #
+  # Some (CREATE TABLE, ...) statements within the current routine would be
+  # usually written into the binary log. This must be avoided.
+  # Otherwise tests processing the content of the binary log like
+  # "rpl.rpl_extra_row_data" will fail.
+  SET @sql_log_bin_save = @@sql_log_bin;
+  SET SESSION sql_log_bin = 0;
+
+--write_file $infile
+# These are the files, directory etc. found in some important directories.
+EOF
+#
+
+--append_file $infile
+#---------------- MYSQLTEST_VARDIR ----------------
+EOF
+--list_files_append_file $infile $MYSQLTEST_VARDIR *
+#
+
+# MYSQLTEST_VARDIR/log contains
+#   bootstrap.log
+#   current_test
+#   <testcase>.log
+#   timer
+# We do not list the content of this directory because
+# - the name of the testcase is currently not available
+# - it's rather unusual to create files there via a test script
+#
+
+--append_file $infile
+#-------------- MYSQLTEST_VARDIR/run --------------
+EOF
+--list_files_append_file $infile $MYSQLTEST_VARDIR/run *
+#
+
+--append_file $infile
+#-------------- MYSQLTEST_VARDIR/tmp --------------
+EOF
+--list_files_append_file $infile $MYSQLTEST_VARDIR/tmp *
+#
+
+--append_file $infile
+#------------- MYSQLTEST_VARDIR/std_data ------------
+EOF
+--list_files_append_file $infile $MYSQLTEST_VARDIR/std_data *
+#
+
+  USE test;
+
+  CREATE TEMPORARY TABLE file_tbl (filename varchar(1024));
+
+  # In case of binlog_killed_simulate the LOAD DATA fails with "query got an
+  # interrupt". Important characteristics of binlog_killed_simulate is the server
+  # system variable "debug" with a content <> "".
+  # On the other hand it is quite thinkable that LOAD DATA fails because of a non
+  # foreseen weakness of the current routine.
+  # Conclusion:
+  #    - use disable abort on error
+  #    - automatic abort of the test if LOAD DATA fails and "debug" = 0
+  #    - ignoring errors if LOAD DATA fails and "debug" <> 0
+
+  --let $debug_condition = query_get_value(SHOW VARIABLES LIKE 'debug',Value, 1)
+  --let $debug_condition_check = `SELECT '$debug_condition' <> ''`
+  if ($debug_condition_check == 1) {
+    --disable_abort_on_error
+  }
+
+  --eval LOAD DATA INFILE '$infile' INTO TABLE file_tbl
+
+  if ($debug_condition_check == 1) {
+    --enable_abort_on_error
+  }
+
+  SELECT filename FROM file_tbl
+  WHERE filename NOT LIKE 'check-mysqld_%'
+  AND filename NOT LIKE 'mysql%.sock%'
+  AND filename NOT LIKE 'file_list_%.flist'
+  AND filename NOT LIKE 'mysqld%.expect'
+  AND filename NOT LIKE 'bootstrap.log';
+
+  DROP TABLE test.file_tbl;
+  SET @@sql_log_bin = @sql_log_bin_save;
+  --remove_file $infile
+
 --enable_query_log
+}
 
+call mtr.check_testcase();

mysql-test/include/check_plugin_dir.inc

@@ -61,3 +61,4 @@ EOF
 if ($skip) {
 	--skip Plugins present in different folders
 }
+--remove_file $MYSQL_TMP_DIR/skip_status.txt

mysql-test/include/escape_sql.inc

@@ -56,3 +56,5 @@ if ($escape_string != '')
 
   --let $escape_string= `SELECT LOAD_FILE('$_output_filename')`
 }
+# Cleanup
+--remove_file $_output_filename

mysql-test/include/mtr_check.sql

@@ -52,6 +52,12 @@ BEGIN
     -- Leave the objects setup in the same state
     SELECT * from performance_schema.setup_objects
       order by OBJECT_TYPE, OBJECT_SCHEMA, OBJECT_NAME;
+
+    -- Leave the prepared statement instances in the same state
+    SELECT * from performance_schema.prepared_statements_instances;
+
+    -- Leave the user defined functions in the same state
+    SELECT * from performance_schema.user_defined_functions;
   END;
   END IF;
 END||
@@ -70,32 +76,37 @@ BEGIN
   SELECT * FROM performance_schema.global_variables
     WHERE variable_name NOT IN ('timestamp', 'server_uuid',
                                 'gtid_executed', 'gtid_purged',
-                                'group_replication_group_name')
+                                'group_replication_group_name',
+                                'keyring_file_data')
+  ORDER BY VARIABLE_NAME;
+
+  -- Dump all persisted variables, those that may change.
+  SELECT * FROM performance_schema.persisted_variables
     ORDER BY VARIABLE_NAME;
 
   -- Dump all databases, there should be none
   -- except those that was created during bootstrap
   SELECT * FROM INFORMATION_SCHEMA.SCHEMATA ORDER BY SCHEMA_NAME;
 
   -- Dump all tablespaces, there should be none
-  SELECT * FROM INFORMATION_SCHEMA.FILES WHERE 
-    FILE_TYPE !='TEMPORARY' AND TABLE_SCHEMA='test' ORDER BY FILE_ID;
+  SELECT FILE_NAME, FILE_TYPE, TABLESPACE_NAME, ENGINE FROM INFORMATION_SCHEMA.FILES
+    WHERE FILE_TYPE !='TEMPORARY' ORDER BY FILE_ID;
 
   -- The test database should not contain any tables
   SELECT table_name AS tables_in_test FROM INFORMATION_SCHEMA.TABLES
     WHERE table_schema='test'
-    ORDER BY TABLE_NAME;
+      ORDER BY TABLE_NAME;
 
   -- Show "mysql" database, tables and columns
   SELECT CONCAT(table_schema, '.', table_name) AS tables_in_mysql
     FROM INFORMATION_SCHEMA.TABLES
       WHERE table_schema='mysql' AND table_name != 'ndb_apply_status'
         ORDER BY tables_in_mysql;
   SELECT CONCAT(table_schema, '.', table_name) AS columns_in_mysql,
-         column_name, ordinal_position, column_default, is_nullable,
-         data_type, character_maximum_length, character_octet_length,
-         numeric_precision, numeric_scale, character_set_name,
-         collation_name, column_type, column_key, extra, column_comment
+       column_name, ordinal_position, column_default, is_nullable,
+       data_type, character_maximum_length, character_octet_length,
+       numeric_precision, numeric_scale, character_set_name,
+       collation_name, column_type, column_key, extra, column_comment
     FROM INFORMATION_SCHEMA.COLUMNS
       WHERE table_schema='mysql' AND table_name != 'ndb_apply_status'
         ORDER BY columns_in_mysql, column_name;
@@ -110,7 +121,7 @@ BEGIN
          ACTION_REFERENCE_OLD_ROW, ACTION_REFERENCE_NEW_ROW, SQL_MODE, DEFINER CHARACTER_SET_CLIENT,
          COLLATION_CONNECTION, DATABASE_COLLATION
     FROM INFORMATION_SCHEMA.TRIGGERS
-   WHERE TRIGGER_NAME NOT IN ('gs_insert', 'ts_insert');
+      WHERE TRIGGER_NAME NOT IN ('gs_insert', 'ts_insert');
   -- Dump all created procedures, only those in the sys schema should exist
   -- do not select the CREATED or LAST_ALTERED columns however, as tests like mysqldump.test / mysql_ugprade.test update this
   SELECT SPECIFIC_NAME,ROUTINE_CATALOG,ROUTINE_SCHEMA,ROUTINE_NAME,ROUTINE_TYPE,DATA_TYPE,CHARACTER_MAXIMUM_LENGTH,
@@ -122,30 +133,56 @@ BEGIN
   -- Dump all views, only those in the sys schema should exist
   SELECT * FROM INFORMATION_SCHEMA.VIEWS
     ORDER BY TABLE_SCHEMA, TABLE_NAME;
+  -- Dump all plugins, loaded with plugin-loading options or through
+  -- INSTALL/UNINSTALL command
+  SELECT * FROM INFORMATION_SCHEMA.PLUGINS;
 
   SHOW GLOBAL STATUS LIKE 'slave_open_temp_tables';
 
+  -- Check for number of active connections before & after the test run.
+  -- It is observed that several rpl tests fails due to mismatch in Threads
+  -- count when we use SHOW GLOBAL STATUS LIKE 'Threads_Connected':
+  --  Variable_name Value
+  -- -Threads_connected    1
+  -- +Threads_connected    2
+  -- This is due to the default value of MASTER_HEARTBEAT_PERIOD which is equal
+  -- to slave_net_timeout/2 = 60secs.
+  -- Due to this, even after the slave is stopped, the master will not close
+  -- the client session until MASTER_HEARTBEAT_PERIOD is reached.
+  -- Hence, excluding the 'Binlog Dump' thread.
+  SELECT USER, HOST, DB, COMMAND, INFO FROM INFORMATION_SCHEMA.PROCESSLIST
+    WHERE COMMAND NOT IN ('Binlog Dump','Binlog Dump GTID','Sleep')
+      ORDER BY COMMAND;
+
   -- Checksum system tables to make sure they have been properly
-  -- restored after test
-  -- skip mysql.proc however, as created timestamps may have been updated by mysqldump.test / mysql_ugprade.test
-  -- the above SELECT on I_S.ROUTINES ensures consistency across runs instead
+  -- restored after test.
+  -- skip mysql.proc however, as created timestamps may have been updated by
+  -- mysqldump.test / mysql_ugprade.test the above SELECT on I_S.ROUTINES
+  -- ensures consistency across runs instead
+  -- We are skipping mysql.plugin from the checksum table list, as it does not
+  -- register plugin-loading options like (--plugin-load,--plugin-load-add ..)
+  -- instead we will use I_S.PLUGINS to ensure consistency across runs.
   checksum table
     mysql.columns_priv,
+    mysql.component,
+    mysql.default_roles,
     mysql.db,
     mysql.func,
+    mysql.global_grants,
     mysql.help_category,
     mysql.help_keyword,
     mysql.help_relation,
-    mysql.host,
+    mysql.help_topic,
     mysql.procs_priv,
+    mysql.proxies_priv,
+    mysql.role_edges,
     mysql.tables_priv,
     mysql.time_zone,
     mysql.time_zone_leap_second,
     mysql.time_zone_name,
     mysql.time_zone_transition,
     mysql.time_zone_transition_type,
-    mysql.user,
-    mysql.global_grants;
+    mysql.user;
 
 END||
 

mysql-test/include/mysqlbinlog.inc

@@ -90,7 +90,7 @@ if (!$mysqlbinlog_skip_replace)
   replace_regex
     /TIMESTAMP=[0-9]*/TIMESTAMP=#/
     /#[0-9]*[ ]*[0-9]*:[0-9]*:[0-9]* server id [0-9]*/# # server id #/
-    /SQL_LOAD_MB-[0-9]-[0-9]/SQL_LOAD_MB-#-#/
+    /SQL_LOAD_MB-[a-f0-9]-[a-f0-9]/SQL_LOAD_MB-#-#/
     /exec_time=[0-9]*/exec_time=#/
     /last_committed=[0-9]*/last_committed=#/
     /sequence_number=[0-9]*/sequence_number=#/

mysql-test/include/mysqlbinlog_have_debug.inc

@@ -25,10 +25,12 @@ close FILE_INC;
 EOF
 --source $temp_inc
 
+--remove_file $temp_out_help_file
+--remove_file $temp_inc
+--enable_query_log
+
 if (!$is_debug)
 {
   --skip mysqlbinlog needs to be debug compiled
 }
---remove_file $temp_out_help_file
---remove_file $temp_inc
---enable_query_log
+

mysql-test/include/mysqld--help.inc

@@ -80,5 +80,6 @@ perl;
   close F;
 EOF
 
+--remove_file $MYSQL_TMP_DIR/mysqld--help.txt
 --rmdir $TMP_DATADIR
 --rmdir $MYDIR

mysql-test/include/rand.inc

@@ -109,3 +109,5 @@ if ($rand_type != decide)
     --let $rand= `SELECT ($_rand_min) + ($rand * (($_rand_max) - ($_rand_min)) / (1 << 32))`
   }
 }
+--error 0,1
+--remove_file $MYSQLTEST_VARDIR/tmp/rand_seed

mysql-test/include/rpl_diff.inc

@@ -129,7 +129,7 @@ while ($_rpl_diff_servers)
   --let $_rpl_diff_first= 0
 }
 --remove_file $_rpl_diff_prev_file
-
+--remove_file $_rpl_diff_statement_file
 
 --let $include_filename= rpl_diff.inc
 --source include/end_include_file.inc