From e56fe393104960eb62043c3777ce7d21de9362f4 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 6 Jul 2021 03:00:27 +0200 Subject: [PATCH 01/22] MDEV-25978: minor post-merge fix for .result file --- mysql-test/suite/galera/r/galera_sst_rsync_logbasename.result | 1 - 1 file changed, 1 deletion(-) diff --git a/mysql-test/suite/galera/r/galera_sst_rsync_logbasename.result b/mysql-test/suite/galera/r/galera_sst_rsync_logbasename.result index dabbaeaab4f53..9b56a09d36989 100644 --- a/mysql-test/suite/galera/r/galera_sst_rsync_logbasename.result +++ b/mysql-test/suite/galera/r/galera_sst_rsync_logbasename.result @@ -172,4 +172,3 @@ COUNT(*) = 0 1 DROP TABLE t1; COMMIT; -SET AUTOCOMMIT=ON; From 78735dcaf757cd71c8f0ff3d21071b0f89018150 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Thu, 8 Jul 2021 17:47:17 -0700 Subject: [PATCH 02/22] MDEV-26108 Crash with query referencing twice CTE that uses embedded recursive CTE This bug could affect queries that had at least two references to a CTE that used an embedded recursive CTE. Starting from version 10.4 some code in With_element::clone_parsed_spec() that assumed a certain order of selects after parsing the specification of a CTE became not valid anymore. It could lead to global select lists where some selects were missing. If a missing CTE happened to belong to the recursive part of a recursive CTE some recursive table references were not set as references to materialized derived tables and this caused a crash of the server. Approved by Oleksandr Byelkin --- mysql-test/main/cte_nonrecursive.result | 2 +- mysql-test/main/cte_recursive.result | 19 +++++++++++++++++++ mysql-test/main/cte_recursive.test | 17 +++++++++++++++++ sql/sql_cte.cc | 10 +++++++--- 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/mysql-test/main/cte_nonrecursive.result b/mysql-test/main/cte_nonrecursive.result index 040afdf7ff434..4cd466ac35099 100644 --- a/mysql-test/main/cte_nonrecursive.result +++ b/mysql-test/main/cte_nonrecursive.result @@ -1126,7 +1126,7 @@ NULL UNION RESULT ALL NULL NULL NULL NULL NULL NULL NULL UNION RESULT ALL NULL NULL NULL NULL NULL NULL NULL UNION RESULT ALL NULL NULL NULL NULL NULL NULL Warnings: -Note 1003 with cte_e as (with cte_o as (with cte_i as (select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` < 7)select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` > 1)select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` < 3 and `test`.`t1`.`a` > 1 and `test`.`t1`.`a` < 7 and `test`.`t1`.`a` > 1 union select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` > 4 and `test`.`t1`.`a` > 1 and `test`.`t1`.`a` < 7 and `test`.`t1`.`a` > 1)select `cte_e1`.`a` AS `a` from `cte_e` `cte_e1` where `cte_e1`.`a` > 1 union select `cte_e2`.`a` AS `a` from `cte_e` `cte_e2` +Note 1003 with cte_e as (with cte_o as (with cte_i as (/* select#2 */ select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` < 7)/* select#3 */ select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` > 1)/* select#4 */ select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` < 3 and `test`.`t1`.`a` > 1 and `test`.`t1`.`a` < 7 and `test`.`t1`.`a` > 1 union /* select#5 */ select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` > 4 and `test`.`t1`.`a` > 1 and `test`.`t1`.`a` < 7 and `test`.`t1`.`a` > 1)/* select#1 */ select `cte_e1`.`a` AS `a` from `cte_e` `cte_e1` where `cte_e1`.`a` > 1 union /* select#6 */ select `cte_e2`.`a` AS `a` from `cte_e` `cte_e2` drop table t1; # # MDEV-13753: embedded CTE in a VIEW created in prepared statement diff --git a/mysql-test/main/cte_recursive.result b/mysql-test/main/cte_recursive.result index 6f30de39b46c7..74b450ff89035 100644 --- a/mysql-test/main/cte_recursive.result +++ b/mysql-test/main/cte_recursive.result @@ -4791,3 +4791,22 @@ a NULL DROP TABLE t1; # End of 10.3 tests +# +# MDEV-26108: Recursive CTE embedded into another CTE which is used twice +# +create table t1 (a int); +insert into t1 values (5), (7); +with cte_e as ( +with recursive cte_r as ( +select a from t1 union select a+1 as a from cte_r r where a < 10 +) select * from cte_r +) select * from cte_e s1, cte_e s2 where s1.a=s2.a; +a a +5 5 +7 7 +6 6 +8 8 +9 9 +10 10 +drop table t1; +# End of 10.4 tests diff --git a/mysql-test/main/cte_recursive.test b/mysql-test/main/cte_recursive.test index c3537e5bd0c9f..3b140b3cc6c62 100644 --- a/mysql-test/main/cte_recursive.test +++ b/mysql-test/main/cte_recursive.test @@ -3087,3 +3087,20 @@ SELECT * FROM cte; DROP TABLE t1; --echo # End of 10.3 tests + +--echo # +--echo # MDEV-26108: Recursive CTE embedded into another CTE which is used twice +--echo # + +create table t1 (a int); +insert into t1 values (5), (7); + +with cte_e as ( + with recursive cte_r as ( + select a from t1 union select a+1 as a from cte_r r where a < 10 + ) select * from cte_r +) select * from cte_e s1, cte_e s2 where s1.a=s2.a; + +drop table t1; + +--echo # End of 10.4 tests diff --git a/sql/sql_cte.cc b/sql/sql_cte.cc index dfcb4e18772bc..c5dcc15f0186c 100644 --- a/sql/sql_cte.cc +++ b/sql/sql_cte.cc @@ -1038,6 +1038,7 @@ st_select_lex_unit *With_element::clone_parsed_spec(LEX *old_lex, bool parse_status= false; st_select_lex *with_select; + st_select_lex *last_clone_select; char save_end= unparsed_spec.str[unparsed_spec.length]; ((char*) &unparsed_spec.str[unparsed_spec.length])[0]= '\0'; @@ -1124,11 +1125,14 @@ st_select_lex_unit *With_element::clone_parsed_spec(LEX *old_lex, lex->unit.include_down(with_table->select_lex); lex->unit.set_slave(with_select); lex->unit.cloned_from= spec; + last_clone_select= lex->all_selects_list; + while (last_clone_select->next_select_in_list()) + last_clone_select= last_clone_select->next_select_in_list(); old_lex->all_selects_list= (st_select_lex*) (lex->all_selects_list-> - insert_chain_before( - (st_select_lex_node **) &(old_lex->all_selects_list), - with_select)); + insert_chain_before( + (st_select_lex_node **) &(old_lex->all_selects_list), + last_clone_select)); /* Now all references to the CTE defined outside of the cloned specification From e3814a74eee4f47b5d58997f90c8ee9742452681 Mon Sep 17 00:00:00 2001 From: Nayuta Yanagisawa Date: Wed, 14 Jul 2021 10:17:54 +0000 Subject: [PATCH 03/22] MDEV-26139 Spider crashes with segmentation fault (signal 11) on CREATE TABLE when COMMENT does not contain embedded double quotes The root cause of the bug MDEV-26139 is the lack of NULL checking on the variable `dq`. Comments on if (dq && (!sq || sq > dq)) {...} else {...}: * The if block corresponds to the case where parameters are quoted by double quotes. In that case, a single quote doesn't appear at all or only appears in the middle of double quotes. * The else block corresponds to the case where parameters are quoted by single quotes. In that case, a double quote doesn't appear at all or only appears in the middle of single quotes. * If the program reaches the if-else statement, `sq || dq` holds. Thus, the negation of `dq && (!sq || sq > dq)` is equivalent to `sq && (!dq || sq <= dq)`. --- storage/spider/mysql-test/spider/r/basic_sql.result | 6 ++++++ storage/spider/mysql-test/spider/t/basic_sql.test | 7 +++++++ storage/spider/spd_table.h | 5 +++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/storage/spider/mysql-test/spider/r/basic_sql.result b/storage/spider/mysql-test/spider/r/basic_sql.result index ba904b5f57744..2443f3488bddf 100644 --- a/storage/spider/mysql-test/spider/r/basic_sql.result +++ b/storage/spider/mysql-test/spider/r/basic_sql.result @@ -721,6 +721,12 @@ connection master_1; create table t2345678911234567892123456789312345678941234567895123234234(id int) ENGINE=SPIDER COMMENT='host "192.168.21.1", user "spider", password "password", database "test32738123123123"'; drop table t2345678911234567892123456789312345678941234567895123234234; +# +# MDEV-26139 Spider crashes with segmentation fault (signal 11) on CREATE TABLE when COMMENT does not contain embedded double quotes +# +create table mdev_26139 (id int) ENGINE=SPIDER +COMMENT="host '192.168.21.1', user 'spider', password 'password', database 'test'"; +drop table mdev_26139; deinit connection master_1; diff --git a/storage/spider/mysql-test/spider/t/basic_sql.test b/storage/spider/mysql-test/spider/t/basic_sql.test index a3184a14beb1f..1298b10f19a60 100644 --- a/storage/spider/mysql-test/spider/t/basic_sql.test +++ b/storage/spider/mysql-test/spider/t/basic_sql.test @@ -2682,6 +2682,13 @@ create table t2345678911234567892123456789312345678941234567895123234234(id int) COMMENT='host "192.168.21.1", user "spider", password "password", database "test32738123123123"'; drop table t2345678911234567892123456789312345678941234567895123234234; +--echo # +--echo # MDEV-26139 Spider crashes with segmentation fault (signal 11) on CREATE TABLE when COMMENT does not contain embedded double quotes +--echo # +create table mdev_26139 (id int) ENGINE=SPIDER + COMMENT="host '192.168.21.1', user 'spider', password 'password', database 'test'"; +drop table mdev_26139; + --echo --echo deinit --disable_warnings diff --git a/storage/spider/spd_table.h b/storage/spider/spd_table.h index 6aaac2046e442..063f459ae8d62 100644 --- a/storage/spider/spd_table.h +++ b/storage/spider/spd_table.h @@ -189,7 +189,8 @@ typedef struct st_spider_param_string_parse { DBUG_RETURN(print_param_error()); } - else if (!sq || sq > dq) + + if (dq && (!sq || sq > dq)) { while (1) { @@ -227,7 +228,7 @@ typedef struct st_spider_param_string_parse } } } - else + else /* sq && (!dq || sq <= dq) */ { while (1) { From bd3ac6758a9615e9a5d7d45a87dde8b448cb4a86 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 10 Jul 2021 12:42:53 +0200 Subject: [PATCH 04/22] fix perfschema.sizing_* tests to run still cannot be enabled permanently, but at least they could be run manually, if needed --- .../include/default_mysqld_autosize.cnf | 53 +++++++++++++++++++ .../include/have_aligned_memory.inc | 5 +- .../suite/perfschema/include/sizing_auto.inc | 4 +- .../suite/perfschema/r/sizing_default.result | 41 +++++++------- .../suite/perfschema/r/sizing_high.result | 39 +++++++------- .../suite/perfschema/r/sizing_low.result | 43 ++++++++------- .../suite/perfschema/r/sizing_med.result | 25 +++++---- .../suite/perfschema/r/sizing_off.result | 4 +- .../suite/perfschema/t/sizing_default.cnf | 4 +- .../suite/perfschema/t/sizing_default.test | 22 ++++---- mysql-test/suite/perfschema/t/sizing_high.cnf | 3 +- mysql-test/suite/perfschema/t/sizing_low.cnf | 3 +- mysql-test/suite/perfschema/t/sizing_med.cnf | 3 +- mysql-test/suite/perfschema/t/sizing_off.cnf | 2 +- 14 files changed, 158 insertions(+), 93 deletions(-) create mode 100644 mysql-test/suite/perfschema/include/default_mysqld_autosize.cnf diff --git a/mysql-test/suite/perfschema/include/default_mysqld_autosize.cnf b/mysql-test/suite/perfschema/include/default_mysqld_autosize.cnf new file mode 100644 index 0000000000000..eee52ede869da --- /dev/null +++ b/mysql-test/suite/perfschema/include/default_mysqld_autosize.cnf @@ -0,0 +1,53 @@ + +# Default values that applies to all MySQL Servers +[mysqld] +local-infile +character-set-server= latin1 +default-storage-engine=myisam + +# Increase default connect_timeout to avoid intermittent +# disconnects when test servers are put under load see BUG#28359 +connect-timeout= 60 + +log-bin-trust-function-creators=1 +key_buffer_size= 1M +sort_buffer_size= 256K +max_heap_table_size= 1M + +loose-innodb_data_file_path= ibdata1:10M:autoextend +loose-innodb_buffer_pool_size= 8M +loose-innodb_lru_scan_depth= 100 +loose-innodb_write_io_threads= 2 +loose-innodb_read_io_threads= 2 +loose-innodb_log_buffer_size= 1M +loose-innodb_log_file_size= 5M +loose-innodb_log_files_in_group= 2 + +slave-net-timeout=120 + +log-bin=mysqld-bin + +# No performance schema sizing provided + +# Disable everything, we only need the sizing data, +# and also need a stable output for show engine performance_schema status +loose-performance-schema-consumer-global-instrumentation=OFF + +loose-performance-schema-instrument='%=ON' + +loose-performance-schema-consumer-events-stages-current=ON +loose-performance-schema-consumer-events-stages-history=ON +loose-performance-schema-consumer-events-stages-history-long=ON +loose-performance-schema-consumer-events-statements-current=ON +loose-performance-schema-consumer-events-statements-history=ON +loose-performance-schema-consumer-events-statements-history-long=ON +loose-performance-schema-consumer-events-transactions-current=ON +loose-performance-schema-consumer-events-transactions-history=ON +loose-performance-schema-consumer-events-transactions-history-long=ON +loose-performance-schema-consumer-events-waits-current=ON +loose-performance-schema-consumer-events-waits-history=ON +loose-performance-schema-consumer-events-waits-history-long=ON +loose-performance-schema-consumer-thread-instrumentation=ON + +binlog-direct-non-transactional-updates + diff --git a/mysql-test/suite/perfschema/include/have_aligned_memory.inc b/mysql-test/suite/perfschema/include/have_aligned_memory.inc index 9638cbe1da46e..d420f0e055ad3 100644 --- a/mysql-test/suite/perfschema/include/have_aligned_memory.inc +++ b/mysql-test/suite/perfschema/include/have_aligned_memory.inc @@ -4,10 +4,7 @@ # For tests sensitive to the internal sizes (show engine performance_schema # status), make sure we use a platform with aligned memory. ---disable_query_log -let $aligned = `SELECT count(*) from performance_schema.session_connect_attrs where PROCESSLIST_ID = connection_id() and ATTR_NAME = '_os' and ATTR_VALUE in ('Linux', 'Windows')`; -if (!$aligned) +if (`SELECT count(*)=0 from performance_schema.session_connect_attrs where PROCESSLIST_ID = connection_id() and ATTR_NAME = '_os' and ATTR_VALUE in ('Linux', 'Windows')`) { skip Need a platform with aligned memory; } ---enable_query_log diff --git a/mysql-test/suite/perfschema/include/sizing_auto.inc b/mysql-test/suite/perfschema/include/sizing_auto.inc index 3bb4db2276fc3..6cb077e3cf769 100644 --- a/mysql-test/suite/perfschema/include/sizing_auto.inc +++ b/mysql-test/suite/perfschema/include/sizing_auto.inc @@ -3,7 +3,7 @@ show variables like "table_definition_cache"; show variables like "table_open_cache"; show variables like "max_connections"; # open_files_limit depends on OS configuration (ulimit -n) -#show variables like "open_files_limit"; +show variables like "open_files_limit"; show variables where `Variable_name` != "performance_schema_max_statement_classes" and `Variable_name` like "performance_schema%"; @@ -17,7 +17,7 @@ show status like "%performance_schema%"; # is very dependent on the platform, # so it is not printed here to ensure stability of the .results files. # To troubleshoot the performance schema memory consumption at different -# configuration settings, comment the following line. +# configuration settings, uncomment the following line. # Debug only: # show engine performance_schema status; diff --git a/mysql-test/suite/perfschema/r/sizing_default.result b/mysql-test/suite/perfschema/r/sizing_default.result index 07fbf35619be8..ea611315be3c9 100644 --- a/mysql-test/suite/perfschema/r/sizing_default.result +++ b/mysql-test/suite/perfschema/r/sizing_default.result @@ -1,43 +1,46 @@ show variables like "table_definition_cache"; Variable_name Value -table_definition_cache 1400 +table_definition_cache 400 show variables like "table_open_cache"; Variable_name Value -table_open_cache 2000 +table_open_cache 421 show variables like "max_connections"; Variable_name Value max_connections 151 +show variables like "open_files_limit"; +Variable_name Value +open_files_limit 1024 show variables where `Variable_name` != "performance_schema_max_statement_classes" and `Variable_name` like "performance_schema%"; Variable_name Value performance_schema ON performance_schema_accounts_size 100 -performance_schema_digests_size 10000 -performance_schema_events_stages_history_long_size 10000 -performance_schema_events_stages_history_size 10 -performance_schema_events_statements_history_long_size 10000 -performance_schema_events_statements_history_size 10 -performance_schema_events_waits_history_long_size 10000 -performance_schema_events_waits_history_size 10 +performance_schema_digests_size 5000 +performance_schema_events_stages_history_long_size 1000 +performance_schema_events_stages_history_size 20 +performance_schema_events_statements_history_long_size 1000 +performance_schema_events_statements_history_size 20 +performance_schema_events_waits_history_long_size 1000 +performance_schema_events_waits_history_size 20 performance_schema_hosts_size 100 -performance_schema_max_cond_classes 80 -performance_schema_max_cond_instances 3504 +performance_schema_max_cond_classes 90 +performance_schema_max_cond_instances 1360 performance_schema_max_digest_length 1024 performance_schema_max_file_classes 50 performance_schema_max_file_handles 32768 -performance_schema_max_file_instances 7693 +performance_schema_max_file_instances 2500 performance_schema_max_mutex_classes 200 -performance_schema_max_mutex_instances 15906 +performance_schema_max_mutex_instances 5648 performance_schema_max_rwlock_classes 40 -performance_schema_max_rwlock_instances 9102 +performance_schema_max_rwlock_instances 3073 performance_schema_max_socket_classes 10 -performance_schema_max_socket_instances 322 -performance_schema_max_stage_classes\t160 -performance_schema_max_table_handles 4000 -performance_schema_max_table_instances 12500 +performance_schema_max_socket_instances 230 +performance_schema_max_stage_classes 160 +performance_schema_max_table_handles 2858 +performance_schema_max_table_instances 667 performance_schema_max_thread_classes 50 -performance_schema_max_thread_instances 402 +performance_schema_max_thread_instances 288 performance_schema_session_connect_attrs_size 512 performance_schema_setup_actors_size 100 performance_schema_setup_objects_size 100 diff --git a/mysql-test/suite/perfschema/r/sizing_high.result b/mysql-test/suite/perfschema/r/sizing_high.result index 1cf84a14d22b0..1f63ac6015a6a 100644 --- a/mysql-test/suite/perfschema/r/sizing_high.result +++ b/mysql-test/suite/perfschema/r/sizing_high.result @@ -3,41 +3,44 @@ Variable_name Value table_definition_cache 5000 show variables like "table_open_cache"; Variable_name Value -table_open_cache 400 +table_open_cache 397 show variables like "max_connections"; Variable_name Value max_connections 200 +show variables like "open_files_limit"; +Variable_name Value +open_files_limit 1024 show variables where `Variable_name` != "performance_schema_max_statement_classes" and `Variable_name` like "performance_schema%"; Variable_name Value performance_schema ON performance_schema_accounts_size 100 -performance_schema_digests_size 10000 -performance_schema_events_stages_history_long_size 10000 -performance_schema_events_stages_history_size 10 -performance_schema_events_statements_history_long_size 10000 -performance_schema_events_statements_history_size 10 -performance_schema_events_waits_history_long_size 10000 -performance_schema_events_waits_history_size 10 +performance_schema_digests_size 5000 +performance_schema_events_stages_history_long_size 1000 +performance_schema_events_stages_history_size 20 +performance_schema_events_statements_history_long_size 1000 +performance_schema_events_statements_history_size 20 +performance_schema_events_waits_history_long_size 1000 +performance_schema_events_waits_history_size 20 performance_schema_hosts_size 100 -performance_schema_max_cond_classes 80 -performance_schema_max_cond_instances 10900 +performance_schema_max_cond_classes 90 +performance_schema_max_cond_instances 1500 performance_schema_max_digest_length 1024 performance_schema_max_file_classes 50 performance_schema_max_file_handles 32768 -performance_schema_max_file_instances 23385 +performance_schema_max_file_instances 2500 performance_schema_max_mutex_classes 200 -performance_schema_max_mutex_instances 52200 +performance_schema_max_mutex_instances 5858 performance_schema_max_rwlock_classes 40 -performance_schema_max_rwlock_instances 30800 +performance_schema_max_rwlock_instances 3143 performance_schema_max_socket_classes 10 -performance_schema_max_socket_instances 420 -performance_schema_max_stage_classes\t160 -performance_schema_max_table_handles 800 -performance_schema_max_table_instances 12500 +performance_schema_max_socket_instances 300 +performance_schema_max_stage_classes 160 +performance_schema_max_table_handles 2858 +performance_schema_max_table_instances 667 performance_schema_max_thread_classes 50 -performance_schema_max_thread_instances 500 +performance_schema_max_thread_instances 358 performance_schema_session_connect_attrs_size 512 performance_schema_setup_actors_size 100 performance_schema_setup_objects_size 100 diff --git a/mysql-test/suite/perfschema/r/sizing_low.result b/mysql-test/suite/perfschema/r/sizing_low.result index 4569ebd7a5e8e..d6d6baeca9a32 100644 --- a/mysql-test/suite/perfschema/r/sizing_low.result +++ b/mysql-test/suite/perfschema/r/sizing_low.result @@ -7,41 +7,44 @@ table_open_cache 100 show variables like "max_connections"; Variable_name Value max_connections 50 +show variables like "open_files_limit"; +Variable_name Value +open_files_limit 1024 show variables where `Variable_name` != "performance_schema_max_statement_classes" and `Variable_name` like "performance_schema%"; Variable_name Value performance_schema ON -performance_schema_accounts_size 10 -performance_schema_digests_size 1000 -performance_schema_events_stages_history_long_size 100 -performance_schema_events_stages_history_size 5 -performance_schema_events_statements_history_long_size 100 -performance_schema_events_statements_history_size 5 -performance_schema_events_waits_history_long_size 100 -performance_schema_events_waits_history_size 5 -performance_schema_hosts_size 20 -performance_schema_max_cond_classes 80 -performance_schema_max_cond_instances 612 +performance_schema_accounts_size 100 +performance_schema_digests_size 5000 +performance_schema_events_stages_history_long_size 1000 +performance_schema_events_stages_history_size 20 +performance_schema_events_statements_history_long_size 1000 +performance_schema_events_statements_history_size 20 +performance_schema_events_waits_history_long_size 1000 +performance_schema_events_waits_history_size 20 +performance_schema_hosts_size 100 +performance_schema_max_cond_classes 90 +performance_schema_max_cond_instances 1072 performance_schema_max_digest_length 1024 performance_schema_max_file_classes 50 performance_schema_max_file_handles 32768 -performance_schema_max_file_instances 1556 +performance_schema_max_file_instances 2500 performance_schema_max_mutex_classes 200 -performance_schema_max_mutex_instances 2945 +performance_schema_max_mutex_instances 5215 performance_schema_max_rwlock_classes 40 -performance_schema_max_rwlock_instances 1612 +performance_schema_max_rwlock_instances 2929 performance_schema_max_socket_classes 10 -performance_schema_max_socket_instances 67 -performance_schema_max_stage_classes\t160 -performance_schema_max_table_handles 112 -performance_schema_max_table_instances 445 +performance_schema_max_socket_instances 86 +performance_schema_max_stage_classes 160 +performance_schema_max_table_handles 2858 +performance_schema_max_table_instances 667 performance_schema_max_thread_classes 50 -performance_schema_max_thread_instances 112 +performance_schema_max_thread_instances 143 performance_schema_session_connect_attrs_size 512 performance_schema_setup_actors_size 100 performance_schema_setup_objects_size 100 -performance_schema_users_size 5 +performance_schema_users_size 100 show status like "%performance_schema%"; Variable_name Value Performance_schema_accounts_lost 0 diff --git a/mysql-test/suite/perfschema/r/sizing_med.result b/mysql-test/suite/perfschema/r/sizing_med.result index 24fba02d16bc1..f662809b5cdfc 100644 --- a/mysql-test/suite/perfschema/r/sizing_med.result +++ b/mysql-test/suite/perfschema/r/sizing_med.result @@ -7,6 +7,9 @@ table_open_cache 401 show variables like "max_connections"; Variable_name Value max_connections 152 +show variables like "open_files_limit"; +Variable_name Value +open_files_limit 1024 show variables where `Variable_name` != "performance_schema_max_statement_classes" and `Variable_name` like "performance_schema%"; @@ -15,27 +18,27 @@ performance_schema ON performance_schema_accounts_size 100 performance_schema_digests_size 5000 performance_schema_events_stages_history_long_size 1000 -performance_schema_events_stages_history_size 10 +performance_schema_events_stages_history_size 20 performance_schema_events_statements_history_long_size 1000 -performance_schema_events_statements_history_size 10 +performance_schema_events_statements_history_size 20 performance_schema_events_waits_history_long_size 1000 -performance_schema_events_waits_history_size 10 +performance_schema_events_waits_history_size 20 performance_schema_hosts_size 100 -performance_schema_max_cond_classes 80 -performance_schema_max_cond_instances 1079 +performance_schema_max_cond_classes 90 +performance_schema_max_cond_instances 1363 performance_schema_max_digest_length 1024 performance_schema_max_file_classes 50 performance_schema_max_file_handles 32768 -performance_schema_max_file_instances 1754 +performance_schema_max_file_instances 2500 performance_schema_max_mutex_classes 200 -performance_schema_max_mutex_instances 4230 +performance_schema_max_mutex_instances 5652 performance_schema_max_rwlock_classes 40 -performance_schema_max_rwlock_instances 2222 +performance_schema_max_rwlock_instances 3075 performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 232 -performance_schema_max_stage_classes\t160 -performance_schema_max_table_handles 573 -performance_schema_max_table_instances 556 +performance_schema_max_stage_classes 160 +performance_schema_max_table_handles 2858 +performance_schema_max_table_instances 667 performance_schema_max_thread_classes 50 performance_schema_max_thread_instances 289 performance_schema_session_connect_attrs_size 512 diff --git a/mysql-test/suite/perfschema/r/sizing_off.result b/mysql-test/suite/perfschema/r/sizing_off.result index e49a9824f4241..d3c0a0de0a126 100644 --- a/mysql-test/suite/perfschema/r/sizing_off.result +++ b/mysql-test/suite/perfschema/r/sizing_off.result @@ -12,7 +12,7 @@ performance_schema_events_statements_history_size -1 performance_schema_events_waits_history_long_size -1 performance_schema_events_waits_history_size -1 performance_schema_hosts_size -1 -performance_schema_max_cond_classes 80 +performance_schema_max_cond_classes 90 performance_schema_max_cond_instances -1 performance_schema_max_digest_length 1024 performance_schema_max_file_classes 50 @@ -24,7 +24,7 @@ performance_schema_max_rwlock_classes 40 performance_schema_max_rwlock_instances -1 performance_schema_max_socket_classes 10 performance_schema_max_socket_instances -1 -performance_schema_max_stage_classes\t160 +performance_schema_max_stage_classes 160 performance_schema_max_table_handles -1 performance_schema_max_table_instances -1 performance_schema_max_thread_classes 50 diff --git a/mysql-test/suite/perfschema/t/sizing_default.cnf b/mysql-test/suite/perfschema/t/sizing_default.cnf index 6a92999832996..0515832d41c1b 100644 --- a/mysql-test/suite/perfschema/t/sizing_default.cnf +++ b/mysql-test/suite/perfschema/t/sizing_default.cnf @@ -1,5 +1,5 @@ -!include include/default_mysqld_autosize.cnf +!include suite/perfschema/include/default_mysqld_autosize.cnf [mysqld.1] @@ -14,6 +14,8 @@ # Automated sizing for everything +loose-enable-performance-schema + loose-performance-schema-accounts-size=-1 loose-performance-schema-digests-size=-1 loose-performance-schema-hosts-size=-1 diff --git a/mysql-test/suite/perfschema/t/sizing_default.test b/mysql-test/suite/perfschema/t/sizing_default.test index d5fb8be9f77c3..f8086cc7afdde 100644 --- a/mysql-test/suite/perfschema/t/sizing_default.test +++ b/mysql-test/suite/perfschema/t/sizing_default.test @@ -5,27 +5,25 @@ --source include/not_valgrind.inc --source ../include/have_aligned_memory.inc +#SELECT @@open_files_limit, @@table_open_cache, @@table_definition_cache, @@max_connections; +#exit; + # Skip test if not defaults is used. -let $max_open_files_limit= `SELECT @@open_files_limit < 5000`; -if ($max_open_files_limit) +if (`SELECT @@open_files_limit < 1024`) { - skip Need open_files_limit to be at least 5000; + skip Need open_files_limit to be at least 1024; } -let $max_table_open_cache= `SELECT @@table_open_cache != 2000`; -if ($max_table_open_cache) +if (`SELECT @@table_open_cache != 421`) { - skip Need table_open_cache to be exactly 2000; + skip Need table_open_cache to be exactly 421; } -let $max_table_definition_cache= `SELECT @@table_definition_cache != 1400`; -if ($max_table_definition_cache) +if (`SELECT @@table_definition_cache != 400`) { - skip Need table_definition_cache to be exactly 1400; + skip Need table_definition_cache to be exactly 400; } -let $max_connections= `SELECT @@max_connections != 151`; -if ($max_connections) +if (`SELECT @@max_connections != 151`) { skip Need max_connections to be exactly 151; } --source ../include/sizing_auto.inc - diff --git a/mysql-test/suite/perfschema/t/sizing_high.cnf b/mysql-test/suite/perfschema/t/sizing_high.cnf index 8445ff609284c..519ad79f3adcf 100644 --- a/mysql-test/suite/perfschema/t/sizing_high.cnf +++ b/mysql-test/suite/perfschema/t/sizing_high.cnf @@ -1,5 +1,5 @@ -!include include/default_mysqld_autosize.cnf +!include suite/perfschema/include/default_mysqld_autosize.cnf [mysqld.1] @@ -32,6 +32,7 @@ open_files_limit=1024 # max_connections*5 = 200*5 = 1000 # Automated sizing for everything +loose-enable-performance-schema loose-performance-schema-accounts-size=-1 loose-performance-schema-digests-size=-1 diff --git a/mysql-test/suite/perfschema/t/sizing_low.cnf b/mysql-test/suite/perfschema/t/sizing_low.cnf index 54c881830cf73..79e891bda60eb 100644 --- a/mysql-test/suite/perfschema/t/sizing_low.cnf +++ b/mysql-test/suite/perfschema/t/sizing_low.cnf @@ -1,5 +1,5 @@ -!include include/default_mysqld_autosize.cnf +!include suite/perfschema/include/default_mysqld_autosize.cnf [mysqld.1] @@ -9,6 +9,7 @@ max_connections=50 open_files_limit=1024 # Automated sizing for everything +loose-enable-performance-schema loose-performance-schema-accounts-size=-1 loose-performance-schema-digests-size=-1 diff --git a/mysql-test/suite/perfschema/t/sizing_med.cnf b/mysql-test/suite/perfschema/t/sizing_med.cnf index 8aff531ea6993..4a113272e3889 100644 --- a/mysql-test/suite/perfschema/t/sizing_med.cnf +++ b/mysql-test/suite/perfschema/t/sizing_med.cnf @@ -1,5 +1,5 @@ -!include include/default_mysqld_autosize.cnf +!include suite/perfschema/include/default_mysqld_autosize.cnf [mysqld.1] @@ -9,6 +9,7 @@ max_connections=152 open_files_limit=1024 # Automated sizing for everything +loose-enable-performance-schema loose-performance-schema-accounts-size=-1 loose-performance-schema-digests-size=-1 diff --git a/mysql-test/suite/perfschema/t/sizing_off.cnf b/mysql-test/suite/perfschema/t/sizing_off.cnf index 24ca58bbb0bcc..80c00494791f9 100644 --- a/mysql-test/suite/perfschema/t/sizing_off.cnf +++ b/mysql-test/suite/perfschema/t/sizing_off.cnf @@ -1,5 +1,5 @@ -!include include/default_mysqld_autosize.cnf +!include suite/perfschema/include/default_mysqld_autosize.cnf [mysqld.1] From 7af5d96f002fedf873c6274b0a919e7fee2af805 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 16 Jul 2021 19:28:29 +0200 Subject: [PATCH 05/22] MDEV-22221: MariaDB with WolfSSL doesn't support AES-GCM cipher for SSL commit the forgotten result file followup for b81803f0657 --- mysql-test/main/wolfssl.result | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 mysql-test/main/wolfssl.result diff --git a/mysql-test/main/wolfssl.result b/mysql-test/main/wolfssl.result new file mode 100644 index 0000000000000..88df540ca95c5 --- /dev/null +++ b/mysql-test/main/wolfssl.result @@ -0,0 +1,3 @@ +SELECT @@ssl_cipher; +@@ssl_cipher +ECDHE-RSA-AES256-GCM-SHA384 From fa45400d77b07530da87e5a5cd92992b886a2f88 Mon Sep 17 00:00:00 2001 From: Dmitry Shulga Date: Mon, 19 Jul 2021 17:16:54 +0700 Subject: [PATCH 06/22] MDEV-26149: The test main.fetch_first fails in case it is run in PS mode The root cause of test failure is that on optimization of the statement clause 'order by (select 1)' in outer select is done incorrect in case the statement run in PS mode. --- mysql-test/main/fetch_first.test | 4 ---- sql/item.h | 9 ++++++++ sql/item_subselect.cc | 39 ++++++++++++++++++++++---------- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/mysql-test/main/fetch_first.test b/mysql-test/main/fetch_first.test index 8a5cb2c8e7352..7c41b922978f9 100644 --- a/mysql-test/main/fetch_first.test +++ b/mysql-test/main/fetch_first.test @@ -1,7 +1,3 @@ -if (`SELECT $PS_PROTOCOL != 0`) -{ - --skip Test temporarily disabled for ps-protocol -} --echo # --echo # The following entries are meant for testing the parser, ensuring --echo # the right values are passed down to the executor, for all possible diff --git a/sql/item.h b/sql/item.h index 8ab4c8a3bd1f0..66e2a583b9865 100644 --- a/sql/item.h +++ b/sql/item.h @@ -1818,6 +1818,14 @@ class Item :public Value_source, their method implementations typically have DBUG_ASSERT(0). */ virtual bool is_evaluable_expression() const { return true; } + + /** + * Check whether the item is a parameter ('?') of stored routine. + * Default implementation returns false. Method is overridden in the class + * Item_param where it returns true. + */ + virtual bool is_stored_routine_parameter() const { return false; } + bool check_is_evaluable_expression_or_error() { if (is_evaluable_expression()) @@ -4281,6 +4289,7 @@ class Item_param :public Item_basic_value, return state == SHORT_DATA_VALUE && value.type_handler()->cmp_type() == INT_RESULT; } + bool is_stored_routine_parameter() const override { return true; } /* This method is used to make a copy of a basic constant item when propagating constants in the optimizer. The reason to create a new diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index 381f015ca9618..3e2ea561dce53 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -1273,22 +1273,37 @@ Item_singlerow_subselect::select_transformer(JOIN *join) Query_arena *arena, backup; arena= thd->activate_stmt_arena_if_needed(&backup); + auto need_to_pull_out_item = [](enum_parsing_place context_analysis_place, + Item *item) { + return + !item->with_sum_func() && + /* + We can't change name of Item_field or Item_ref, because it will + prevent its correct resolving, but we should save name of + removed item => we do not make optimization if top item of + list is field or reference. + TODO: solve above problem + */ + item->type() != FIELD_ITEM && item->type() != REF_ITEM && + /* + The item can be pulled out to upper level in case it doesn't represent + the constant in the clause 'ORDER/GROUP BY (constant)'. + */ + !((item->is_order_clause_position() || + item->is_stored_routine_parameter()) && + (context_analysis_place == IN_ORDER_BY || + context_analysis_place == IN_GROUP_BY) + ); + }; + if (!select_lex->master_unit()->is_unit_op() && !select_lex->table_list.elements && select_lex->item_list.elements == 1 && - !select_lex->item_list.head()->with_sum_func() && - /* - We can't change name of Item_field or Item_ref, because it will - prevent its correct resolving, but we should save name of - removed item => we do not make optimization if top item of - list is field or reference. - TODO: solve above problem - */ - !(select_lex->item_list.head()->type() == FIELD_ITEM || - select_lex->item_list.head()->type() == REF_ITEM) && !join->conds && !join->having && - thd->stmt_arena->state != Query_arena::STMT_INITIALIZED_FOR_SP - ) + need_to_pull_out_item( + join->select_lex->outer_select()->context_analysis_place, + select_lex->item_list.head()) && + thd->stmt_arena->state != Query_arena::STMT_INITIALIZED_FOR_SP) { have_to_be_excluded= 1; if (thd->lex->describe) From ddad20c63be0a3725e6b349a15a03eae82b2d381 Mon Sep 17 00:00:00 2001 From: zwang28 <84491488@qq.com> Date: Tue, 29 Jun 2021 20:23:29 +0800 Subject: [PATCH 07/22] MDEV-26043: Fix performance_schema instrument "threadpool/group_mutex" The performance_schema data related to instrument "wait/synch/mutex/threadpool/group_mutex" was incorrect. The root cause is the group_mutex was initialized in thread_group_init before registerd in PSI_register(mutex). The fix is to put PSI_register before thread_group_init, which ensures the right order. --- sql/threadpool_generic.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sql/threadpool_generic.cc b/sql/threadpool_generic.cc index f59c7cbea9d3b..b34104dbd0855 100644 --- a/sql/threadpool_generic.cc +++ b/sql/threadpool_generic.cc @@ -1601,6 +1601,9 @@ int TP_pool_generic::init() sql_print_error("Allocation failed"); DBUG_RETURN(-1); } + PSI_register(mutex); + PSI_register(cond); + PSI_register(thread); scheduler_init(); threadpool_started= true; for (uint i= 0; i < threadpool_max_size; i++) @@ -1614,10 +1617,6 @@ int TP_pool_generic::init() sql_print_error("Can't set threadpool size to %d",threadpool_size); DBUG_RETURN(-1); } - PSI_register(mutex); - PSI_register(cond); - PSI_register(thread); - pool_timer.tick_interval= threadpool_stall_limit; start_timer(&pool_timer); DBUG_RETURN(0); From 2916a7e7425f9bab7e58969938d0f8b00de237d2 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Mon, 19 Jul 2021 16:30:27 +0300 Subject: [PATCH 08/22] fix clang build --- sql/sql_table.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/sql_table.cc b/sql/sql_table.cc index b182401e3dbf6..7f6fa541640f2 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -2241,7 +2241,7 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists, bool trans_tmp_table_deleted= 0, non_trans_tmp_table_deleted= 0; bool non_tmp_table_deleted= 0; bool is_drop_tmp_if_exists_added= 0; - bool was_view= 0, was_table= 0, log_if_exists= if_exists; + bool was_view= 0, log_if_exists= if_exists; const char *object_to_drop= (drop_sequence) ? "SEQUENCE" : "TABLE"; String normal_tables; String built_trans_tmp_query, built_non_trans_tmp_query; @@ -2467,7 +2467,6 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists, . "DROP SEQUENCE", but it's not a sequence */ wrong_drop_sequence= drop_sequence && hton; - was_table|= wrong_drop_sequence; error= table_type == TABLE_TYPE_UNKNOWN ? ENOENT : -1; tdc_remove_table(thd, db.str, table_name.str); } From 832e473d5ea62c72e39fae3b6bc691994ff6a5fb Mon Sep 17 00:00:00 2001 From: Dmitry Shulga Date: Mon, 19 Jul 2021 23:13:18 +0700 Subject: [PATCH 09/22] MDEV-26181: The test compat/oracle.sp-row fails in case it is run in PS mode. There were several places where a statement delimiter missed so such statements were interpreted as multi-statements and expectedly failed in PS mode. An appropriate statement delimiters have been added to fix the issues. Addinitinally, the operators --enable_prepare_warnings/--disable_prepare_warnings have been added around statements that use depricated syntax SELECT INTO to don't miss warnings. --- mysql-test/suite/compat/oracle/r/sp-row.result | 2 ++ mysql-test/suite/compat/oracle/t/sp-row.test | 12 +++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/compat/oracle/r/sp-row.result b/mysql-test/suite/compat/oracle/r/sp-row.result index cecc737c9fb90..7fd986a71c890 100644 --- a/mysql-test/suite/compat/oracle/r/sp-row.result +++ b/mysql-test/suite/compat/oracle/r/sp-row.result @@ -252,6 +252,7 @@ AS BEGIN RETURN a; END; +$$ CREATE PROCEDURE p1() AS a ROW (a INT,b INT); @@ -268,6 +269,7 @@ AS BEGIN RETURN a; END; +$$ CREATE PROCEDURE p1() AS a ROW (a INT); diff --git a/mysql-test/suite/compat/oracle/t/sp-row.test b/mysql-test/suite/compat/oracle/t/sp-row.test index e2725e3a769d5..ebd0a2a2137ec 100644 --- a/mysql-test/suite/compat/oracle/t/sp-row.test +++ b/mysql-test/suite/compat/oracle/t/sp-row.test @@ -1,7 +1,3 @@ -if (`SELECT $PS_PROTOCOL != 0`) -{ - --skip Test temporarily disabled for ps-protocol -} SET sql_mode=ORACLE; @@ -329,6 +325,7 @@ AS BEGIN RETURN a; END; +$$ CREATE PROCEDURE p1() AS a ROW (a INT,b INT); @@ -349,6 +346,7 @@ AS BEGIN RETURN a; END; +$$ CREATE PROCEDURE p1() AS a ROW (a INT); @@ -866,6 +864,8 @@ DROP PROCEDURE p2; --echo # ROW fields as SELECT..INTO targets --echo # +--enable_prepare_warnings + DELIMITER $$; CREATE PROCEDURE p1 AS @@ -879,6 +879,7 @@ DELIMITER ;$$ CALL p1; DROP PROCEDURE p1; +--disable_prepare_warnings --echo # --echo # Implicit default NULL handling @@ -2088,6 +2089,7 @@ DROP PROCEDURE p1; --echo # +--enable_prepare_warnings --echo # ROW variable with a wrong column count CREATE TABLE t1 (a INT, b VARCHAR(32)); INSERT INTO t1 VALUES (10,'b10'); @@ -2248,7 +2250,7 @@ DELIMITER ;$$ CALL p1(); DROP TABLE t1; DROP PROCEDURE p1; - +--disable_prepare_warnings --echo # --echo # MDEV-12347 Valgrind reports invalid read errors in Item_field_row::element_index_by_name From 3e5a11e4889c97b781bb5828c8eed62251948a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 20 Jul 2021 09:29:45 +0300 Subject: [PATCH 10/22] MDEV-25236 fixup: instant_alter_debug,dynamic result We only support the instantaneous removal of the NOT NULL attribute for ROW_FORMAT=REDUNDANT tables. --- mysql-test/suite/innodb/r/instant_alter_debug,dynamic.rdiff | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 mysql-test/suite/innodb/r/instant_alter_debug,dynamic.rdiff diff --git a/mysql-test/suite/innodb/r/instant_alter_debug,dynamic.rdiff b/mysql-test/suite/innodb/r/instant_alter_debug,dynamic.rdiff new file mode 100644 index 0000000000000..379514edad9dc --- /dev/null +++ b/mysql-test/suite/innodb/r/instant_alter_debug,dynamic.rdiff @@ -0,0 +1,6 @@ +@@ -470,4 +470,4 @@ + FROM information_schema.global_status + WHERE variable_name = 'innodb_instant_alter_column'; + instants +-33 ++32 From ae62f115beea8cb7c532ef1a72c359548316df9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 20 Jul 2021 10:55:03 +0300 Subject: [PATCH 11/22] MDEV-26166: non-functional changes --- storage/innobase/buf/buf0flu.cc | 2 +- storage/innobase/log/log0log.cc | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 94c62a304f92c..7f09e607c5666 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1670,7 +1670,7 @@ ulint buf_flush_LRU(ulint max_n) if (buf_pool.n_flush_LRU()) return 0; - log_buffer_flush_to_disk(true); + log_buffer_flush_to_disk(); mysql_mutex_lock(&buf_pool.mutex); if (buf_pool.n_flush_LRU_) diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 1c0a617191e58..f0dc7852c159a 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -802,12 +802,10 @@ void log_write_up_to(lsn_t lsn, bool flush_to_disk, bool rotate_key, } if (flush_to_disk && - flush_lock.acquire(lsn, callback) != group_commit_lock::ACQUIRED) - { + flush_lock.acquire(lsn, callback) != group_commit_lock::ACQUIRED) return; - } - if (write_lock.acquire(lsn, flush_to_disk?0:callback) == + if (write_lock.acquire(lsn, flush_to_disk ? nullptr : callback) == group_commit_lock::ACQUIRED) { mysql_mutex_lock(&log_sys.mutex); @@ -821,9 +819,7 @@ void log_write_up_to(lsn_t lsn, bool flush_to_disk, bool rotate_key, } if (!flush_to_disk) - { return; - } /* Flush the highest written lsn.*/ auto flush_lsn = write_lock.value(); From 68694c8ed98b0c9b1c98a11fd629d56220221152 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 20 Jul 2021 12:36:35 +0300 Subject: [PATCH 12/22] Update libmariadb --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index fffa8167d883b..2adf5c6bafe20 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit fffa8167d883bbf841ecb04a77abe2fbf1d1dfc9 +Subproject commit 2adf5c6bafe206f93be711f6526916182f5d44a2 From ed0a7b1b3fe6874acc5c18763b765c428709ac22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 20 Jul 2021 17:35:03 +0300 Subject: [PATCH 13/22] MDEV-24626 fixup: Remove useless code fil_ibd_create(): Remove code that should have been removed in commit 86dc7b4d4cfe15a2d37f8b5f60c4fce5dba9491d already. We no longer wrote an initialized page to the file, but we would still allocate a page image in memory and write it. xb_space_create_file(): Remove an unnecessary page write. (This is a functional change for Mariabackup.) --- extra/mariabackup/xtrabackup.cc | 47 ---------------------------- storage/innobase/fil/fil0crypt.cc | 27 ---------------- storage/innobase/fil/fil0fil.cc | 26 --------------- storage/innobase/fsp/fsp0fsp.cc | 20 ------------ storage/innobase/include/fil0crypt.h | 6 ---- storage/innobase/include/fsp0fsp.h | 13 +------- 6 files changed, 1 insertion(+), 138 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 21f27c0c8ed4d..0b0cd5f55e360 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -4909,53 +4909,6 @@ xb_space_create_file( return ret; } - /* Align the memory for file i/o if we might have O_DIRECT set */ - byte* page = static_cast(aligned_malloc(2 * srv_page_size, - srv_page_size)); - - memset(page, '\0', srv_page_size); - - fsp_header_init_fields(page, space_id, flags); - mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); - - const ulint zip_size = fil_space_t::zip_size(flags); - - if (!zip_size) { - buf_flush_init_for_writing( - NULL, page, NULL, - fil_space_t::full_crc32(flags)); - - ret = os_file_write(IORequestWrite, path, *file, page, 0, - srv_page_size); - } else { - page_zip_des_t page_zip; - page_zip_set_size(&page_zip, zip_size); - page_zip.data = page + srv_page_size; - fprintf(stderr, "zip_size = " ULINTPF "\n", zip_size); - -#ifdef UNIV_DEBUG - page_zip.m_start = 0; -#endif /* UNIV_DEBUG */ - page_zip.m_end = 0; - page_zip.m_nonempty = 0; - page_zip.n_blobs = 0; - - buf_flush_init_for_writing(NULL, page, &page_zip, false); - - ret = os_file_write(IORequestWrite, path, *file, - page_zip.data, 0, zip_size); - } - - aligned_free(page); - - if (ret != DB_SUCCESS) { - msg("mariabackup: could not write the first page to %s", - path); - os_file_close(*file); - os_file_delete(0, path); - return ret; - } - return TRUE; } diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index a0b4b45287f72..82c26b59675dc 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -372,33 +372,6 @@ void fil_crypt_parse(fil_space_t* space, const byte* data) } } -/** Fill crypt data information to the give page. -It should be called during ibd file creation. -@param[in] flags tablespace flags -@param[in,out] page first page of the tablespace */ -void -fil_space_crypt_t::fill_page0( - ulint flags, - byte* page) -{ - const uint len = sizeof(iv); - const ulint offset = FSP_HEADER_OFFSET - + fsp_header_get_encryption_offset( - fil_space_t::zip_size(flags)); - - memcpy(page + offset, CRYPT_MAGIC, MAGIC_SZ); - mach_write_to_1(page + offset + MAGIC_SZ, type); - mach_write_to_1(page + offset + MAGIC_SZ + 1, len); - memcpy(page + offset + MAGIC_SZ + 2, &iv, len); - - mach_write_to_4(page + offset + MAGIC_SZ + 2 + len, - min_key_version); - mach_write_to_4(page + offset + MAGIC_SZ + 2 + len + 4, - key_id); - mach_write_to_1(page + offset + MAGIC_SZ + 2 + len + 8, - encryption); -} - /** Write encryption metadata to the first page. @param[in,out] block first page of the tablespace @param[in,out] mtr mini-transaction */ diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index c841eb794975a..3ebf0cc04dc4f 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1962,7 +1962,6 @@ fil_ibd_create( dberr_t* err) { pfs_os_file_t file; - byte* page; bool success; mtr_t mtr; bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags) != 0; @@ -2043,43 +2042,18 @@ fil_ibd_create( return NULL; } - /* We have to write the space id to the file immediately and flush the - file to disk. This is because in crash recovery we must be aware what - tablespaces exist and what are their space id's, so that we can apply - the log records to the right file. It may take quite a while until - buffer pool flush algorithms write anything to the file and flush it to - disk. If we would not write here anything, the file would be filled - with zeros from the call of os_file_set_size(), until a buffer pool - flush would write to it. */ - - /* Align the memory for file i/o if we might have O_DIRECT set */ - page = static_cast(aligned_malloc(2 * srv_page_size, - srv_page_size)); - - memset(page, '\0', srv_page_size); - if (fil_space_t::full_crc32(flags)) { flags |= FSP_FLAGS_FCRC32_PAGE_SSIZE(); } else { flags |= FSP_FLAGS_PAGE_SSIZE(); } - fsp_header_init_fields(page, space_id, flags); - mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); - /* Create crypt data if the tablespace is either encrypted or user has requested it to remain unencrypted. */ crypt_data = (mode != FIL_ENCRYPTION_DEFAULT || srv_encrypt_tables) ? fil_space_create_crypt_data(mode, key_id) : NULL; - if (crypt_data) { - /* Write crypt data information in page0 while creating - ibd file. */ - crypt_data->fill_page0(flags, page); - } - - aligned_free(page); fil_space_t::name_type space_name; if (has_data_dir) { diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index c35ac118e05fb..8477e74c5f148 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -525,26 +525,6 @@ void fil_space_t::modify_check(const mtr_t& mtr) const } #endif -/**********************************************************************//** -Writes the space id and flags to a tablespace header. The flags contain -row type, physical/compressed page size, and logical/uncompressed page -size of the tablespace. */ -void -fsp_header_init_fields( -/*===================*/ - page_t* page, /*!< in/out: first page in the space */ - ulint space_id, /*!< in: space id */ - ulint flags) /*!< in: tablespace flags (FSP_SPACE_FLAGS) */ -{ - flags &= ~FSP_FLAGS_MEM_MASK; - ut_a(fil_space_t::is_valid_flags(flags, space_id)); - - mach_write_to_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page, - space_id); - mach_write_to_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page, - flags); -} - /** Initialize a tablespace header. @param[in,out] space tablespace @param[in] size current size in blocks diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index eb63c5cf427c5..29a76defec1e7 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -172,12 +172,6 @@ struct fil_space_crypt_t : st_encryption_scheme return (encryption == FIL_ENCRYPTION_OFF); } - /** Fill crypt data information to the give page. - It should be called during ibd file creation. - @param[in] flags tablespace flags - @param[in,out] page first page of the tablespace */ - void fill_page0(ulint flags, byte* page); - /** Write encryption metadata to the first page. @param[in,out] block first page of the tablespace @param[in,out] mtr mini-transaction */ diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index 10765852529f2..51333cb5955b7 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2020, MariaDB Corporation. +Copyright (c) 2013, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -342,17 +342,6 @@ fsp_header_check_encryption_key( ulint fsp_flags, page_t* page); -/**********************************************************************//** -Writes the space id and flags to a tablespace header. The flags contain -row type, physical/compressed page size, and logical/uncompressed page -size of the tablespace. */ -void -fsp_header_init_fields( -/*===================*/ - page_t* page, /*!< in/out: first page in the space */ - ulint space_id, /*!< in: space id */ - ulint flags); /*!< in: tablespace flags (FSP_SPACE_FLAGS): - 0, or table->flags if newer than COMPACT */ /** Initialize a tablespace header. @param[in,out] space tablespace @param[in] size current size in blocks From 61fcbed920c0ed1373725c4122af5a483ae7ffb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 20 Jul 2021 17:36:46 +0300 Subject: [PATCH 14/22] MDEV-26192: Sparse files are being created on thinly provisioned storage In MDEV-26029 the intention was that page_compressed tables would be written as regular (non-sparse) files if the file is stored on a thinly provisioned block device. We were incorrectly requesting os_file_set_size() to create sparse files even on thinly provisioned storage. fil_space_extend_must_retry(): Extend the file in the correct fashion. fil_ibd_create(), recv_sys_t::recover_deferred(): Only create a sparse file for page_compressed tables if thin provisioning is not detected. --- storage/innobase/fil/fil0fil.cc | 35 ++++++++++++++++++-------------- storage/innobase/log/log0recv.cc | 12 ++++++++++- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 3ebf0cc04dc4f..91e4df7ca2646 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -577,7 +577,7 @@ fil_space_extend_must_retry( os_offset_t(FIL_IBD_FILE_INITIAL_SIZE << srv_page_size_shift)); *success = os_file_set_size(node->name, node->handle, new_size, - space->is_compressed()); + node->punch_hole == 1); os_has_said_disk_full = *success; if (*success) { @@ -2024,24 +2024,17 @@ fil_ibd_create( } const bool is_compressed = fil_space_t::is_compressed(flags); - fil_space_crypt_t* crypt_data = nullptr; #ifdef _WIN32 + const bool is_sparse = is_compressed; if (is_compressed) { os_file_set_sparse_win32(file); } +#else + const bool is_sparse = is_compressed + && DB_SUCCESS == os_file_punch_hole(file, 0, 4096) + && !my_test_if_thinly_provisioned(file); #endif - if (!os_file_set_size( - path, file, - os_offset_t(size) << srv_page_size_shift, is_compressed)) { - *err = DB_OUT_OF_FILE_SPACE; -err_exit: - os_file_close(file); - os_file_delete(innodb_data_file_key, path); - free(crypt_data); - return NULL; - } - if (fil_space_t::full_crc32(flags)) { flags |= FSP_FLAGS_FCRC32_PAGE_SSIZE(); } else { @@ -2050,9 +2043,21 @@ fil_ibd_create( /* Create crypt data if the tablespace is either encrypted or user has requested it to remain unencrypted. */ - crypt_data = (mode != FIL_ENCRYPTION_DEFAULT || srv_encrypt_tables) + fil_space_crypt_t* crypt_data = (mode != FIL_ENCRYPTION_DEFAULT + || srv_encrypt_tables) ? fil_space_create_crypt_data(mode, key_id) - : NULL; + : nullptr; + + if (!os_file_set_size(path, file, + os_offset_t(size) << srv_page_size_shift, + is_sparse)) { + *err = DB_OUT_OF_FILE_SPACE; +err_exit: + os_file_close(file); + os_file_delete(innodb_data_file_key, path); + free(crypt_data); + return nullptr; + } fil_space_t::name_type space_name; diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 1625092884529..f1e1c350135c3 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -839,9 +839,19 @@ bool recv_sys_t::recover_deferred(recv_sys_t::map::iterator &p, node->deferred= true; if (!space->acquire()) goto fail; + const bool is_compressed= fil_space_t::is_compressed(flags); +#ifdef _WIN32 + const bool is_sparse= is_compressed; + if (is_compressed) + os_file_set_sparse_win32(node->handle); +#else + const bool is_sparse= is_compressed && + DB_SUCCESS == os_file_punch_hole(node->handle, 0, 4096) && + !my_test_if_thinly_provisioned(node->handle); +#endif if (!os_file_set_size(node->name, node->handle, size * fil_space_t::physical_size(flags), - space->is_compressed())) + is_sparse)) { space->release(); goto fail; From 8642f592e6e447267eeb0db79a8a38752519d2b9 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 20 Jul 2021 09:19:23 +1000 Subject: [PATCH 15/22] debian/salsa: Show complete auth and plugin situtation SHOW PLUGINS has a more complete view of the installed plugins into the server. The mysql.user is a compatibility view that doesn't show the complete authentication picture. Use global_priv. Add `show create user` for default users to more clearly represent its contents. --- debian/salsa-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/salsa-ci.yml b/debian/salsa-ci.yml index 29ea9afd9d0c4..4584790919203 100644 --- a/debian/salsa-ci.yml +++ b/debian/salsa-ci.yml @@ -751,8 +751,8 @@ mariadb.org-10.2 to mariadb-10.5 upgrade: # prepending with --defaults-file=/etc/mysql/debian.cnf is needed in upstream 5.5–10.3 - mysql --defaults-file=/etc/mysql/debian.cnf --skip-column-names -e "SELECT @@version, @@version_comment" - echo 'SHOW DATABASES;' | mysql --defaults-file=/etc/mysql/debian.cnf - - mysql --defaults-file=/etc/mysql/debian.cnf -e "SELECT Host,User,plugin,authentication_string FROM user;" mysql - - mysql --defaults-file=/etc/mysql/debian.cnf -e "SELECT * FROM plugin;" mysql + - mysql --defaults-file=/etc/mysql/debian.cnf -e "SELECT * FROM mysql.global_priv; SHOW CREATE USER root@localhost; SHOW CREATE USER 'mariadb.sys'@localhost" + - mysql --defaults-file=/etc/mysql/debian.cnf -e "SELECT * FROM mysql.plugin; SHOW PLUGINS" - *test-install - service mysql status - sleep 5 # Give the mysql_upgrade a bit of time to complete before querying the server From f43370baf3b00733ddfe0f704520a07092fc9a0e Mon Sep 17 00:00:00 2001 From: Dmitry Shulga Date: Wed, 21 Jul 2021 19:31:17 +0700 Subject: [PATCH 16/22] MDEV-26150: follow-up patch to add missing fill opt_trace,ps.rdiff --- mysql-test/main/opt_trace,ps.rdiff | 92 ++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 mysql-test/main/opt_trace,ps.rdiff diff --git a/mysql-test/main/opt_trace,ps.rdiff b/mysql-test/main/opt_trace,ps.rdiff new file mode 100644 index 0000000000000..3e2218de673b0 --- /dev/null +++ b/mysql-test/main/opt_trace,ps.rdiff @@ -0,0 +1,92 @@ +--- /Users/shulga/projects/mariadb/server-10.6/mysql-test/main/opt_trace.result 2021-07-21 19:17:11.000000000 +0700 ++++ /Users/shulga/projects/mariadb/server-10.6/mysql-test/main/opt_trace.reject 2021-07-21 19:17:48.000000000 +0700 +@@ -2829,14 +2829,6 @@ + } + }, + { +- "transformation": { +- "select_id": 2, +- "from": "IN (SELECT)", +- "to": "semijoin", +- "chosen": true +- } +- }, +- { + "expanded_query": "/* select#2 */ select t10.pk from t10" + } + ] +@@ -4402,14 +4394,6 @@ + } + }, + { +- "transformation": { +- "select_id": 2, +- "from": "IN (SELECT)", +- "to": "semijoin", +- "chosen": true +- } +- }, +- { + "expanded_query": "/* select#2 */ select t_inner_1.a from t1 t_inner_1 join t1 t_inner_2" + } + ] +@@ -4852,14 +4836,6 @@ + } + }, + { +- "transformation": { +- "select_id": 2, +- "from": "IN (SELECT)", +- "to": "semijoin", +- "chosen": true +- } +- }, +- { + "expanded_query": "/* select#2 */ select t_inner_1.a from t2 t_inner_2 join t1 t_inner_1" + } + ] +@@ -4879,14 +4855,6 @@ + } + }, + { +- "transformation": { +- "select_id": 3, +- "from": "IN (SELECT)", +- "to": "semijoin", +- "chosen": true +- } +- }, +- { + "expanded_query": "/* select#3 */ select t_inner_3.a from t2 t_inner_3 join t1 t_inner_4" + } + ] +@@ -6432,14 +6400,6 @@ + } + }, + { +- "transformation": { +- "select_id": 2, +- "from": "IN (SELECT)", +- "to": "semijoin", +- "chosen": true +- } +- }, +- { + "expanded_query": "/* select#2 */ select t_inner_1.a from t2 t_inner_2 join t1 t_inner_1" + } + ] +@@ -6459,14 +6419,6 @@ + } + }, + { +- "transformation": { +- "select_id": 3, +- "from": "IN (SELECT)", +- "to": "semijoin", +- "chosen": true +- } +- }, +- { + "expanded_query": "/* select#3 */ select t_inner_3.a from t2 t_inner_3 join t1 t_inner_4" + } + ] From 1519013f518af5331fa48a5f602808aa2346df05 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 22 Jul 2021 15:22:47 +1000 Subject: [PATCH 17/22] mtr: aix - stack-trace is optional --- mysql-test/main/crash_commit_before-master.opt | 2 +- mysql-test/main/myisam_crash_before_flush_keys-master.opt | 2 +- mysql-test/suite/binlog/t/binlog_index-master.opt | 2 +- mysql-test/suite/binlog/t/binlog_mdev342-master.opt | 2 +- mysql-test/suite/binlog/t/binlog_xa_recover.opt | 2 +- mysql-test/suite/innodb/t/group_commit_binlog_pos-master.opt | 2 +- .../t/group_commit_binlog_pos_no_optimize_thread-master.opt | 2 +- mysql-test/suite/innodb/t/group_commit_crash-master.opt | 2 +- .../innodb/t/group_commit_crash_no_optimize_thread-master.opt | 2 +- mysql-test/suite/innodb/t/innodb_bug14147491-master.opt | 2 +- mysql-test/suite/maria/bulk_insert_crash.opt | 2 +- mysql-test/suite/maria/maria-gis-recovery.opt | 2 +- mysql-test/suite/maria/maria-recovery-big-master.opt | 2 +- mysql-test/suite/maria/maria-recovery-bitmap-master.opt | 2 +- mysql-test/suite/maria/maria-recovery-master.opt | 2 +- mysql-test/suite/maria/maria-recovery-rtree-ft-master.opt | 2 +- mysql-test/suite/maria/maria-recovery2-master.opt | 2 +- mysql-test/suite/maria/maria-recovery3-master.opt | 2 +- mysql-test/suite/parts/t/debug_innodb_crash-master.opt | 2 +- mysql-test/suite/parts/t/debug_myisam_crash-master.opt | 2 +- mysql-test/suite/parts/t/partition_debug_innodb-master.opt | 2 +- mysql-test/suite/rpl/t/rpl_gtid_crash-master.opt | 2 +- mysql-test/suite/rpl/t/rpl_gtid_crash_myisam-master.opt | 2 +- mysql-test/suite/rpl/t/rpl_parallel_temptable-master.opt | 2 +- 24 files changed, 24 insertions(+), 24 deletions(-) diff --git a/mysql-test/main/crash_commit_before-master.opt b/mysql-test/main/crash_commit_before-master.opt index 9bcf94dca9776..f464a1013d7d8 100644 --- a/mysql-test/main/crash_commit_before-master.opt +++ b/mysql-test/main/crash_commit_before-master.opt @@ -1,3 +1,3 @@ ---skip-stack-trace --skip-core-file +--loose-skip-stack-trace --skip-core-file --default-storage-engine=MyISAM --loose-skip-innodb-file-per-table diff --git a/mysql-test/main/myisam_crash_before_flush_keys-master.opt b/mysql-test/main/myisam_crash_before_flush_keys-master.opt index 425fda95086dc..590d44a6d12ba 100644 --- a/mysql-test/main/myisam_crash_before_flush_keys-master.opt +++ b/mysql-test/main/myisam_crash_before_flush_keys-master.opt @@ -1 +1 @@ ---skip-stack-trace --skip-core-file +--loose-skip-stack-trace --skip-core-file diff --git a/mysql-test/suite/binlog/t/binlog_index-master.opt b/mysql-test/suite/binlog/t/binlog_index-master.opt index a1ad7417ad38f..c3754b4d8d0bf 100644 --- a/mysql-test/suite/binlog/t/binlog_index-master.opt +++ b/mysql-test/suite/binlog/t/binlog_index-master.opt @@ -1 +1 @@ ---skip-stack-trace --log-warnings=0 --log-bin=master-bin --log-bin-index=master-bin +--loose-skip-stack-trace --log-warnings=0 --log-bin=master-bin --log-bin-index=master-bin diff --git a/mysql-test/suite/binlog/t/binlog_mdev342-master.opt b/mysql-test/suite/binlog/t/binlog_mdev342-master.opt index 425fda95086dc..590d44a6d12ba 100644 --- a/mysql-test/suite/binlog/t/binlog_mdev342-master.opt +++ b/mysql-test/suite/binlog/t/binlog_mdev342-master.opt @@ -1 +1 @@ ---skip-stack-trace --skip-core-file +--loose-skip-stack-trace --skip-core-file diff --git a/mysql-test/suite/binlog/t/binlog_xa_recover.opt b/mysql-test/suite/binlog/t/binlog_xa_recover.opt index 3c44f9fad10ae..f645a058a7b8e 100644 --- a/mysql-test/suite/binlog/t/binlog_xa_recover.opt +++ b/mysql-test/suite/binlog/t/binlog_xa_recover.opt @@ -1 +1 @@ ---skip-stack-trace --skip-core-file --loose-debug-dbug=+d,xa_recover_expect_master_bin_000004 +--loose-skip-stack-trace --skip-core-file --loose-debug-dbug=+d,xa_recover_expect_master_bin_000004 diff --git a/mysql-test/suite/innodb/t/group_commit_binlog_pos-master.opt b/mysql-test/suite/innodb/t/group_commit_binlog_pos-master.opt index 425fda95086dc..590d44a6d12ba 100644 --- a/mysql-test/suite/innodb/t/group_commit_binlog_pos-master.opt +++ b/mysql-test/suite/innodb/t/group_commit_binlog_pos-master.opt @@ -1 +1 @@ ---skip-stack-trace --skip-core-file +--loose-skip-stack-trace --skip-core-file diff --git a/mysql-test/suite/innodb/t/group_commit_binlog_pos_no_optimize_thread-master.opt b/mysql-test/suite/innodb/t/group_commit_binlog_pos_no_optimize_thread-master.opt index 18d43988ffdc0..b8c4666a7308a 100644 --- a/mysql-test/suite/innodb/t/group_commit_binlog_pos_no_optimize_thread-master.opt +++ b/mysql-test/suite/innodb/t/group_commit_binlog_pos_no_optimize_thread-master.opt @@ -1 +1 @@ ---binlog-optimize-thread-scheduling=0 --skip-stack-trace --skip-core-file +--binlog-optimize-thread-scheduling=0 --loose-skip-stack-trace --skip-core-file diff --git a/mysql-test/suite/innodb/t/group_commit_crash-master.opt b/mysql-test/suite/innodb/t/group_commit_crash-master.opt index 425fda95086dc..590d44a6d12ba 100644 --- a/mysql-test/suite/innodb/t/group_commit_crash-master.opt +++ b/mysql-test/suite/innodb/t/group_commit_crash-master.opt @@ -1 +1 @@ ---skip-stack-trace --skip-core-file +--loose-skip-stack-trace --skip-core-file diff --git a/mysql-test/suite/innodb/t/group_commit_crash_no_optimize_thread-master.opt b/mysql-test/suite/innodb/t/group_commit_crash_no_optimize_thread-master.opt index 18d43988ffdc0..b8c4666a7308a 100644 --- a/mysql-test/suite/innodb/t/group_commit_crash_no_optimize_thread-master.opt +++ b/mysql-test/suite/innodb/t/group_commit_crash_no_optimize_thread-master.opt @@ -1 +1 @@ ---binlog-optimize-thread-scheduling=0 --skip-stack-trace --skip-core-file +--binlog-optimize-thread-scheduling=0 --loose-skip-stack-trace --skip-core-file diff --git a/mysql-test/suite/innodb/t/innodb_bug14147491-master.opt b/mysql-test/suite/innodb/t/innodb_bug14147491-master.opt index 410738202bdb8..614dd9356f1ee 100644 --- a/mysql-test/suite/innodb/t/innodb_bug14147491-master.opt +++ b/mysql-test/suite/innodb/t/innodb_bug14147491-master.opt @@ -1,4 +1,4 @@ --innodb_file_per_table=1 ---skip-stack-trace +--loose-skip-stack-trace --skip-core-file --loose-innodb_buffer_pool_load_at_startup=OFF diff --git a/mysql-test/suite/maria/bulk_insert_crash.opt b/mysql-test/suite/maria/bulk_insert_crash.opt index f85a8d9c973fe..ecc7a8e8d9271 100644 --- a/mysql-test/suite/maria/bulk_insert_crash.opt +++ b/mysql-test/suite/maria/bulk_insert_crash.opt @@ -1,2 +1,2 @@ ---skip-stack-trace --skip-core-file +--loose-skip-stack-trace --skip-core-file --default-storage-engine=Aria diff --git a/mysql-test/suite/maria/maria-gis-recovery.opt b/mysql-test/suite/maria/maria-gis-recovery.opt index 58d0d012c548d..839411b10b8b0 100644 --- a/mysql-test/suite/maria/maria-gis-recovery.opt +++ b/mysql-test/suite/maria/maria-gis-recovery.opt @@ -1 +1 @@ ---skip-stack-trace --skip-core-file --loose-aria-log-dir-path=$MYSQLTEST_VARDIR/tmp +--loose-skip-stack-trace --skip-core-file --loose-aria-log-dir-path=$MYSQLTEST_VARDIR/tmp diff --git a/mysql-test/suite/maria/maria-recovery-big-master.opt b/mysql-test/suite/maria/maria-recovery-big-master.opt index d24a11c924f94..f5119bbce3f6a 100644 --- a/mysql-test/suite/maria/maria-recovery-big-master.opt +++ b/mysql-test/suite/maria/maria-recovery-big-master.opt @@ -1 +1 @@ ---skip-stack-trace --skip-core-file --max_allowed_packet=32000000 +--loose-skip-stack-trace --skip-core-file --max_allowed_packet=32000000 diff --git a/mysql-test/suite/maria/maria-recovery-bitmap-master.opt b/mysql-test/suite/maria/maria-recovery-bitmap-master.opt index 425fda95086dc..590d44a6d12ba 100644 --- a/mysql-test/suite/maria/maria-recovery-bitmap-master.opt +++ b/mysql-test/suite/maria/maria-recovery-bitmap-master.opt @@ -1 +1 @@ ---skip-stack-trace --skip-core-file +--loose-skip-stack-trace --skip-core-file diff --git a/mysql-test/suite/maria/maria-recovery-master.opt b/mysql-test/suite/maria/maria-recovery-master.opt index 58d0d012c548d..839411b10b8b0 100644 --- a/mysql-test/suite/maria/maria-recovery-master.opt +++ b/mysql-test/suite/maria/maria-recovery-master.opt @@ -1 +1 @@ ---skip-stack-trace --skip-core-file --loose-aria-log-dir-path=$MYSQLTEST_VARDIR/tmp +--loose-skip-stack-trace --skip-core-file --loose-aria-log-dir-path=$MYSQLTEST_VARDIR/tmp diff --git a/mysql-test/suite/maria/maria-recovery-rtree-ft-master.opt b/mysql-test/suite/maria/maria-recovery-rtree-ft-master.opt index 425fda95086dc..590d44a6d12ba 100644 --- a/mysql-test/suite/maria/maria-recovery-rtree-ft-master.opt +++ b/mysql-test/suite/maria/maria-recovery-rtree-ft-master.opt @@ -1 +1 @@ ---skip-stack-trace --skip-core-file +--loose-skip-stack-trace --skip-core-file diff --git a/mysql-test/suite/maria/maria-recovery2-master.opt b/mysql-test/suite/maria/maria-recovery2-master.opt index ca9560676a58e..eeb6de091fd53 100644 --- a/mysql-test/suite/maria/maria-recovery2-master.opt +++ b/mysql-test/suite/maria/maria-recovery2-master.opt @@ -1 +1 @@ ---skip-stack-trace --skip-core-file --loose-aria-log-dir-path=$MYSQLTEST_VARDIR/tmp --myisam-recover-options= +--loose-skip-stack-trace --skip-core-file --loose-aria-log-dir-path=$MYSQLTEST_VARDIR/tmp --myisam-recover-options= diff --git a/mysql-test/suite/maria/maria-recovery3-master.opt b/mysql-test/suite/maria/maria-recovery3-master.opt index 58d0d012c548d..839411b10b8b0 100644 --- a/mysql-test/suite/maria/maria-recovery3-master.opt +++ b/mysql-test/suite/maria/maria-recovery3-master.opt @@ -1 +1 @@ ---skip-stack-trace --skip-core-file --loose-aria-log-dir-path=$MYSQLTEST_VARDIR/tmp +--loose-skip-stack-trace --skip-core-file --loose-aria-log-dir-path=$MYSQLTEST_VARDIR/tmp diff --git a/mysql-test/suite/parts/t/debug_innodb_crash-master.opt b/mysql-test/suite/parts/t/debug_innodb_crash-master.opt index 4fa3cb12e2972..b7f94e14e12c0 100644 --- a/mysql-test/suite/parts/t/debug_innodb_crash-master.opt +++ b/mysql-test/suite/parts/t/debug_innodb_crash-master.opt @@ -1 +1 @@ ---loose-innodb-file-per-table=1 --skip-stack-trace --skip-core-file --loose-innodb-buffer-pool-size=32M +--loose-innodb-file-per-table=1 --loose-skip-stack-trace --skip-core-file --loose-innodb-buffer-pool-size=32M diff --git a/mysql-test/suite/parts/t/debug_myisam_crash-master.opt b/mysql-test/suite/parts/t/debug_myisam_crash-master.opt index eadc9396657fd..826d7c97aae31 100644 --- a/mysql-test/suite/parts/t/debug_myisam_crash-master.opt +++ b/mysql-test/suite/parts/t/debug_myisam_crash-master.opt @@ -1 +1 @@ ---skip-stack-trace --skip-core-file --myisam-recover-options=off +--loose-skip-stack-trace --skip-core-file --myisam-recover-options=off diff --git a/mysql-test/suite/parts/t/partition_debug_innodb-master.opt b/mysql-test/suite/parts/t/partition_debug_innodb-master.opt index 4fa3cb12e2972..b7f94e14e12c0 100644 --- a/mysql-test/suite/parts/t/partition_debug_innodb-master.opt +++ b/mysql-test/suite/parts/t/partition_debug_innodb-master.opt @@ -1 +1 @@ ---loose-innodb-file-per-table=1 --skip-stack-trace --skip-core-file --loose-innodb-buffer-pool-size=32M +--loose-innodb-file-per-table=1 --loose-skip-stack-trace --skip-core-file --loose-innodb-buffer-pool-size=32M diff --git a/mysql-test/suite/rpl/t/rpl_gtid_crash-master.opt b/mysql-test/suite/rpl/t/rpl_gtid_crash-master.opt index 425fda95086dc..590d44a6d12ba 100644 --- a/mysql-test/suite/rpl/t/rpl_gtid_crash-master.opt +++ b/mysql-test/suite/rpl/t/rpl_gtid_crash-master.opt @@ -1 +1 @@ ---skip-stack-trace --skip-core-file +--loose-skip-stack-trace --skip-core-file diff --git a/mysql-test/suite/rpl/t/rpl_gtid_crash_myisam-master.opt b/mysql-test/suite/rpl/t/rpl_gtid_crash_myisam-master.opt index 32711eb97264d..16d8af5b6f139 100644 --- a/mysql-test/suite/rpl/t/rpl_gtid_crash_myisam-master.opt +++ b/mysql-test/suite/rpl/t/rpl_gtid_crash_myisam-master.opt @@ -1 +1 @@ ---skip-stack-trace --skip-core-file --skip-innodb +--loose-skip-stack-trace --skip-core-file --skip-innodb diff --git a/mysql-test/suite/rpl/t/rpl_parallel_temptable-master.opt b/mysql-test/suite/rpl/t/rpl_parallel_temptable-master.opt index 425fda95086dc..590d44a6d12ba 100644 --- a/mysql-test/suite/rpl/t/rpl_parallel_temptable-master.opt +++ b/mysql-test/suite/rpl/t/rpl_parallel_temptable-master.opt @@ -1 +1 @@ ---skip-stack-trace --skip-core-file +--loose-skip-stack-trace --skip-core-file From 316a8cebf5c5833ca1dde5ff6e81d9c27c3af0ed Mon Sep 17 00:00:00 2001 From: nia Date: Sat, 3 Jul 2021 11:27:56 +0200 Subject: [PATCH 18/22] Fix building crc32_arm64 on NetBSD/aarch64 pmull_supported is not necessarily defined before crc32c_aarch64 Signed-off-by: Nia Alarie --- mysys/crc32/crc32_arm64.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mysys/crc32/crc32_arm64.c b/mysys/crc32/crc32_arm64.c index 62606c7ddea53..0e70c21812a09 100644 --- a/mysys/crc32/crc32_arm64.c +++ b/mysys/crc32/crc32_arm64.c @@ -2,13 +2,13 @@ #include #include +static int pmull_supported; + #if defined(HAVE_ARMV8_CRC) #if defined(__APPLE__) #include -static int pmull_supported; - int crc32_aarch64_available(void) { int ret; @@ -48,8 +48,6 @@ static unsigned long getauxval(unsigned int key) # define HWCAP_PMULL (1 << 4) #endif -static int pmull_supported; - /* ARM made crc32 default from ARMv8.1 but optional in ARMv8A * Runtime check API. */ From bf435a3f4daa90dd6b4b94ed62f05a8e30fecc3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 22 Jul 2021 08:34:49 +0300 Subject: [PATCH 19/22] MDEV-26200 buf_pool.flush_list corrupted by buffer pool resizing or ROW_FORMAT=COMPRESSED The lazy deletion of clean blocks from buf_pool.flush_list that was introduced in commit 6441bc614a99f5cd6357c8a23b9f583c56d0a90c (MDEV-25113) introduced a race condition around the function buf_flush_relocate_on_flush_list(). The test innodb_zip.wl5522_debug_zip as well as the buffer pool resizing tests would occasionally fail in debug builds due to buf_pool.flush_list.count disagreeing with the actual length of the doubly-linked list. The safe procedure for relocating a block in buf_pool.flush_list should be as follows, now that we implement lazy deletion from buf_pool.flush_list: 1. Acquire buf_pool.mutex. 2. Acquire the exclusive buf_pool.page_hash.latch. 3. Acquire buf_pool.flush_list_mutex. 4. Copy the block descriptor. 5. Invoke buf_flush_relocate_on_flush_list(). 6. Release buf_pool.flush_list_mutex. buf_flush_relocate_on_flush_list(): Assert that buf_pool.flush_list_mutex is being held. Invoke buf_page_t::oldest_modification() only once, using std::memory_order_relaxed, now that the mutex protects us. buf_LRU_free_page(), buf_LRU_block_remove_hashed(): Avoid an unlock-lock cycle on hash_lock. (We must not acquire hash_lock while already holding buf_pool.flush_list_mutex, because that could lead to a deadlock due to latching order violation.) --- storage/innobase/buf/buf0buf.cc | 6 ++++++ storage/innobase/buf/buf0flu.cc | 31 ++++++++----------------------- storage/innobase/buf/buf0lru.cc | 14 +++++++++++--- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index dcf95fdd5f24a..aca0d6b5ae807 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1566,6 +1566,7 @@ inline bool buf_pool_t::realloc(buf_block_t *block) if (block->page.can_relocate()) { memcpy_aligned( new_block->frame, block->frame, srv_page_size); + mysql_mutex_lock(&buf_pool.flush_list_mutex); new (&new_block->page) buf_page_t(block->page); /* relocate LRU list */ @@ -1625,6 +1626,7 @@ inline bool buf_pool_t::realloc(buf_block_t *block) buf_flush_relocate_on_flush_list(&block->page, &new_block->page); } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); block->page.set_corrupt_id(); /* set other flags of buf_block_t */ @@ -3131,12 +3133,14 @@ buf_page_get_low( /* Note: this is the uncompressed block and it is not accessible by other threads yet because it is not in any list or hash table */ + mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_relocate(bpage, &block->page); /* Set after buf_relocate(). */ block->page.set_buf_fix_count(1); buf_flush_relocate_on_flush_list(bpage, &block->page); + mysql_mutex_unlock(&buf_pool.flush_list_mutex); /* Buffer-fix, I/O-fix, and X-latch the block for the duration of the decompression. @@ -3646,8 +3650,10 @@ buf_page_create(fil_space_t *space, uint32_t offset, } rw_lock_x_lock(&free_block->lock); + mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_relocate(&block->page, &free_block->page); buf_flush_relocate_on_flush_list(&block->page, &free_block->page); + mysql_mutex_unlock(&buf_pool.flush_list_mutex); free_block->page.set_state(BUF_BLOCK_FILE_PAGE); buf_unzip_LRU_add_block(free_block, FALSE); diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 1c425f308eff9..10a84d99a2ef3 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -286,43 +286,29 @@ buf_flush_relocate_on_flush_list( { buf_page_t* prev; - mysql_mutex_assert_owner(&buf_pool.mutex); + mysql_mutex_assert_owner(&buf_pool.flush_list_mutex); ut_ad(!fsp_is_system_temporary(bpage->id().space())); - const lsn_t lsn = bpage->oldest_modification_acquire(); + const lsn_t lsn = bpage->oldest_modification(); if (!lsn) { return; } ut_ad(lsn == 1 || lsn > 2); - - mysql_mutex_lock(&buf_pool.flush_list_mutex); - - /* FIXME: Can we avoid holding buf_pool.mutex here? */ ut_ad(dpage->oldest_modification() == lsn); - if (ut_d(const lsn_t o_lsn =) bpage->oldest_modification()) { - ut_ad(o_lsn == lsn); + /* Important that we adjust the hazard pointer before removing + the bpage from the flush list. */ + buf_pool.flush_hp.adjust(bpage); - /* Important that we adjust the hazard pointer before removing - the bpage from the flush list. */ - buf_pool.flush_hp.adjust(bpage); + prev = UT_LIST_GET_PREV(list, bpage); + UT_LIST_REMOVE(buf_pool.flush_list, bpage); - prev = UT_LIST_GET_PREV(list, bpage); - UT_LIST_REMOVE(buf_pool.flush_list, bpage); - - bpage->clear_oldest_modification(); - } else { - /* bpage was removed from buf_pool.flush_list - since we last checked, and before we acquired - buf_pool.flush_list_mutex. */ - goto was_clean; - } + bpage->clear_oldest_modification(); if (lsn == 1) { buf_pool.stat.flush_list_bytes -= dpage->physical_size(); -was_clean: dpage->list.prev = nullptr; dpage->list.next = nullptr; dpage->clear_oldest_modification(); @@ -334,7 +320,6 @@ buf_flush_relocate_on_flush_list( } ut_d(buf_flush_validate_low()); - mysql_mutex_unlock(&buf_pool.flush_list_mutex); } /** Complete write of a file page from buf_pool. diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index cb42d4e2ae377..b282eb17daec7 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -845,6 +845,7 @@ bool buf_LRU_free_page(buf_page_t *bpage, bool zip) } else if (bpage->state() == BUF_BLOCK_FILE_PAGE) { b = buf_page_alloc_descriptor(); ut_a(b); + mysql_mutex_lock(&buf_pool.flush_list_mutex); new (b) buf_page_t(*bpage); b->set_state(BUF_BLOCK_ZIP_PAGE); } @@ -859,6 +860,8 @@ bool buf_LRU_free_page(buf_page_t *bpage, bool zip) ut_ad(bpage->can_relocate()); if (!buf_LRU_block_remove_hashed(bpage, id, hash_lock, zip)) { + ut_ad(!b); + mysql_mutex_assert_not_owner(&buf_pool.flush_list_mutex); return(true); } @@ -872,8 +875,6 @@ bool buf_LRU_free_page(buf_page_t *bpage, bool zip) if (UNIV_LIKELY_NULL(b)) { buf_page_t* prev_b = UT_LIST_GET_PREV(LRU, b); - hash_lock->write_lock(); - ut_ad(!buf_pool.page_hash_get_low(id, fold)); ut_ad(b->zip_size()); @@ -940,6 +941,7 @@ bool buf_LRU_free_page(buf_page_t *bpage, bool zip) } buf_flush_relocate_on_flush_list(bpage, b); + mysql_mutex_unlock(&buf_pool.flush_list_mutex); bpage->zip.data = nullptr; @@ -950,6 +952,8 @@ bool buf_LRU_free_page(buf_page_t *bpage, bool zip) hash_lock. */ b->set_io_fix(BUF_IO_PIN); hash_lock->write_unlock(); + } else if (!zip) { + hash_lock->write_unlock(); } buf_block_t* block = reinterpret_cast(bpage); @@ -1182,6 +1186,10 @@ static bool buf_LRU_block_remove_hashed(buf_page_t *bpage, const page_id_t id, MEM_UNDEFINED(((buf_block_t*) bpage)->frame, srv_page_size); bpage->set_state(BUF_BLOCK_REMOVE_HASH); + if (!zip) { + return true; + } + /* Question: If we release hash_lock here then what protects us against: 1) Some other thread buffer fixing this page @@ -1203,7 +1211,7 @@ static bool buf_LRU_block_remove_hashed(buf_page_t *bpage, const page_id_t id, page_hash. */ hash_lock->write_unlock(); - if (zip && bpage->zip.data) { + if (bpage->zip.data) { /* Free the compressed page. */ void* data = bpage->zip.data; bpage->zip.data = NULL; From 82d5994520c239da1b6edf1b24e08138ae0c753d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 22 Jul 2021 09:50:20 +0300 Subject: [PATCH 20/22] MDEV-26110: Do not rely on alignment on static allocation It is implementation-defined whether alignment requirements that are larger than std::max_align_t (typically 8 or 16 bytes) will be honored by the compiler and linker. It turns out that on IBM AIX, both alignas() and MY_ALIGNED() only guarantees alignment up to 16 bytes. For some data structures, specifying alignment to the CPU cache line size (typically 64 or 128 bytes) is a mere performance optimization, and we do not really care whether the requested alignment is guaranteed. But, for the correct operation of direct I/O, we do require that the buffers be aligned at a block size boundary. field_ref_zero: Define as a pointer, not an array. For innochecksum, we can make this point to unaligned memory; for anything else, we will allocate an aligned buffer from the heap. This buffer will be used for overwriting freed data pages when innodb_immediate_scrub_data_uncompressed=ON. And exactly that code hit an assertion failure on AIX, in the test innodb.innodb_scrub. log_sys.checkpoint_buf: Define as a pointer to aligned memory that is allocated from heap. log_t::file::write_header_durable(): Reuse log_sys.checkpoint_buf instead of trying to allocate an aligned buffer from the stack. --- extra/innochecksum.cc | 3 ++ extra/mariabackup/xtrabackup.cc | 39 ++++++++++++++++------- storage/innobase/buf/buf0buf.cc | 22 +++++++++++-- storage/innobase/handler/handler0alter.cc | 2 +- storage/innobase/include/buf0types.h | 8 ++--- storage/innobase/include/log0log.h | 5 ++- storage/innobase/log/log0log.cc | 19 +++++++---- storage/innobase/page/page0zip.cc | 14 ++------ storage/innobase/row/row0ins.cc | 2 +- 9 files changed, 72 insertions(+), 42 deletions(-) diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index b2aade70d6862..0d931129afaa4 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -97,6 +97,9 @@ FILE* log_file = NULL; /* Enabled for log write option. */ static bool is_log_enabled = false; +static byte field_ref_zero_buf[UNIV_PAGE_SIZE_MAX]; +const byte *field_ref_zero = field_ref_zero_buf; + #ifndef _WIN32 /* advisory lock for non-window system. */ struct flock lk; diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 7ec6fbb8f5965..bf4a80832dc54 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -4501,6 +4501,14 @@ static bool xtrabackup_backup_func() goto fail; } + + if (auto b = aligned_malloc(UNIV_PAGE_SIZE_MAX, 4096)) { + field_ref_zero = static_cast( + memset_aligned<4096>(b, 0, UNIV_PAGE_SIZE_MAX)); + } else { + goto fail; + } + { /* definition from recv_recovery_from_checkpoint_start() */ ulint max_cp_field; @@ -4515,14 +4523,17 @@ static bool xtrabackup_backup_func() if (err != DB_SUCCESS) { msg("Error: cannot read redo log header"); +unlock_and_fail: mysql_mutex_unlock(&log_sys.mutex); +free_and_fail: + aligned_free(const_cast(field_ref_zero)); + field_ref_zero = nullptr; goto fail; } if (log_sys.log.format == 0) { msg("Error: cannot process redo log before MariaDB 10.2.2"); - mysql_mutex_unlock(&log_sys.mutex); - goto fail; + goto unlock_and_fail; } byte* buf = log_sys.checkpoint_buf; @@ -4543,7 +4554,7 @@ static bool xtrabackup_backup_func() xtrabackup_init_datasinks(); if (!select_history()) { - goto fail; + goto free_and_fail; } /* open the log file */ @@ -4552,12 +4563,13 @@ static bool xtrabackup_backup_func() if (dst_log_file == NULL) { msg("Error: failed to open the target stream for '%s'.", LOG_FILE_NAME); - goto fail; + goto free_and_fail; } /* label it */ - byte MY_ALIGNED(OS_FILE_LOG_BLOCK_SIZE) log_hdr_buf[LOG_FILE_HDR_SIZE]; - memset(log_hdr_buf, 0, sizeof log_hdr_buf); + byte* log_hdr_buf = static_cast( + aligned_malloc(LOG_FILE_HDR_SIZE, OS_FILE_LOG_BLOCK_SIZE)); + memset(log_hdr_buf, 0, LOG_FILE_HDR_SIZE); byte *log_hdr_field = log_hdr_buf; mach_write_to_4(LOG_HEADER_FORMAT + log_hdr_field, log_sys.log.format); @@ -4586,11 +4598,13 @@ static bool xtrabackup_backup_func() log_block_calc_checksum_crc32(log_hdr_field)); /* Write log header*/ - if (ds_write(dst_log_file, log_hdr_buf, sizeof(log_hdr_buf))) { + if (ds_write(dst_log_file, log_hdr_buf, LOG_FILE_HDR_SIZE)) { msg("error: write to logfile failed"); - goto fail; + aligned_free(log_hdr_buf); + goto free_and_fail; } + aligned_free(log_hdr_buf); log_copying_running = true; /* start io throttle */ if(xtrabackup_throttle) { @@ -4608,7 +4622,7 @@ static bool xtrabackup_backup_func() " error %s.", ut_strerr(err)); fail_before_log_copying_thread_start: log_copying_running = false; - goto fail; + goto free_and_fail; } /* copy log file by current position */ @@ -4625,7 +4639,7 @@ static bool xtrabackup_backup_func() /* FLUSH CHANGED_PAGE_BITMAPS call */ if (!flush_changed_page_bitmaps()) { - goto fail; + goto free_and_fail; } ut_a(xtrabackup_parallel > 0); @@ -4647,7 +4661,7 @@ static bool xtrabackup_backup_func() datafiles_iter_t *it = datafiles_iter_new(); if (it == NULL) { msg("mariabackup: Error: datafiles_iter_new() failed."); - goto fail; + goto free_and_fail; } /* Create data copying threads */ @@ -4702,6 +4716,9 @@ static bool xtrabackup_backup_func() if (opt_log_innodb_page_corruption) ok = corrupted_pages.print_to_file(MB_CORRUPTED_PAGES_FILE); + aligned_free(const_cast(field_ref_zero)); + field_ref_zero = nullptr; + if (!ok) { goto fail; } diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index aca0d6b5ae807..8de876ebe6b9c 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -321,6 +321,12 @@ constexpr ulint BUF_PAGE_READ_MAX_RETRIES= 100; read-ahead buffer. (Divide buf_pool size by this amount) */ constexpr uint32_t BUF_READ_AHEAD_PORTION= 32; +/** A 64KiB buffer of NUL bytes, for use in assertions and checks, +and dummy default values of instantly dropped columns. +Initially, BLOB field references are set to NUL bytes, in +dtuple_convert_big_rec(). */ +const byte *field_ref_zero; + /** The InnoDB buffer pool */ buf_pool_t buf_pool; buf_pool_t::chunk_t::map *buf_pool_t::chunk_t::map_reg; @@ -698,7 +704,7 @@ static void buf_page_check_lsn(bool check_lsn, const byte* read_buf) @return whether the buffer is all zeroes */ bool buf_is_zeroes(span buf) { - ut_ad(buf.size() <= sizeof field_ref_zero); + ut_ad(buf.size() <= UNIV_PAGE_SIZE_MAX); return memcmp(buf.data(), field_ref_zero, buf.size()) == 0; } @@ -1391,11 +1397,17 @@ bool buf_pool_t::create() ut_ad(srv_buf_pool_size % srv_buf_pool_chunk_unit == 0); ut_ad(!is_initialised()); ut_ad(srv_buf_pool_size > 0); + ut_ad(!resizing); + ut_ad(!chunks_old); + ut_ad(!field_ref_zero); NUMA_MEMPOLICY_INTERLEAVE_IN_SCOPE; - ut_ad(!resizing); - ut_ad(!chunks_old); + if (auto b= aligned_malloc(UNIV_PAGE_SIZE_MAX, 4096)) + field_ref_zero= static_cast + (memset_aligned<4096>(b, 0, UNIV_PAGE_SIZE_MAX)); + else + return true; chunk_t::map_reg= UT_NEW_NOKEY(chunk_t::map()); @@ -1426,6 +1438,8 @@ bool buf_pool_t::create() chunks= nullptr; UT_DELETE(chunk_t::map_reg); chunk_t::map_reg= nullptr; + aligned_free(const_cast(field_ref_zero)); + field_ref_zero= nullptr; ut_ad(!is_initialised()); return true; } @@ -1541,6 +1555,8 @@ void buf_pool_t::close() io_buf.close(); UT_DELETE(chunk_t::map_reg); chunk_t::map_reg= chunk_t::map_ref= nullptr; + aligned_free(const_cast(field_ref_zero)); + field_ref_zero= nullptr; } /** Try to reallocate a control block. diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 9d94aa5b06207..caae4035f7cd8 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -526,7 +526,7 @@ inline bool dict_table_t::instant_column(const dict_table_t& table, } DBUG_ASSERT(c.is_added()); - if (c.def_val.len <= sizeof field_ref_zero + if (c.def_val.len <= UNIV_PAGE_SIZE_MAX && (!c.def_val.len || !memcmp(c.def_val.data, field_ref_zero, c.def_val.len))) { diff --git a/storage/innobase/include/buf0types.h b/storage/innobase/include/buf0types.h index ba1e2e5eaa621..5dd581097f971 100644 --- a/storage/innobase/include/buf0types.h +++ b/storage/innobase/include/buf0types.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2015, Oracle and/or its affiliates. All rights reserved. -Copyright (c) 2019, 2020, MariaDB Corporation. +Copyright (c) 2019, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -177,11 +177,11 @@ class page_id_t uint64_t m_id; }; -/** A field reference full of zero, for use in assertions and checks, +/** A 64KiB buffer of NUL bytes, for use in assertions and checks, and dummy default values of instantly dropped columns. -Initially, BLOB field references are set to zero, in +Initially, BLOB field references are set to NUL bytes, in dtuple_convert_big_rec(). */ -extern const byte field_ref_zero[UNIV_PAGE_SIZE_MAX]; +extern const byte *field_ref_zero; #ifndef UNIV_INNOCHECKSUM diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index fbcc30ee7caf4..460acaf5b664d 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -617,9 +617,8 @@ struct log_t{ /*!< number of currently pending checkpoint writes */ - /** buffer for checkpoint header */ - MY_ALIGNED(OS_FILE_LOG_BLOCK_SIZE) - byte checkpoint_buf[OS_FILE_LOG_BLOCK_SIZE]; + /** buffer for checkpoint header */ + byte *checkpoint_buf; /* @} */ private: diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index ece26ded72eb0..a6fa50dd753f0 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -219,6 +219,8 @@ void log_t::create() log_block_set_first_rec_group(buf, LOG_BLOCK_HDR_SIZE); buf_free= LOG_BLOCK_HDR_SIZE; + checkpoint_buf= static_cast + (aligned_malloc(OS_FILE_LOG_BLOCK_SIZE, OS_FILE_LOG_BLOCK_SIZE)); } mapped_file_t::~mapped_file_t() noexcept @@ -461,8 +463,8 @@ void log_t::file::write_header_durable(lsn_t lsn) ut_ad(log_sys.log.format == log_t::FORMAT_10_5 || log_sys.log.format == log_t::FORMAT_ENC_10_5); - // man 2 open suggests this buffer to be aligned by 512 for O_DIRECT - MY_ALIGNED(OS_FILE_LOG_BLOCK_SIZE) byte buf[OS_FILE_LOG_BLOCK_SIZE] = {0}; + byte *buf= log_sys.checkpoint_buf; + memset_aligned(buf, 0, OS_FILE_LOG_BLOCK_SIZE); mach_write_to_4(buf + LOG_HEADER_FORMAT, log_sys.log.format); mach_write_to_4(buf + LOG_HEADER_SUBFORMAT, log_sys.log.subformat); @@ -475,7 +477,7 @@ void log_t::file::write_header_durable(lsn_t lsn) DBUG_PRINT("ib_log", ("write " LSN_PF, lsn)); - log_sys.log.write(0, buf); + log_sys.log.write(0, {buf, OS_FILE_LOG_BLOCK_SIZE}); if (!log_sys.log.writes_are_durable()) log_sys.log.flush(); } @@ -880,7 +882,7 @@ ATTRIBUTE_COLD void log_write_checkpoint_info(lsn_t end_lsn) log_sys.next_checkpoint_lsn)); byte* buf = log_sys.checkpoint_buf; - memset(buf, 0, OS_FILE_LOG_BLOCK_SIZE); + memset_aligned(buf, 0, OS_FILE_LOG_BLOCK_SIZE); mach_write_to_8(buf + LOG_CHECKPOINT_NO, log_sys.next_checkpoint_no); mach_write_to_8(buf + LOG_CHECKPOINT_LSN, log_sys.next_checkpoint_lsn); @@ -1282,18 +1284,21 @@ void log_t::close() { ut_ad(this == &log_sys); if (!is_initialised()) return; - m_initialised = false; + m_initialised= false; log.close(); ut_free_dodump(buf, srv_log_buffer_size); - buf = NULL; + buf= nullptr; ut_free_dodump(flush_buf, srv_log_buffer_size); - flush_buf = NULL; + flush_buf= nullptr; mysql_mutex_destroy(&mutex); mysql_mutex_destroy(&flush_order_mutex); recv_sys.close(); + + aligned_free(checkpoint_buf); + checkpoint_buf= nullptr; } std::string get_log_file_path(const char *filename) diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index 1ed9efafc6990..74989fb018dc7 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -35,12 +35,6 @@ Created June 2005 by Marko Makela using st_::span; -/** A BLOB field reference full of zero, for use in assertions and tests. -Initially, BLOB field references are set to zero, in -dtuple_convert_big_rec(). */ -alignas(UNIV_PAGE_SIZE_MIN) -const byte field_ref_zero[UNIV_PAGE_SIZE_MAX] = { 0, }; - #ifndef UNIV_INNOCHECKSUM #include "mtr0log.h" #include "dict0dict.h" @@ -92,16 +86,12 @@ static const byte supremum_extra_data alignas(4) [] = { }; /** Assert that a block of memory is filled with zero bytes. -Compare at most sizeof(field_ref_zero) bytes. @param b in: memory block @param s in: size of the memory block, in bytes */ -#define ASSERT_ZERO(b, s) \ - ut_ad(!memcmp(b, field_ref_zero, \ - std::min(s, sizeof field_ref_zero))); +#define ASSERT_ZERO(b, s) ut_ad(!memcmp(b, field_ref_zero, s)) /** Assert that a BLOB pointer is filled with zero bytes. @param b in: BLOB pointer */ -#define ASSERT_ZERO_BLOB(b) \ - ut_ad(!memcmp(b, field_ref_zero, FIELD_REF_SIZE)) +#define ASSERT_ZERO_BLOB(b) ASSERT_ZERO(b, FIELD_REF_SIZE) /* Enable some extra debugging output. This code can be enabled independently of any UNIV_ debugging conditions. */ diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 4b1926c0ec0df..96cc3aa413e79 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -3424,7 +3424,7 @@ row_ins_index_entry_set_vals( field->len = UNIV_SQL_NULL; field->type.prtype = DATA_BINARY_TYPE; } else { - ut_ad(col->len <= sizeof field_ref_zero); + ut_ad(col->len <= UNIV_PAGE_SIZE_MAX); ut_ad(ind_field->fixed_len <= col->len); dfield_set_data(field, field_ref_zero, ind_field->fixed_len); From a4dc9265792919eb1ad156780e1a93dc4c7c6ce8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 21 Jul 2021 15:44:32 +0300 Subject: [PATCH 21/22] MDEV-26193: Wake up purge less often Starting with commit 6e12ebd4a748eba738e08ad1d7f5dec782ff63ee (MDEV-25062), srv_wake_purge_thread_if_not_active() became more expensive operation, especially on NUMA systems, because instead of reading an atomic global variable trx_sys.rseg_history_len we are traversing up to 128 cache lines in trx_sys.history_exists(). trx_t::commit_cleanup(): Do not wake up purge at all. We will wake up purge about once per second in srv_master_callback(). srv_master_do_active_tasks(), srv_master_do_idle_tasks(): Move some duplicated code to srv_master_callback(). srv_master_callback(): Invoke purge_coordinator_timer_callback() to ensure that purge will be periodically woken up, even if the latest execution of trx_t::commit_cleanup() allowed the purge view to advance but did not wake up purge. Do not call log_free_check(), because every thread that is going to generate redo log is supposed to call that function anyway, before acquiring any page latches. Additional calls to the function every few seconds should not make any difference. srv_shutdown_threads(): Ensure that srv_shutdown_state can be at most SRV_SHUTDOWN_INITIATED in srv_master_callback(), by first invoking srv_master_timer.reset() before changing srv_shutdown_state. (Note: We first terminate the srv_master_callback and only then terminate the purge tasks. Thus, the purge subsystem should exist when srv_master_callback() invokes purge_coordinator_timer_callback() if it was initiated in the first place. --- storage/innobase/srv/srv0srv.cc | 100 ++++++------------------------ storage/innobase/srv/srv0start.cc | 2 +- storage/innobase/trx/trx0trx.cc | 2 - 3 files changed, 20 insertions(+), 84 deletions(-) diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 8692eaa0e8925..81e0a3d2b7968 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1552,97 +1552,34 @@ void srv_master_thread_enable() } #endif /* UNIV_DEBUG */ -/*********************************************************************//** -Perform the tasks that the master thread is supposed to do when the -server is active. There are two types of tasks. The first category is -of such tasks which are performed at each inovcation of this function. -We assume that this function is called roughly every second when the -server is active. The second category is of such tasks which are -performed at some interval e.g.: purge, dict_LRU cleanup etc. */ -static -void -srv_master_do_active_tasks(void) -/*============================*/ +/** Perform periodic tasks whenever the server is active. +@param counter_time microsecond_interval_timer() */ +static void srv_master_do_active_tasks(ulonglong counter_time) { - time_t cur_time = time(NULL); - ulonglong counter_time = microsecond_interval_timer(); - - /* First do the tasks that we are suppose to do at each - invocation of this function. */ - ++srv_main_active_loops; MONITOR_INC(MONITOR_MASTER_ACTIVE_LOOPS); - ut_d(srv_master_do_disabled_loop()); - - if (srv_shutdown_state > SRV_SHUTDOWN_INITIATED) { - return; - } - - /* make sure that there is enough reusable space in the redo - log files */ - srv_main_thread_op_info = "checking free log space"; - log_free_check(); - - /* Flush logs if needed */ - srv_main_thread_op_info = "flushing log"; - srv_sync_log_buffer_in_background(); - MONITOR_INC_TIME_IN_MICRO_SECS( - MONITOR_SRV_LOG_FLUSH_MICROSECOND, counter_time); - - /* Now see if various tasks that are performed at defined - intervals need to be performed. */ - - if (srv_shutdown_state > SRV_SHUTDOWN_INITIATED) { - return; - } - - if (cur_time % SRV_MASTER_DICT_LRU_INTERVAL == 0) { + if (!(counter_time % (SRV_MASTER_DICT_LRU_INTERVAL * 1000000ULL))) { srv_main_thread_op_info = "enforcing dict cache limit"; - ulint n_evicted = dict_sys.evict_table_LRU(true); - if (n_evicted != 0) { + if (ulint n_evicted = dict_sys.evict_table_LRU(true)) { MONITOR_INC_VALUE( - MONITOR_SRV_DICT_LRU_EVICT_COUNT_ACTIVE, n_evicted); + MONITOR_SRV_DICT_LRU_EVICT_COUNT_ACTIVE, + n_evicted); } MONITOR_INC_TIME_IN_MICRO_SECS( MONITOR_SRV_DICT_LRU_MICROSECOND, counter_time); } } -/*********************************************************************//** -Perform the tasks that the master thread is supposed to do whenever the -server is idle. We do check for the server state during this function -and if the server has entered the shutdown phase we may return from -the function without completing the required tasks. -Note that the server can move to active state when we are executing this -function but we don't check for that as we are suppose to perform more -or less same tasks when server is active. */ -static -void -srv_master_do_idle_tasks(void) -/*==========================*/ +/** Perform periodic tasks whenever the server is idle. +@param counter_time microsecond_interval_timer() */ +static void srv_master_do_idle_tasks(ulonglong counter_time) { ++srv_main_idle_loops; MONITOR_INC(MONITOR_MASTER_IDLE_LOOPS); - ut_d(srv_master_do_disabled_loop()); - - if (srv_shutdown_state > SRV_SHUTDOWN_INITIATED) { - return; - } - - /* make sure that there is enough reusable space in the redo - log files */ - srv_main_thread_op_info = "checking free log space"; - log_free_check(); - - if (srv_shutdown_state > SRV_SHUTDOWN_INITIATED) { - return; - } - - ulonglong counter_time = microsecond_interval_timer(); srv_main_thread_op_info = "enforcing dict cache limit"; if (ulint n_evicted = dict_sys.evict_table_LRU(false)) { MONITOR_INC_VALUE( @@ -1650,11 +1587,6 @@ srv_master_do_idle_tasks(void) } MONITOR_INC_TIME_IN_MICRO_SECS( MONITOR_SRV_DICT_LRU_MICROSECOND, counter_time); - - /* Flush logs if needed */ - srv_sync_log_buffer_in_background(); - MONITOR_INC_TIME_IN_MICRO_SECS( - MONITOR_SRV_LOG_FLUSH_MICROSECOND, counter_time); } /** @@ -1695,12 +1627,18 @@ void srv_master_callback(void*) ut_a(srv_shutdown_state <= SRV_SHUTDOWN_INITIATED); - srv_main_thread_op_info = ""; MONITOR_INC(MONITOR_MASTER_THREAD_SLEEP); + ut_d(srv_master_do_disabled_loop()); + purge_coordinator_timer_callback(nullptr); + ulonglong counter_time = microsecond_interval_timer(); + srv_sync_log_buffer_in_background(); + MONITOR_INC_TIME_IN_MICRO_SECS( + MONITOR_SRV_LOG_FLUSH_MICROSECOND, counter_time); + if (srv_check_activity(&old_activity_count)) { - srv_master_do_active_tasks(); + srv_master_do_active_tasks(counter_time); } else { - srv_master_do_idle_tasks(); + srv_master_do_idle_tasks(counter_time); } srv_main_thread_op_info = "sleeping"; } diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index bac9a3b25b5e7..3f0ccf44b1e67 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -812,9 +812,9 @@ srv_open_tmp_tablespace(bool create_new_db) static void srv_shutdown_threads() { ut_ad(!srv_undo_sources); - srv_shutdown_state = SRV_SHUTDOWN_EXIT_THREADS; ut_d(srv_master_thread_enable()); srv_master_timer.reset(); + srv_shutdown_state = SRV_SHUTDOWN_EXIT_THREADS; if (purge_sys.enabled()) { srv_purge_shutdown(); diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 66ca04bbf0429..3763ed67c98f5 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1392,8 +1392,6 @@ void trx_t::commit_cleanup() mutex.wr_unlock(); ut_a(error_state == DB_SUCCESS); - if (!srv_read_only_mode) - srv_wake_purge_thread_if_not_active(); } /** Commit the transaction in a mini-transaction. From 5f4314f0e6033e81361641a6731c0ef1656cc35c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 22 Jul 2021 11:17:44 +0300 Subject: [PATCH 22/22] MDEV-26210: Disable mariabackup.log_page_corruption MDEV-26193 exposed a race condition in the test mariabackup.log_page_corruption by no longer waking up purge tasks on every transaction commit. (Note that there never was a guarantee that the purge tasks would actually run as quickly as the test expected it to happen; that would depend on the CPU load.) --- mysql-test/suite/mariabackup/disabled.def | 1 + 1 file changed, 1 insertion(+) create mode 100644 mysql-test/suite/mariabackup/disabled.def diff --git a/mysql-test/suite/mariabackup/disabled.def b/mysql-test/suite/mariabackup/disabled.def new file mode 100644 index 0000000000000..d272540cec8ef --- /dev/null +++ b/mysql-test/suite/mariabackup/disabled.def @@ -0,0 +1 @@ +log_page_corruption : MDEV-26210