Improve my.cnf draft

The binlog tests still fail.

Signed-off-by: Morgan Tocker <tocker@gmail.com>

config/mycnf/backup.cnf

@@ -1 +0,0 @@
-# reserved for future tuning

config/mycnf/benchmark.cnf

@@ -1,7 +0,0 @@
-innodb_doublewrite=0
-innodb_flush_log_at_trx_commit=0
-innodb_log_file_size=128M
-innodb_buffer_pool_size=1G
-max_connections=500
-open_files_limit=8192
-sync_binlog=0

config/mycnf/default-fast.cnf

@@ -1,37 +1,19 @@
-# basic config parameters for all db instances in the grid
+# This sets some unsafe settings specifically for 
+# the test-suite which is currently MySQL 5.7 based
+# In future it should be renamed testsuite.cnf
 
-sql_mode = STRICT_TRANS_TABLES
-character_set_server = utf8
-collation_server = utf8_general_ci
-connect_timeout = 30
-datadir = {{.DataDir}}
-expire_logs_days = 3
-innodb_buffer_pool_size = 64M
-innodb_data_home_dir = {{.InnodbDataHomeDir}}
-innodb_flush_log_at_trx_commit = 2
-innodb_lock_wait_timeout = 20
+innodb_buffer_pool_size = 32M
+innodb_flush_log_at_trx_commit = 0
 innodb_log_buffer_size = 1M
-innodb_log_file_size = 4M
-innodb_log_group_home_dir = {{.InnodbLogGroupHomeDir}}
+innodb_log_file_size = 5M
+
+# Native AIO tends to run into aio-max-nr limit during test startup.
+innodb_use_native_aio = 0
+
 key_buffer_size = 2M
-log-error = {{.ErrorLogPath}}
-long_query_time = 2
-pid-file = {{.PidFile}}
-port = {{.MysqlPort}}
-# all db instances should start in read-only mode - once the db is started and
-# fully functional, we'll push it into read-write mode
-read-only
-server-id = {{.ServerID}}
-skip-name-resolve
-# we now need networking for replication. this is a tombstone to simpler times.
-#skip_networking
-# all db instances should skip the slave startup - that way we can do any
-# out-of-bounds checking before we restart everything - in case we need to do
-# some extra work to skip mangled transactions or fudge the slave start
-skip_slave_start
-slave_net_timeout = 60
-slave_load_tmpdir = {{.SlaveLoadTmpDir}}
-slow-query-log
-slow-query-log-file = {{.SlowLogPath}}
-socket = {{.SocketFile}}
-tmpdir = {{.TmpDir}}
+sync_binlog=0
+innodb_doublewrite=0
+
+# Some tests don't work with full strict yet
+sql_mode='STRICT_TRANS_TABLES'
+

config/mycnf/default.cnf

@@ -1,35 +1,35 @@
-# basic config parameters for all db instances in the grid
+# Global configuration that is auto-included for all MySQL/MariaDB versions
 
-sql_mode = STRICT_TRANS_TABLES
-binlog_format = statement
-character_set_server = utf8
-collation_server = utf8_general_ci
-connect_timeout = 30
 datadir = {{.DataDir}}
-expire_logs_days = 3
-innodb_buffer_pool_size = 32M
 innodb_data_home_dir = {{.InnodbDataHomeDir}}
-innodb_flush_log_at_trx_commit = 2
-innodb_lock_wait_timeout = 20
 innodb_log_group_home_dir = {{.InnodbLogGroupHomeDir}}
 log-error = {{.ErrorLogPath}}
-long_query_time = 2
-max_allowed_packet = 64M
-max_connections = 500
 pid-file = {{.PidFile}}
 port = {{.MysqlPort}}
+
 # all db instances should start in read-only mode - once the db is started and
 # fully functional, we'll push it into read-write mode
 read-only
 server-id = {{.ServerID}}
-skip-name-resolve
+
 # all db instances should skip the slave startup - that way we can do any
 # additional configuration (like enabling semi-sync) before we connect to
 # the master.
 skip_slave_start
-slave_net_timeout = 60
 slave_load_tmpdir = {{.SlaveLoadTmpDir}}
-slow-query-log
-slow-query-log-file = {{.SlowLogPath}}
 socket = {{.SocketFile}}
 tmpdir = {{.TmpDir}}
+
+slow-query-log-file = {{.SlowLogPath}}
+
+# These are sensible defaults that apply to all MySQL/MariaDB versions
+
+long_query_time = 2
+slow-query-log
+skip-name-resolve
+connect_timeout = 30
+innodb_lock_wait_timeout = 20
+max_allowed_packet = 64M
+
+
+

config/mycnf/master.cnf

@@ -1,5 +0,0 @@
-# master.cnf parameters
-
-log-bin = {{.BinLogPath}}
-log-slave-updates
-sync_binlog = 1

config/mycnf/master_mariadb.cnf

@@ -1,12 +0,0 @@
-# This file is auto-included when MariaDB (any version) is detected.
-
-# enable strict mode so it's safe to compare sequence numbers across different server IDs.
-gtid_strict_mode = 1
-innodb_stats_persistent = 0
-
-# When semi-sync is enabled, don't allow fallback to async
-# if you get no ack, or have no slaves. This is necessary to
-# prevent alternate futures when doing a failover in response to
-# a master that becomes unresponsive.
-rpl_semi_sync_master_timeout = 1000000000000000000
-rpl_semi_sync_master_wait_no_slave = 1

config/mycnf/master_mariadb100.cnf

@@ -1,7 +1,5 @@
 # This file is auto-included when MariaDB 10.0 is detected.
 
-innodb_support_xa = 0
-
 # Semi-sync replication is required for automated unplanned failover
 # (when the master goes away). Here we just load the plugin so it's
 # available if desired, but it's disabled at startup.
@@ -10,3 +8,38 @@ innodb_support_xa = 0
 # at the proper time when replication is set up, or when masters are
 # promoted or demoted.
 plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so
+
+slave_net_timeout = 60
+
+# MariaDB 10.0 is unstrict by default
+sql_mode = STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION
+
+# enable strict mode so it's safe to compare sequence numbers across different server IDs.
+gtid_strict_mode = 1
+innodb_stats_persistent = 0
+
+# When semi-sync is enabled, don't allow fallback to async
+# if you get no ack, or have no slaves. This is necessary to
+# prevent alternate futures when doing a failover in response to
+# a master that becomes unresponsive.
+rpl_semi_sync_master_timeout = 1000000000000000000
+rpl_semi_sync_master_wait_no_slave = 1
+
+
+character_set_server = utf8
+collation_server = utf8_general_ci
+
+expire_logs_days = 3
+
+log_bin
+sync_binlog = 1
+binlog_format = ROW
+log_slave_updates
+expire_logs_days = 3
+
+# In MariaDB the default charset is latin1
+
+character_set_server = utf8
+collation_server = utf8_general_ci
+
+

config/mycnf/master_mariadb101.cnf

@@ -1,7 +1,5 @@
 # This file is auto-included when MariaDB 10.1 is detected.
 
-innodb_support_xa = 0
-
 # Semi-sync replication is required for automated unplanned failover
 # (when the master goes away). Here we just load the plugin so it's
 # available if desired, but it's disabled at startup.
@@ -10,3 +8,38 @@ innodb_support_xa = 0
 # at the proper time when replication is set up, or when masters are
 # promoted or demoted.
 plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so
+
+slave_net_timeout = 60
+
+# MariaDB 10.1 default is only no-engine-substitution and no-auto-create-user
+sql_mode = STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION,NO_AUTO_CREATE_USER
+
+# enable strict mode so it's safe to compare sequence numbers across different server IDs.
+gtid_strict_mode = 1
+innodb_stats_persistent = 0
+
+# When semi-sync is enabled, don't allow fallback to async
+# if you get no ack, or have no slaves. This is necessary to
+# prevent alternate futures when doing a failover in response to
+# a master that becomes unresponsive.
+rpl_semi_sync_master_timeout = 1000000000000000000
+rpl_semi_sync_master_wait_no_slave = 1
+
+
+character_set_server = utf8
+collation_server = utf8_general_ci
+
+expire_logs_days = 3
+
+log_bin
+sync_binlog = 1
+binlog_format = ROW
+log_slave_updates
+expire_logs_days = 3
+
+# In MariaDB the default charset is latin1
+
+character_set_server = utf8
+collation_server = utf8_general_ci
+
+

config/mycnf/master_mariadb102.cnf

@@ -1,7 +1,5 @@
 # This file is auto-included when MariaDB 10.2 is detected.
 
-innodb_support_xa = 0
-
 # Semi-sync replication is required for automated unplanned failover
 # (when the master goes away). Here we just load the plugin so it's
 # available if desired, but it's disabled at startup.
@@ -10,3 +8,33 @@ innodb_support_xa = 0
 # at the proper time when replication is set up, or when masters are
 # promoted or demoted.
 plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so
+
+# enable strict mode so it's safe to compare sequence numbers across different server IDs.
+gtid_strict_mode = 1
+innodb_stats_persistent = 0
+
+# When semi-sync is enabled, don't allow fallback to async
+# if you get no ack, or have no slaves. This is necessary to
+# prevent alternate futures when doing a failover in response to
+# a master that becomes unresponsive.
+rpl_semi_sync_master_timeout = 1000000000000000000
+rpl_semi_sync_master_wait_no_slave = 1
+
+
+character_set_server = utf8
+collation_server = utf8_general_ci
+
+expire_logs_days = 3
+
+log_bin
+sync_binlog = 1
+binlog_format = ROW
+log_slave_updates
+expire_logs_days = 3
+
+# In MariaDB the default charset is latin1
+
+character_set_server = utf8
+collation_server = utf8_general_ci
+
+

config/mycnf/master_mariadb103.cnf

@@ -4,20 +4,28 @@
 gtid_strict_mode = 1
 innodb_stats_persistent = 0
 
-# Semi-sync replication is required for automated unplanned failover
-# (when the master goes away). Here we just load the plugin so it's
-# available if desired, but it's disabled at startup.
-#
-# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync
-# at the proper time when replication is set up, or when masters are
-# promoted or demoted.
-
-# semi_sync has been merged into master as of mariadb 10.3 so this is no longer needed 
-#plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so
-
 # When semi-sync is enabled, don't allow fallback to async
 # if you get no ack, or have no slaves. This is necessary to
 # prevent alternate futures when doing a failover in response to
 # a master that becomes unresponsive.
 rpl_semi_sync_master_timeout = 1000000000000000000
 rpl_semi_sync_master_wait_no_slave = 1
+
+
+character_set_server = utf8
+collation_server = utf8_general_ci
+
+expire_logs_days = 3
+
+log_bin
+sync_binlog = 1
+binlog_format = ROW
+log_slave_updates
+expire_logs_days = 3
+
+# In MariaDB the default charset is latin1
+
+character_set_server = utf8
+collation_server = utf8_general_ci
+
+

config/mycnf/master_mysql56.cnf

@@ -1,20 +1,29 @@
 # This file is auto-included when MySQL 5.6 is detected.
 
-# Options for enabling GTID
-# https://dev.mysql.com/doc/refman/5.6/en/replication-gtids-howto.html
-gtid_mode = ON
+# MySQL 5.6 does not enable the binary log by default, and 
+# the default for sync_binlog is unsafe. The format is TABLE, and
+# info repositories also default to file.
+
 log_bin
+sync_binlog = 1
+gtid_mode = ON
+binlog_format = ROW
 log_slave_updates
 enforce_gtid_consistency
-
-# Crash-safe replication settings.
+expire_logs_days = 3
 master_info_repository = TABLE
 relay_log_info_repository = TABLE
 relay_log_purge = 1
 relay_log_recovery = 1
+slave_net_timeout = 60
 
-# Native AIO tends to run into aio-max-nr limit during test startup.
-innodb_use_native_aio = 0
+# In MySQL 5.6 the default charset is latin1
+
+character_set_server = utf8
+collation_server = utf8_general_ci
+
+# MySQL 5.6 is unstrict by default
+sql_mode = STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION
 
 # Semi-sync replication is required for automated unplanned failover
 # (when the master goes away). Here we just load the plugin so it's
@@ -31,3 +40,4 @@ plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisy
 # a master that becomes unresponsive.
 rpl_semi_sync_master_timeout = 1000000000000000000
 rpl_semi_sync_master_wait_no_slave = 1
+

config/mycnf/master_mysql57.cnf

@@ -1,19 +1,23 @@
 # This file is auto-included when MySQL 5.7 is detected.
 
-# Options for enabling GTID
-# https://dev.mysql.com/doc/refman/5.6/en/replication-gtids-howto.html
+# MySQL 5.7 does not enable the binary log by default, and 
+# info repositories default to file
+
 gtid_mode = ON
 log_bin
 log_slave_updates
 enforce_gtid_consistency
-innodb_use_native_aio = 0
-
-# Crash-safe replication settings.
+expire_logs_days = 3
 master_info_repository = TABLE
 relay_log_info_repository = TABLE
 relay_log_purge = 1
 relay_log_recovery = 1
 
+# In MySQL 5.7 the default charset is latin1
+
+character_set_server = utf8
+collation_server = utf8_general_ci
+
 # Semi-sync replication is required for automated unplanned failover
 # (when the master goes away). Here we just load the plugin so it's
 # available if desired, but it's disabled at startup.

config/mycnf/master_mysql80.cnf

@@ -1,20 +1,18 @@
 # This file is auto-included when MySQL 8.0 is detected.
 
-# Options for enabling GTID
-# https://dev.mysql.com/doc/refman/5.6/en/replication-gtids-howto.html
+# MySQL 8.0 enables binlog by default with sync_binlog and TABLE info repositories
+# It does not enable GTIDs or enforced GTID consistency
+
 gtid_mode = ON
-log_bin
-log_slave_updates
 enforce_gtid_consistency
-
-# Crash-safe replication settings.
-master_info_repository = TABLE
-relay_log_info_repository = TABLE
-relay_log_purge = 1
 relay_log_recovery = 1
+binlog_expire_logs_seconds = 259200
 
-# Native AIO tends to run into aio-max-nr limit during test startup.
-innodb_use_native_aio = 0
+# disable mysqlx
+mysqlx = 0
+
+# 8.0 changes the default auth-plugin to caching_sha2_password
+default_authentication_plugin = mysql_native_password
 
 # Semi-sync replication is required for automated unplanned failover
 # (when the master goes away). Here we just load the plugin so it's
@@ -32,9 +30,3 @@ plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisy
 rpl_semi_sync_master_timeout = 1000000000000000000
 rpl_semi_sync_master_wait_no_slave = 1
 
-# disable mysqlx
-mysqlx = 0
-
-# 8.0 changes the default auth-plugin to caching_sha2_password
-default_authentication_plugin = mysql_native_password
-secure_file_priv = NULL

config/mycnf/production.cnf

@@ -1,5 +0,0 @@
-# Values for a production vitess deployment
-innodb_buffer_pool_size = 1024M
-innodb_log_file_size = 512M
-innodb_log_buffer_size = 64M
-max_connections = 1000

config/mycnf/rbr.cnf

@@ -1 +1,3 @@
-binlog-format=row
+# This file is used to allow legacy tests to pass
+# In theory it should not be required
+binlog_format=row

config/mycnf/rdonly.cnf

@@ -1 +0,0 @@
-# reserved for future tuning

config/mycnf/replica.cnf

@@ -1,13 +0,0 @@
-# replica.cnf - reserved for future tuning
-
-relay-log = {{.RelayLogPath}}
-relay-log-index =  {{.RelayLogIndexPath}}
-relay-log-info-file =  {{.RelayLogInfoPath}}
-master-info-file = {{.MasterInfoFile}}
-
-# required if this master is chained
-# probably safe to turn on all the time at the expense of some disk I/O
-# note: this is in the master conf too
-log-slave-updates
-
-#slave_compressed_protocol

config/mycnf/sbr.cnf

@@ -0,0 +1,2 @@
+# This file is used to allow legacy tests to pass
+binlog_format=statement

config/mycnf/vtcombo.cnf

@@ -1 +0,0 @@
-max_connections = 5000

examples/local/vttablet-up.sh

@@ -38,11 +38,6 @@ source $script_root/env.sh
 
 init_db_sql_file="$VTROOT/config/init_db.sql"
 
-# Previously this file set EXTRA_MY_CNF based on MYSQL_FLAVOR
-# It now relies on mysqlctl to autodetect
-
-export EXTRA_MY_CNF=$VTROOT/config/mycnf/default-fast.cnf:$VTROOT/config/mycnf/rbr.cnf
-
 mkdir -p $VTDATAROOT/backups
 
 # Start 3 vttablets by default.

go/vt/mysqlctl/mycnf_test.go

@@ -43,8 +43,6 @@ func TestMycnf(t *testing.T) {
 	}
 	cnfTemplatePaths := []string{
 		path.Join(root, "src/vitess.io/vitess/config/mycnf/default.cnf"),
-		path.Join(root, "src/vitess.io/vitess/config/mycnf/replica.cnf"),
-		path.Join(root, "src/vitess.io/vitess/config/mycnf/master.cnf"),
 	}
 	data, err := cnf.makeMycnf(cnfTemplatePaths)
 	if err != nil {

go/vt/mysqlctl/mysqld.go

@@ -762,8 +762,6 @@ func (mysqld *Mysqld) getMycnfTemplates(root string) []string {
 
 	cnfTemplatePaths := []string{
 		path.Join(root, "config/mycnf/default.cnf"),
-		path.Join(root, "config/mycnf/master.cnf"),
-		path.Join(root, "config/mycnf/replica.cnf"),
 	}
 
 	if extraCnf := os.Getenv("EXTRA_MY_CNF"); extraCnf != "" {
@@ -771,24 +769,14 @@ func (mysqld *Mysqld) getMycnfTemplates(root string) []string {
 		cnfTemplatePaths = append(cnfTemplatePaths, parts...)
 	}
 
-	// Only include these files if they exist.
-	// master_{flavor}.cnf
-	// Percona Server == MySQL in this context
-
 	f := flavorMariaDB
 	if mysqld.capabilities.isMySQLLike() {
 		f = flavorMySQL
 	}
 
-	p := path.Join(root, fmt.Sprintf("config/mycnf/master_%s.cnf", f))
-	_, err := os.Stat(p)
-	if err == nil && !contains(cnfTemplatePaths, p) {
-		cnfTemplatePaths = append(cnfTemplatePaths, p)
-	}
-
 	// master_{flavor}{major}{minor}.cnf
-	p = path.Join(root, fmt.Sprintf("config/mycnf/master_%s%d%d.cnf", f, mysqld.capabilities.version.Major, mysqld.capabilities.version.Minor))
-	_, err = os.Stat(p)
+	p := path.Join(root, fmt.Sprintf("config/mycnf/master_%s%d%d.cnf", f, mysqld.capabilities.version.Major, mysqld.capabilities.version.Minor))
+	_, err := os.Stat(p)
 	if err == nil && !contains(cnfTemplatePaths, p) {
 		cnfTemplatePaths = append(cnfTemplatePaths, p)
 	}

go/vt/vttest/environment.go

@@ -116,22 +116,20 @@ func GetMySQLOptions(flavor string) (string, []string, error) {
 		flavor = DefaultMySQLFlavor
 	}
 
-	mycnf := []string{"config/mycnf/vtcombo.cnf"}
+	mycnf := []string{}
 	switch flavor {
 	case "MariaDB103":
 		mycnf = append(mycnf, "config/mycnf/default-fast.cnf")
 		mycnf = append(mycnf, "config/mycnf/master_mariadb103.cnf")
 	case "MariaDB":
 		mycnf = append(mycnf, "config/mycnf/default-fast.cnf")
-		mycnf = append(mycnf, "config/mycnf/master_mariadb.cnf")
-
+		mycnf = append(mycnf, "config/mycnf/master_mariadb100.cnf")
 	case "MySQL80":
 		mycnf = append(mycnf, "config/mycnf/default-fast.cnf")
 		mycnf = append(mycnf, "config/mycnf/master_mysql80.cnf")
 	case "MySQL56":
 		mycnf = append(mycnf, "config/mycnf/default-fast.cnf")
 		mycnf = append(mycnf, "config/mycnf/master_mysql56.cnf")
-
 	default:
 		return "", nil, fmt.Errorf("unknown mysql flavor: %s", flavor)
 	}

py/vttest/mysql_flavor.py

@@ -49,7 +49,7 @@ class MariaDB(MysqlFlavor):
   def my_cnf(self):
     files = [
         os.path.join(vttop, "config/mycnf/default-fast.cnf"),
-        os.path.join(vttop, "config/mycnf/master_mariadb.cnf"),
+        os.path.join(vttop, "config/mycnf/master_mariadb100.cnf"),
     ]
     return ":".join(files)
 

py/vttest/run_local_database.py

@@ -98,7 +98,7 @@ def main(cmdline_options):
     init_data_opts.max_table_shard_size = cmdline_options.max_table_shard_size
     init_data_opts.null_probability = cmdline_options.null_probability
 
-  extra_my_cnf = os.path.join(os.environ['VTTOP'], 'config/mycnf/vtcombo.cnf')
+  extra_my_cnf = ''
   if cmdline_options.extra_my_cnf:
     extra_my_cnf += ':' + cmdline_options.extra_my_cnf
 

test/mysql_flavor.py

@@ -126,7 +126,7 @@ class MariaDB(MysqlFlavor):
     ]
 
   def extra_my_cnf(self):
-    return environment.vttop + "/config/mycnf/master_mariadb.cnf"
+    return environment.vttop + "/config/mycnf/master_mariadb100.cnf"
 
   def master_position(self, tablet):
     gtid = tablet.mquery("", "SELECT @@GLOBAL.gtid_binlog_pos")[0][0]