Merge pull request #1712 from enisoc/mysql57

Support MySQL 5.7
This commit is contained in:
Anthony Yeh 2016-05-20 22:56:53 -07:00
Родитель 4af82f384e 12f29bc5a1
Коммит 9b93e53741
16 изменённых файлов: 167 добавлений и 47 удалений

Просмотреть файл

@ -1,4 +1,4 @@
FROM vitess/bootstrap:mysql56
FROM vitess/bootstrap:mysql57
# Re-copy sources from working tree
USER root

12
Dockerfile.mysql56 Normal file
Просмотреть файл

@ -0,0 +1,12 @@
FROM vitess/bootstrap:mysql56
# Re-copy sources from working tree
USER root
COPY . /vt/src/github.com/youtube/vitess
# Fix permissions
RUN chown -R vitess:vitess /vt
USER vitess
# Build Vitess
RUN make build

Просмотреть файл

@ -118,6 +118,7 @@ docker_bootstrap:
docker/bootstrap/build.sh common
docker/bootstrap/build.sh mariadb
docker/bootstrap/build.sh mysql56
docker/bootstrap/build.sh mysql57
docker/bootstrap/build.sh percona
docker_base:
@ -125,6 +126,10 @@ docker_base:
chmod -R o=g *
docker build -t vitess/base .
docker_base_mysql56:
chmod -R o=g *
docker build -f Dockerfile.percona -t vitess/base:mysql56 .
docker_base_percona:
chmod -R o=g *
docker build -f Dockerfile.percona -t vitess/base:percona .
@ -136,6 +141,9 @@ docker_base_mariadb:
docker_lite:
cd docker/lite && ./build.sh
docker_lite_mysql56:
cd docker/lite && ./build.sh mysql56
docker_lite_mariadb:
cd docker/lite && ./build.sh mariadb

Просмотреть файл

@ -154,14 +154,14 @@ if [ -z "$MYSQL_FLAVOR" ]; then
fi
case "$MYSQL_FLAVOR" in
"MySQL56")
myversion=`$VT_MYSQL_ROOT/bin/mysql --version | grep 'Distrib 5\.6'`
[ "$myversion" != "" ] || fail "Couldn't find MySQL 5.6 in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
echo "Found MySQL 5.6 installation in $VT_MYSQL_ROOT."
myversion=`$VT_MYSQL_ROOT/bin/mysql --version`
[[ "$myversion" =~ Distrib\ 5\.[67] ]] || fail "Couldn't find MySQL 5.6+ in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
echo "Found MySQL 5.6+ installation in $VT_MYSQL_ROOT."
;;
"MariaDB")
myversion=`$VT_MYSQL_ROOT/bin/mysql --version | grep MariaDB`
[ "$myversion" != "" ] || fail "Couldn't find MariaDB in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
myversion=`$VT_MYSQL_ROOT/bin/mysql --version`
[[ "$myversion" =~ MariaDB ]] || fail "Couldn't find MariaDB in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
echo "Found MariaDB installation in $VT_MYSQL_ROOT."
;;
@ -179,9 +179,10 @@ echo "$MYSQL_FLAVOR" > $VTROOT/dist/MYSQL_FLAVOR
[ -x $VT_MYSQL_ROOT/bin/mysql_config ] || fail "Cannot execute $VT_MYSQL_ROOT/bin/mysql_config. Did you install a client dev package?"
cp $VTTOP/config/gomysql.pc.tmpl $VTROOT/lib/gomysql.pc
echo "Version:" "$($VT_MYSQL_ROOT/bin/mysql_config --version)" >> $VTROOT/lib/gomysql.pc
myversion=`$VT_MYSQL_ROOT/bin/mysql_config --version`
echo "Version:" "$myversion" >> $VTROOT/lib/gomysql.pc
echo "Cflags:" "$($VT_MYSQL_ROOT/bin/mysql_config --cflags) -ggdb -fPIC" >> $VTROOT/lib/gomysql.pc
if [ "$MYSQL_FLAVOR" == "MariaDB" ]; then
if [[ "$MYSQL_FLAVOR" == "MariaDB" || "$myversion" =~ ^5\.7\. ]]; then
# Use static linking because the shared library doesn't export
# some internal functions we use, like cli_safe_read.
echo "Libs:" "$($VT_MYSQL_ROOT/bin/mysql_config --libs_r | sed -r 's,-lmysqlclient(_r)?,-l:libmysqlclient.a -lstdc++,')" >> $VTROOT/lib/gomysql.pc

Просмотреть файл

@ -9,7 +9,6 @@ connect_timeout = 30
datadir = {{.DataDir}}
default-storage-engine = innodb
expire_logs_days = 3
innodb_additional_mem_pool_size = 32M
innodb_autoextend_increment = 1
innodb_buffer_pool_size = 64M
innodb_data_file_path = ibdata1:10M:autoextend
@ -25,7 +24,7 @@ innodb_log_group_home_dir = {{.InnodbLogGroupHomeDir}}
innodb_max_dirty_pages_pct = 75
innodb_support_xa = 0
innodb_thread_concurrency = 2
key_buffer = 2M
key_buffer_size = 2M
log-error = {{.ErrorLogPath}}
long_query_time = 2
max_allowed_packet = 16M
@ -55,8 +54,7 @@ slow-query-log-file = {{.SlowLogPath}}
socket = {{.SocketFile}}
sort_buffer_size = 2M
table_open_cache = 2048
thread_cache = 200
thread_concurrency = 2
thread_cache_size = 200
tmpdir = {{.TmpDir}}
tmp_table_size = 32M
transaction-isolation = REPEATABLE-READ

Просмотреть файл

@ -9,7 +9,6 @@ connect_timeout = 30
datadir = {{.DataDir}}
default-storage-engine = innodb
expire_logs_days = 3
innodb_additional_mem_pool_size = 32M
innodb_autoextend_increment = 64
innodb_buffer_pool_size = 32M
innodb_data_file_path = ibdata1:10M:autoextend
@ -25,7 +24,7 @@ innodb_log_group_home_dir = {{.InnodbLogGroupHomeDir}}
innodb_max_dirty_pages_pct = 75
innodb_support_xa = 0
innodb_thread_concurrency = 20
key_buffer = 32M
key_buffer_size = 32M
log-error = {{.ErrorLogPath}}
long_query_time = 2
max_allowed_packet = 16M
@ -53,8 +52,7 @@ slow-query-log-file = {{.SlowLogPath}}
socket = {{.SocketFile}}
sort_buffer_size = 2M
table_open_cache = 2048
thread_cache = 200
thread_concurrency = 24
thread_cache_size = 200
tmpdir = {{.TmpDir}}
tmp_table_size = 32M
transaction-isolation = REPEATABLE-READ

Просмотреть файл

@ -4,3 +4,4 @@ gtid_mode = ON
log_bin
log_slave_updates
enforce_gtid_consistency
relay_log_recovery

Просмотреть файл

@ -0,0 +1,14 @@
FROM vitess/bootstrap:common
# Install MySQL 5.7
RUN apt-key adv --recv-keys --keyserver ha.pool.sks-keyservers.net 5072E1F5 && \
add-apt-repository 'deb http://repo.mysql.com/apt/debian/ jessie mysql-5.7' && \
apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev && \
rm -rf /var/lib/apt/lists/*
# Bootstrap Vitess
WORKDIR /vt/src/github.com/youtube/vitess
USER vitess
ENV MYSQL_FLAVOR MySQL56
RUN ./bootstrap.sh --skip_root_installs

Просмотреть файл

@ -3,7 +3,7 @@ FROM debian:jessie
# Install dependencies
RUN apt-key adv --recv-keys --keyserver pgp.mit.edu 5072E1F5 \
&& echo 'deb http://repo.mysql.com/apt/debian/ jessie mysql-5.6' > /etc/apt/sources.list.d/mysql.list \
&& echo 'deb http://repo.mysql.com/apt/debian/ jessie mysql-5.7' > /etc/apt/sources.list.d/mysql.list \
&& apt-get update \
&& DEBIAN_FRONTEND=noninteractive \
apt-get install -y --no-install-recommends \

Просмотреть файл

@ -0,0 +1,37 @@
# This image is only meant to be built from within the build.sh script.
FROM debian:jessie
# Install dependencies
RUN apt-key adv --recv-keys --keyserver pgp.mit.edu 5072E1F5 \
&& echo 'deb http://repo.mysql.com/apt/debian/ jessie mysql-5.6' > /etc/apt/sources.list.d/mysql.list \
&& apt-get update \
&& DEBIAN_FRONTEND=noninteractive \
apt-get install -y --no-install-recommends \
bzip2 \
memcached \
libmysqlclient18 \
mysql-client \
mysql-server \
&& rm -rf /var/lib/apt/lists/*
# Set up Vitess environment (just enough to run pre-built Go binaries)
ENV VTTOP /vt/src/github.com/youtube/vitess
ENV VTROOT /vt
ENV GOTOP $VTTOP/go
ENV VTDATAROOT $VTROOT/vtdataroot
ENV GOBIN $VTROOT/bin
ENV GOPATH $VTROOT
ENV PATH $VTROOT/bin:$PATH
ENV VT_MYSQL_ROOT /usr
ENV PKG_CONFIG_PATH $VTROOT/lib
ENV LD_LIBRARY_PATH $VTROOT/dist/vt-zookeeper-3.4.6/lib
# Copy binaries (placed by build.sh)
COPY lite/vt /vt
# Create vitess user
RUN groupadd -r vitess && useradd -r -g vitess vitess && \
mkdir -p /vt/vtdataroot && chown -R vitess:vitess /vt
# Create mount point for actual data (e.g. MySQL data dir)
VOLUME /vt/vtdataroot

Просмотреть файл

@ -162,7 +162,15 @@ unsigned long vt_cli_safe_read(VT_CONN *conn) {
unsigned long len;
mysql_thread_init();
#if MYSQL_VERSION_ID >= 50700 && MYSQL_VERSION_ID < 100000
// MySQL 5.7
len = cli_safe_read(conn->mysql, NULL);
#else
// MySQL 5.6 and MariaDB
len = cli_safe_read(conn->mysql);
#endif // MYSQL_VERSION_ID >= 50700 && MYSQL_VERSION_ID < 100000
return len == packet_error ? 0 : len;
}

Просмотреть файл

@ -27,8 +27,15 @@ my_socket vio_fd(Vio* vio);
#define vio_socket_shutdown(vio, how) shutdown(vio_fd(vio), how)
#endif
// cli_safe_read is declared in sql_common.h.
// cli_safe_read is declared in sql_common.h, which we can't assume
// we have, so we declare it manually.
#if MYSQL_VERSION_ID >= 50700 && MYSQL_VERSION_ID < 100000
// MySQL 5.7
unsigned long cli_safe_read(MYSQL *mysql, my_bool *is_data_packet);
#else
// MySQL 5.6 and MariaDB
unsigned long cli_safe_read(MYSQL *mysql);
#endif // MYSQL_VERSION_ID >= 50700 && MYSQL_VERSION_ID < 100000
// st_mysql_methods and simple_command are declared in mysql.h in
// Google MySQL 5.1 (VERSION_ID=501xx), but were moved to sql_common.h in

Просмотреть файл

@ -16,7 +16,7 @@ import (
"github.com/youtube/vitess/go/vt/mysqlctl/replication"
)
// mysql56 is the implementation of MysqlFlavor for MySQL 5.6.
// mysql56 is the implementation of MysqlFlavor for MySQL 5.6+.
type mysql56 struct {
}
@ -24,7 +24,7 @@ const mysql56FlavorID = "MySQL56"
// VersionMatch implements MysqlFlavor.VersionMatch().
func (*mysql56) VersionMatch(version string) bool {
return strings.HasPrefix(version, "5.6")
return strings.HasPrefix(version, "5.6") || strings.HasPrefix(version, "5.7")
}
// MasterPosition implements MysqlFlavor.MasterPosition().

Просмотреть файл

@ -323,7 +323,7 @@ func (mysqld *Mysqld) Shutdown(ctx context.Context, waitForMysqld bool) error {
// possibly mysql is already shutdown, check for a few files first
_, socketPathErr := os.Stat(mysqld.config.SocketFile)
_, pidPathErr := os.Stat(mysqld.config.PidFile)
if socketPathErr != nil && pidPathErr != nil {
if os.IsNotExist(socketPathErr) && os.IsNotExist(pidPathErr) {
log.Warningf("assuming mysqld already shut down - no socket, no pid file found")
return nil
}
@ -348,7 +348,7 @@ func (mysqld *Mysqld) Shutdown(ctx context.Context, waitForMysqld bool) error {
env := []string{
os.ExpandEnv("LD_LIBRARY_PATH=$VT_MYSQL_ROOT/lib/mysql"),
}
_, err = execCmd(name, arg, env, dir, nil)
_, _, err = execCmd(name, arg, env, dir, nil)
if err != nil {
return err
}
@ -357,10 +357,13 @@ func (mysqld *Mysqld) Shutdown(ctx context.Context, waitForMysqld bool) error {
return fmt.Errorf("mysqld_shutdown hook failed: %v", hr.String())
}
// Wait for mysqld to really stop. Use the sock file as a
// Wait for mysqld to really stop. Use the socket and pid files as a
// proxy for that since we can't call wait() in a process we
// didn't start.
if waitForMysqld {
log.Infof("Mysqld.Shutdown: waiting for socket file (%v) and pid file (%v) to disappear",
mysqld.config.SocketFile, mysqld.config.PidFile)
for {
select {
case <-ctx.Done():
@ -368,12 +371,12 @@ func (mysqld *Mysqld) Shutdown(ctx context.Context, waitForMysqld bool) error {
default:
}
_, statErr := os.Stat(mysqld.config.SocketFile)
if statErr != nil && os.IsNotExist(statErr) {
_, socketPathErr = os.Stat(mysqld.config.SocketFile)
_, pidPathErr = os.Stat(mysqld.config.PidFile)
if os.IsNotExist(socketPathErr) && os.IsNotExist(pidPathErr) {
return nil
}
log.Infof("Mysqld.Shutdown: sleeping for 1s waiting for socket file %v", mysqld.config.SocketFile)
time.Sleep(time.Second)
time.Sleep(100 * time.Millisecond)
}
}
return nil
@ -381,7 +384,7 @@ func (mysqld *Mysqld) Shutdown(ctx context.Context, waitForMysqld bool) error {
// execCmd searches the PATH for a command and runs it, logging the output.
// If input is not nil, pipe it to the command's stdin.
func execCmd(name string, args, env []string, dir string, input io.Reader) (cmd *exec.Cmd, err error) {
func execCmd(name string, args, env []string, dir string, input io.Reader) (cmd *exec.Cmd, output string, err error) {
cmdPath, _ := exec.LookPath(name)
log.Infof("execCmd: %v %v %v", name, cmdPath, args)
@ -391,12 +394,13 @@ func execCmd(name string, args, env []string, dir string, input io.Reader) (cmd
if input != nil {
cmd.Stdin = input
}
output, err := cmd.CombinedOutput()
out, err := cmd.CombinedOutput()
output = string(out)
if err != nil {
err = errors.New(name + ": " + string(output))
err = errors.New(name + ": " + output)
}
log.Infof("execCmd: command returned: %v", string(output))
return cmd, err
log.Infof("execCmd: command returned: %v", output)
return cmd, output, err
}
// Init will create the default directory structure for the mysqld process,
@ -422,18 +426,7 @@ func (mysqld *Mysqld) Init(ctx context.Context, initDBSQLFile string) error {
}
// Install data dir.
mysqlRoot, err := vtenv.VtMysqlRoot()
if err != nil {
log.Errorf("%v", err)
return err
}
log.Infof("Installing data dir with mysql_install_db")
args := []string{
"--defaults-file=" + mysqld.config.path,
"--basedir=" + mysqlRoot,
}
if _, err = execCmd(path.Join(mysqlRoot, "bin/mysql_install_db"), args, nil, mysqlRoot, nil); err != nil {
log.Errorf("mysql_install_db failed: %v", err)
if err = mysqld.installDataDir(); err != nil {
return err
}
@ -456,6 +449,49 @@ func (mysqld *Mysqld) Init(ctx context.Context, initDBSQLFile string) error {
return nil
}
func (mysqld *Mysqld) installDataDir() error {
mysqlRoot, err := vtenv.VtMysqlRoot()
if err != nil {
log.Errorf("%v", err)
return err
}
// Check mysqld version.
_, version, err := execCmd(path.Join(mysqlRoot, "sbin/mysqld"),
[]string{"--version"}, nil, mysqlRoot, nil)
if err != nil {
return err
}
if strings.Contains(version, "Ver 5.7.") {
// MySQL 5.7 GA and up have deprecated mysql_install_db.
// Instead, initialization is built into mysqld.
log.Infof("Installing data dir with mysqld --initialize-insecure")
args := []string{
"--defaults-file=" + mysqld.config.path,
"--basedir=" + mysqlRoot,
"--initialize-insecure", // Use empty 'root'@'localhost' password.
}
if _, _, err = execCmd(path.Join(mysqlRoot, "sbin/mysqld"), args, nil, mysqlRoot, nil); err != nil {
log.Errorf("mysqld --initialize-insecure failed: %v", err)
return err
}
return nil
}
log.Infof("Installing data dir with mysql_install_db")
args := []string{
"--defaults-file=" + mysqld.config.path,
"--basedir=" + mysqlRoot,
}
if _, _, err = execCmd(path.Join(mysqlRoot, "bin/mysql_install_db"), args, nil, mysqlRoot, nil); err != nil {
log.Errorf("mysql_install_db failed: %v", err)
return err
}
return nil
}
func (mysqld *Mysqld) initConfig(root string) error {
var err error
var configData string
@ -598,7 +634,7 @@ func (mysqld *Mysqld) executeMysqlScript(user string, sql io.Reader) error {
env := []string{
"LD_LIBRARY_PATH=" + path.Join(dir, "lib/mysql"),
}
_, err = execCmd(name, arg, env, dir, sql)
_, _, err = execCmd(name, arg, env, dir, sql)
if err != nil {
return err
}

Просмотреть файл

@ -63,7 +63,7 @@ For example:
// Flags
var (
flavor = flag.String("flavor", "mysql56", "bootstrap flavor to run against")
flavor = flag.String("flavor", "mysql57", "bootstrap flavor to run against")
runCount = flag.Int("runs", 1, "run each test this many times")
retryMax = flag.Int("retry", 3, "max number of retries, to detect flaky tests")
logPass = flag.Bool("log-pass", false, "log test output even if it passes")

Просмотреть файл

@ -307,7 +307,7 @@ class Tablet(object):
rows = self.mquery('', 'show databases')
for row in rows:
dbname = row[0]
if dbname in ['information_schema', 'mysql']:
if dbname in ['information_schema', 'performance_schema', 'mysql', 'sys']:
continue
self.drop_db(dbname)