René Cannaò
Коммит
5ef8de3eff
|
|
@ -0,0 +1,13 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [ ! -d /opt/proxysql ]; then
|
||||
mkdir -p /opt/proxysql
|
||||
cp -R /tmp/proxysql-src/* /opt/proxysql
|
||||
cd /opt/proxysql
|
||||
make clean && make > /tmp/proxysql_compile.log
|
||||
fi
|
||||
|
||||
cd /opt/proxysql/src
|
||||
# TODO(andrei): re-enable the commented line when figuring out interactive mode
|
||||
# gdbserver 0.0.0.0:2345 ./proxysql --initial -f -c /etc/proxysql.cnf
|
||||
./proxysql --initial -f -c /etc/proxysql.cnf
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
# The only purpose for which this image exists is to add ProxySQL-relevant
|
||||
# labels to it.
|
||||
FROM mysql:latest
|
||||
|
||||
LABEL vendor=proxysql\
|
||||
com.proxysql.type=mysql
|
||||
|
|
@ -3,5 +3,9 @@
|
|||
# There is a proposed improvement to "docker cp" but it's still being
|
||||
# discussed (https://github.com/docker/docker/issues/5846).
|
||||
FROM mysql:latest
|
||||
|
||||
LABEL vendor=proxysql\
|
||||
com.proxysql.type=mysql
|
||||
|
||||
ADD ./schema.sql /tmp/
|
||||
ADD ./import_schema.sh /tmp/
|
||||
|
|
@ -1,8 +1,13 @@
|
|||
# We're using Ubuntu 14:04 because ProxySQL compilation needs one of the latest
|
||||
# g++ compilers. Also, it's a long term release.
|
||||
FROM centos:centos7
|
||||
MAINTAINER Andrei Ismail <iandrei@gmail.com>
|
||||
|
||||
LABEL vendor=proxysql\
|
||||
com.proxysql.type=proxysql\
|
||||
com.proxysql.os=centos7\
|
||||
com.proxysql.interactive=false\
|
||||
com.proxysql.config=simple\
|
||||
com.proxysql.purpose=packaging
|
||||
|
||||
RUN yum install -y automake
|
||||
RUN yum install -y bzip2
|
||||
RUN yum install -y cmake
|
||||
|
|
@ -1,7 +1,13 @@
|
|||
# We're using Ubuntu 14:04 because ProxySQL compilation needs one of the latest
|
||||
# g++ compilers. Also, it's a long term release.
|
||||
FROM ubuntu:14.04
|
||||
MAINTAINER Andrei Ismail <iandrei@gmail.com>
|
||||
|
||||
LABEL vendor=proxysql\
|
||||
com.proxysql.type=proxysql\
|
||||
com.proxysql.os=ubuntu14\
|
||||
com.proxysql.interactive=false\
|
||||
com.proxysql.config=simple\
|
||||
com.proxysql.purpose=testing
|
||||
|
||||
RUN apt-get update && apt-get install -y\
|
||||
automake\
|
||||
cmake\
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [ ! -d /opt/proxysql ]; then
|
||||
mkdir -p /opt/proxysql
|
||||
cp -R /tmp/proxysql-src/* /opt/proxysql
|
||||
cd /opt/proxysql
|
||||
make clean && make > /tmp/proxysql_compile.log
|
||||
fi
|
||||
|
||||
cd /opt/proxysql/src
|
||||
# TODO(andrei): re-enable the commented line when figuring out interactive mode
|
||||
# gdbserver 0.0.0.0:2345 ./proxysql --initial -f -c /etc/proxysql.cnf
|
||||
./proxysql --initial -f -c /etc/proxysql.cnf
|
||||
|
|
@ -1,7 +1,13 @@
|
|||
# We're using Ubuntu 14:04 because ProxySQL compilation needs one of the latest
|
||||
# g++ compilers. Also, it's a long term release.
|
||||
FROM ubuntu:12.04
|
||||
MAINTAINER Andrei Ismail <iandrei@gmail.com>
|
||||
|
||||
LABEL vendor=proxysql\
|
||||
com.proxysql.type=proxysql\
|
||||
com.proxysql.os=ubuntu12\
|
||||
com.proxysql.interactive=false\
|
||||
com.proxysql.config=simple\
|
||||
com.proxysql.purpose=packaging
|
||||
|
||||
RUN apt-get update && apt-get install -y\
|
||||
automake\
|
||||
cmake\
|
||||
|
|
@ -1,7 +1,13 @@
|
|||
# We're using Ubuntu 14:04 because ProxySQL compilation needs one of the latest
|
||||
# g++ compilers. Also, it's a long term release.
|
||||
FROM ubuntu:14.04
|
||||
MAINTAINER Andrei Ismail <iandrei@gmail.com>
|
||||
|
||||
LABEL vendor=proxysql\
|
||||
com.proxysql.type=proxysql\
|
||||
com.proxysql.os=ubuntu14\
|
||||
com.proxysql.interactive=false\
|
||||
com.proxysql.config=simple\
|
||||
com.proxysql.purpose=packaging
|
||||
|
||||
RUN apt-get update && apt-get install -y\
|
||||
automake\
|
||||
cmake\
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
proxysql:
|
||||
image: proxysql:{{proxysql_image}}
|
||||
links:
|
||||
- backend1hostgroup0
|
||||
ports:
|
||||
# ProxySQL admin port for MySQL commands
|
||||
- "6032:6032"
|
||||
# ProxySQL main port
|
||||
- "6033:6033"
|
||||
# gdbserver
|
||||
- "2345:2345"
|
||||
volumes:
|
||||
# Sharing the data with
|
||||
- /tmp/proxysql-tests:/tmp/proxysql-src
|
||||
privileged: true
|
||||
|
||||
# TODO(aismail): remove these hardcoded labels and add them in the template
|
||||
# as a variable by using docker-inspect on the image.
|
||||
labels:
|
||||
"com.proxysql.config": "simple"
|
||||
"com.proxysql.interactive": "false"
|
||||
"com.proxysql.os": "ubuntu14"
|
||||
"com.proxysql.purpose": "testing"
|
||||
"com.proxysql.type": "proxysql"
|
||||
"vendor": "proxysql"
|
||||
|
||||
backend1hostgroup0:
|
||||
image: proxysql:mysql-simple-dump
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
expose:
|
||||
- "3306"
|
||||
ports:
|
||||
- "13306:3306"
|
||||
# TODO(aismail): remove these hardcoded labels and add them in the template
|
||||
# as a variable by using docker-inspect on the image.
|
||||
labels:
|
||||
"com.proxysql.type": "mysql"
|
||||
"vendor": "proxysql"
|
||||
"com.proxysql.hostgroup": "0"
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
proxysql:
|
||||
image: proxysql:{{proxysql_image}}
|
||||
links:
|
||||
- backend1hostgroup0
|
||||
- backend1hostgroup1
|
||||
- backend2hostgroup1
|
||||
- backend3hostgroup1
|
||||
- backend4hostgroup1
|
||||
ports:
|
||||
# ProxySQL admin port for MySQL commands
|
||||
- "6032:6032"
|
||||
# ProxySQL main port
|
||||
- "6033:6033"
|
||||
# gdbserver
|
||||
- "2345:2345"
|
||||
volumes:
|
||||
# Sharing the data with
|
||||
- /tmp/proxysql-tests:/tmp/proxysql-src
|
||||
privileged: true
|
||||
|
||||
# TODO(aismail): remove these hardcoded labels and add them in the template
|
||||
# as a variable by using docker-inspect on the image.
|
||||
labels:
|
||||
"com.proxysql.config": "simple"
|
||||
"com.proxysql.interactive": "false"
|
||||
"com.proxysql.os": "ubuntu14"
|
||||
"com.proxysql.purpose": "testing"
|
||||
"com.proxysql.type": "proxysql"
|
||||
"vendor": "proxysql"
|
||||
|
||||
backend1hostgroup0:
|
||||
image: proxysql:mysql-simple-dump
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
expose:
|
||||
- "3306"
|
||||
ports:
|
||||
- "13306:3306"
|
||||
# TODO(aismail): remove these hardcoded labels and add them in the template
|
||||
# as a variable by using docker-inspect on the image.
|
||||
labels:
|
||||
"com.proxysql.type": "mysql"
|
||||
"vendor": "proxysql"
|
||||
"com.proxysql.hostgroup": "0"
|
||||
volumes:
|
||||
- ./master-conf.d:/etc/mysql/conf.d
|
||||
|
||||
backend1hostgroup1:
|
||||
image: proxysql:mysql-empty
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
NEW_MASTER: True
|
||||
links:
|
||||
- backend1hostgroup0
|
||||
expose:
|
||||
- "3306"
|
||||
ports:
|
||||
- "13307:3306"
|
||||
# TODO(aismail): remove these hardcoded labels and add them in the template
|
||||
# as a variable by using docker-inspect on the image.
|
||||
labels:
|
||||
"com.proxysql.type": "mysql"
|
||||
"vendor": "proxysql"
|
||||
"com.proxysql.hostgroup": "1"
|
||||
volumes:
|
||||
- ./slave1-conf.d:/etc/mysql/conf.d
|
||||
|
||||
backend2hostgroup1:
|
||||
image: proxysql:mysql-empty
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
links:
|
||||
- backend1hostgroup0
|
||||
expose:
|
||||
- "3306"
|
||||
ports:
|
||||
- "13308:3306"
|
||||
# TODO(aismail): remove these hardcoded labels and add them in the template
|
||||
# as a variable by using docker-inspect on the image.
|
||||
labels:
|
||||
"com.proxysql.type": "mysql"
|
||||
"vendor": "proxysql"
|
||||
"com.proxysql.hostgroup": "1"
|
||||
volumes:
|
||||
- ./slave2-conf.d:/etc/mysql/conf.d
|
||||
|
||||
backend3hostgroup1:
|
||||
image: proxysql:mysql-empty
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
links:
|
||||
- backend1hostgroup0
|
||||
expose:
|
||||
- "3306"
|
||||
ports:
|
||||
- "13309:3306"
|
||||
# TODO(aismail): remove these hardcoded labels and add them in the template
|
||||
# as a variable by using docker-inspect on the image.
|
||||
labels:
|
||||
"com.proxysql.type": "mysql"
|
||||
"vendor": "proxysql"
|
||||
"com.proxysql.hostgroup": "1"
|
||||
volumes:
|
||||
- ./slave3-conf.d:/etc/mysql/conf.d
|
||||
|
||||
backend4hostgroup1:
|
||||
image: proxysql:mysql-empty
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
links:
|
||||
- backend1hostgroup0
|
||||
expose:
|
||||
- "3306"
|
||||
ports:
|
||||
- "13310:3306"
|
||||
# TODO(aismail): remove these hardcoded labels and add them in the template
|
||||
# as a variable by using docker-inspect on the image.
|
||||
labels:
|
||||
"com.proxysql.type": "mysql"
|
||||
"vendor": "proxysql"
|
||||
"com.proxysql.hostgroup": "1"
|
||||
volumes:
|
||||
- ./slave4-conf.d:/etc/mysql/conf.d
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
[mysqld]
|
||||
server_id = 1
|
||||
innodb_file_per_table
|
||||
innodb_flush_log_at_trx_commit=2
|
||||
query_cache_size = 0
|
||||
sync_binlog = 0
|
||||
log_bin
|
||||
binlog_format = MIXED
|
||||
gtid_mode = ON
|
||||
log-slave-updates
|
||||
enforce-gtid-consistency
|
||||
expire_logs_days = 3
|
||||
max_binlog_size = 100M
|
||||
bind_address = 0.0.0.0
|
||||
max_connections = 5000
|
||||
skip_name_resolve
|
||||
|
||||
[mysqld_safe]
|
||||
open_files_limit = 102400
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
[mysqld]
|
||||
server_id = 2
|
||||
innodb_file_per_table
|
||||
innodb_flush_log_at_trx_commit=2
|
||||
query_cache_size = 0
|
||||
sync_binlog = 0
|
||||
log_bin
|
||||
binlog_format = MIXED
|
||||
gtid_mode = ON
|
||||
log-slave-updates
|
||||
enforce-gtid-consistency
|
||||
expire_logs_days = 3
|
||||
max_binlog_size = 100M
|
||||
bind_address = 0.0.0.0
|
||||
max_connections = 5000
|
||||
skip_name_resolve
|
||||
|
||||
[mysqld_safe]
|
||||
open_files_limit = 102400
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
[mysqld]
|
||||
server_id = 3
|
||||
innodb_file_per_table
|
||||
innodb_flush_log_at_trx_commit=2
|
||||
query_cache_size = 0
|
||||
sync_binlog = 0
|
||||
log_bin
|
||||
binlog_format = MIXED
|
||||
gtid_mode = ON
|
||||
log-slave-updates
|
||||
enforce-gtid-consistency
|
||||
expire_logs_days = 3
|
||||
max_binlog_size = 100M
|
||||
bind_address = 0.0.0.0
|
||||
max_connections = 5000
|
||||
skip_name_resolve
|
||||
|
||||
[mysqld_safe]
|
||||
open_files_limit = 102400
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
[mysqld]
|
||||
server_id = 4
|
||||
innodb_file_per_table
|
||||
innodb_flush_log_at_trx_commit=2
|
||||
query_cache_size = 0
|
||||
sync_binlog = 0
|
||||
log_bin
|
||||
binlog_format = MIXED
|
||||
gtid_mode = ON
|
||||
log-slave-updates
|
||||
enforce-gtid-consistency
|
||||
expire_logs_days = 3
|
||||
max_binlog_size = 100M
|
||||
bind_address = 0.0.0.0
|
||||
max_connections = 5000
|
||||
skip_name_resolve
|
||||
|
||||
[mysqld_safe]
|
||||
open_files_limit = 102400
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
[mysqld]
|
||||
server_id = 5
|
||||
innodb_file_per_table
|
||||
innodb_flush_log_at_trx_commit=2
|
||||
query_cache_size = 0
|
||||
sync_binlog = 0
|
||||
log_bin
|
||||
binlog_format = MIXED
|
||||
gtid_mode = ON
|
||||
log-slave-updates
|
||||
enforce-gtid-consistency
|
||||
expire_logs_days = 3
|
||||
max_binlog_size = 100M
|
||||
bind_address = 0.0.0.0
|
||||
max_connections = 5000
|
||||
skip_name_resolve
|
||||
|
||||
[mysqld_safe]
|
||||
open_files_limit = 102400
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
docker-compose==1.1.0
|
||||
docker-compose==1.4.2
|
||||
Jinja2==2.8
|
||||
MySQL-python==1.2.5
|
||||
nose==1.3.6
|
||||
requests==2.4.3
|
||||
requests==2.6.2
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
#!/bin/bash
|
||||
cd /opt/proxysql
|
||||
make clean && make
|
||||
cd src
|
||||
# TODO(andrei): re-enable the commented line when figuring out interactive mode
|
||||
# gdbserver 0.0.0.0:2345 ./proxysql --initial -f -c /etc/proxysql.cnf
|
||||
./proxysql --initial -f -c /etc/proxysql.cnf
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
proxysql:
|
||||
build: ../base/proxysql
|
||||
links:
|
||||
- backend1hostgroup0
|
||||
ports:
|
||||
# ProxySQL admin port for MySQL commands
|
||||
- "6032:6032"
|
||||
# ProxySQL main port
|
||||
- "6033:6033"
|
||||
# gdbserver
|
||||
- "2345:2345"
|
||||
volumes:
|
||||
- /tmp/proxysql-tests:/opt/proxysql
|
||||
privileged: true
|
||||
|
||||
backend1hostgroup0:
|
||||
build: mysql
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
expose:
|
||||
- "3306"
|
||||
ports:
|
||||
- "13306:3306"
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
#!/bin/bash
|
||||
cd /opt/proxysql
|
||||
make clean && make
|
||||
cd src
|
||||
# TODO(andrei): re-enable the commented line when figuring out interactive mode
|
||||
# gdbserver 0.0.0.0:2345 ./proxysql --initial -f -c /etc/proxysql.cnf
|
||||
./proxysql --initial -f -c /etc/proxysql.cnf
|
||||
|
|
@ -5,9 +5,7 @@ from proxysql_base_test import ProxySQLBaseTest
|
|||
|
||||
class AdminTablesTest(ProxySQLBaseTest):
|
||||
|
||||
SCENARIO = "./scenarios/1backend"
|
||||
|
||||
def test_monitor_tables_locking_errors(self):
|
||||
def _test_monitor_tables_locking_errors(self):
|
||||
"""Test that intensive read/write operations to the MySQL Monitor tables
|
||||
do not trigger locking errors.
|
||||
|
||||
|
|
@ -18,9 +16,9 @@ class AdminTablesTest(ProxySQLBaseTest):
|
|||
# Setting these variables will cause the Monitor to connect more
|
||||
# frequently to the backend hosts to check their health, thus increasing
|
||||
# the probability of locking errors to appear.
|
||||
ProxySQLBaseTest.run_query_proxysql_admin("UPDATE global_variables SET variable_value=100 WHERE variable_name='mysql-monitor_connect_interval'")
|
||||
ProxySQLBaseTest.run_query_proxysql_admin("UPDATE global_variables SET variable_value=100 WHERE variable_name='mysql-monitor_ping_interval'")
|
||||
ProxySQLBaseTest.run_query_proxysql_admin("LOAD MYSQL VARIABLES TO RUNTIME")
|
||||
self.run_query_proxysql_admin("UPDATE global_variables SET variable_value=100 WHERE variable_name='mysql-monitor_connect_interval'")
|
||||
self.run_query_proxysql_admin("UPDATE global_variables SET variable_value=100 WHERE variable_name='mysql-monitor_ping_interval'")
|
||||
self.run_query_proxysql_admin("LOAD MYSQL VARIABLES TO RUNTIME")
|
||||
|
||||
queries = []
|
||||
q1 = "select * from monitor.mysql_server_connect_log ORDER BY RANDOM() LIMIT 10"
|
||||
|
|
@ -29,9 +27,11 @@ class AdminTablesTest(ProxySQLBaseTest):
|
|||
queries.append(random.choice([q1, q2]))
|
||||
|
||||
pool = ThreadPool(processes=5)
|
||||
pool.map(ProxySQLBaseTest.run_query_proxysql_admin,
|
||||
queries)
|
||||
pool.map(self.run_query_proxysql_admin, queries)
|
||||
|
||||
# If we reached this point without an error, it means that the test
|
||||
# has passed.
|
||||
self.assertEqual(1, 1)
|
||||
self.assertEqual(1, 1)
|
||||
|
||||
def test_monitor_tables_locking_errors(self):
|
||||
self.run_in_docker_scenarios(self._test_monitor_tables_locking_errors)
|
||||
|
|
@ -1,16 +1,19 @@
|
|||
from proxysql_base_test import ProxySQLBaseTest
|
||||
|
||||
from MySQLdb import OperationalError
|
||||
from nose.tools import raises
|
||||
|
||||
class AdminTest(ProxySQLBaseTest):
|
||||
|
||||
SCENARIO = "./scenarios/1backend"
|
||||
def _test_stop_main_thread(self):
|
||||
try:
|
||||
# This test will just assert that PROXYSQL STOP works correctly
|
||||
# Since September 2015, the behaviour has been changed - PROXYSQL STOP
|
||||
# executes faster and immediately shuts down the connections, thus this
|
||||
# test is expected to raise OperationalError
|
||||
self.run_query_proxysql_admin("PROXYSQL STOP")
|
||||
self.assertEqual(0, 1)
|
||||
except OperationalError:
|
||||
self.assertEqual(1, 1)
|
||||
|
||||
@raises(OperationalError)
|
||||
def test_stop_main_thread(self):
|
||||
# This test will just assert that PROXYSQL STOP works correctly
|
||||
# Since September 2015, the behaviour has been changed - PROXYSQL STOP
|
||||
# executes faster and immediately shuts down the connections, thus this
|
||||
# test is expected to raise OperationalError
|
||||
ProxySQLBaseTest.run_query_proxysql_admin("PROXYSQL STOP")
|
||||
self.run_in_docker_scenarios(self._test_stop_main_thread)
|
||||
|
|
@ -6,38 +6,58 @@ from proxysql_base_test import ProxySQLBaseTest
|
|||
|
||||
class AuthenticationTest(ProxySQLBaseTest):
|
||||
|
||||
SCENARIO = "./scenarios/1backend"
|
||||
|
||||
def test_existing_user_with_correct_password_works(self):
|
||||
version1 = ProxySQLBaseTest.run_query_mysql(
|
||||
def _test_existing_user_with_correct_password_works(self):
|
||||
version1 = self.run_query_mysql(
|
||||
"SELECT @@version_comment LIMIT 1", "test",
|
||||
return_result=True,
|
||||
username="john", password="doe")
|
||||
|
||||
version2 = ProxySQLBaseTest.run_query_proxysql(
|
||||
version2 = self.run_query_proxysql(
|
||||
"SELECT @@version_comment LIMIT 1", "test",
|
||||
return_result=True,
|
||||
username="john", password="doe")
|
||||
|
||||
self.assertEqual(version1, version2)
|
||||
|
||||
@raises(OperationalError)
|
||||
def test_existing_user_with_correct_password_works(self):
|
||||
self.run_in_docker_scenarios(self._test_existing_user_with_correct_password_works)
|
||||
|
||||
def _test_existing_user_with_correct_password_but_not_registerd_within_proxysql_does_not_work(self):
|
||||
try:
|
||||
self.run_query_proxysql("SELECT @@version_comment LIMIT 1", "test",
|
||||
return_result=True,
|
||||
username="danny", password="white")
|
||||
# We shouldn't reach this point successfully, because it means we
|
||||
# managed to authenticate with the wrong password
|
||||
self.assertEqual(1, 0)
|
||||
except OperationalError:
|
||||
self.assertEqual(1, 1)
|
||||
|
||||
def test_existing_user_with_correct_password_but_not_registerd_within_proxysql_does_not_work(self):
|
||||
version1 = ProxySQLBaseTest.run_query_proxysql(
|
||||
"SELECT @@version_comment LIMIT 1", "test",
|
||||
return_result=True,
|
||||
username="danny", password="white")
|
||||
self.run_in_docker_scenarios(self._test_existing_user_with_correct_password_works)
|
||||
|
||||
def _test_existing_user_with_incorrect_password_does_not_work(self):
|
||||
try:
|
||||
self.run_query_proxysql("SELECT @@version_comment LIMIT 1", "test",
|
||||
return_result=True,
|
||||
username="john", password="doe2")
|
||||
# We shouldn't reach this point successfully, because it means we
|
||||
# managed to authenticate with the wrong password
|
||||
self.assertEqual(1, 0)
|
||||
except OperationalError:
|
||||
self.assertEqual(1, 1)
|
||||
|
||||
@raises(OperationalError)
|
||||
def test_existing_user_with_incorrect_password_does_not_work(self):
|
||||
version = ProxySQLBaseTest.run_query_proxysql(
|
||||
"SELECT @@version_comment LIMIT 1", "test",
|
||||
return_result=True,
|
||||
username="john", password="doe2")
|
||||
self.run_in_docker_scenarios(self._test_existing_user_with_incorrect_password_does_not_work)
|
||||
|
||||
def _test_inexisting_user_with_random_password_does_not_work(self):
|
||||
try:
|
||||
self.run_query_proxysql("SELECT @@version_comment LIMIT 1", "test",
|
||||
return_result=True,
|
||||
username="johnny", password="randomdoe")
|
||||
self.assertEqual(1, 0)
|
||||
except OperationalError:
|
||||
self.assertEqual(1, 1)
|
||||
|
||||
@raises(OperationalError)
|
||||
def test_inexisting_user_with_random_password_does_not_work(self):
|
||||
version = ProxySQLBaseTest.run_query_proxysql(
|
||||
"SELECT @@version_comment LIMIT 1", "test",
|
||||
return_result=True,
|
||||
username="johnny", password="randomdoe")
|
||||
self.run_in_docker_scenarios(self._test_inexisting_user_with_random_password_does_not_work)
|
||||
|
|
@ -1,9 +1,10 @@
|
|||
from proxysql_base_test import ProxySQLBaseTest
|
||||
|
||||
# TODO(aismail): re-enable this test once ProxySQLBaseTest.run_in_docker_scenarios
|
||||
# gets some filtering capabilities on which types of scenarios to run, etc.
|
||||
"""
|
||||
class ConfigFileParsingTest(ProxySQLBaseTest):
|
||||
|
||||
SCENARIO = "./scenarios/1backend-complex-config"
|
||||
|
||||
|
||||
# The complex config scenario includes a configuration file
|
||||
# where most of the values of the configurations are changed in order to
|
||||
# detect whether ProxySQL is actually able to parse them correctly from the
|
||||
|
|
@ -18,24 +19,23 @@ class ConfigFileParsingTest(ProxySQLBaseTest):
|
|||
}
|
||||
}
|
||||
|
||||
def test_parse_config_file_to_admin_db(self):
|
||||
"""ProxySQL keeps its configuration in an admin database, so that it
|
||||
def _test_parse_config_file_to_admin_db(self):
|
||||
ProxySQL keeps its configuration in an admin database, so that it
|
||||
is available for inspection at runtime as well.
|
||||
|
||||
For the initial run, the configuration file is parsed into this admin
|
||||
database. We will test that the correct values of the variables
|
||||
exposed in the config file are exposed via the admin interface that
|
||||
is accessible through MySQL.
|
||||
"""
|
||||
|
||||
users = ConfigFileParsingTest.run_query_proxysql_admin("SELECT COUNT(*) FROM mysql_users")
|
||||
users = self.run_query_proxysql_admin("SELECT COUNT(*) FROM mysql_users")
|
||||
self.assertEqual(int(users[0][0]), 2)
|
||||
|
||||
servers = ConfigFileParsingTest.run_query_proxysql_admin("SELECT COUNT(*) FROM mysql_servers")
|
||||
servers = self.run_query_proxysql_admin("SELECT COUNT(*) FROM mysql_servers")
|
||||
# 3 in the config file, 1 auto-populated by the test
|
||||
self.assertEqual(int(servers[0][0]), 4)
|
||||
|
||||
variables = ConfigFileParsingTest.run_query_proxysql_admin("SELECT * FROM global_variables")
|
||||
variables = self.run_query_proxysql_admin("SELECT * FROM global_variables")
|
||||
mysql_variables = {}
|
||||
admin_variables = {}
|
||||
for (k, v) in variables:
|
||||
|
|
@ -73,4 +73,9 @@ class ConfigFileParsingTest(ProxySQLBaseTest):
|
|||
|
||||
self.assertEqual(admin_variables['admin_credentials'], 'admin2:admin2')
|
||||
self.assertEqual(admin_variables['mysql_ifaces'], '0.0.0.0:6032')
|
||||
self.assertEqual(admin_variables['refresh_interval'], '2000')
|
||||
self.assertEqual(admin_variables['refresh_interval'], '2000')
|
||||
|
||||
def test_parse_config_file_to_admin_db(self):
|
||||
self.run_in_docker_scenarios(self._test_parse_config_file_to_admin_db)
|
||||
|
||||
"""
|
||||
|
|
@ -0,0 +1,602 @@
|
|||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from os import path
|
||||
import random
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
from jinja2 import Template
|
||||
import MySQLdb
|
||||
|
||||
from proxysql_tests_config import ProxySQL_Tests_Config
|
||||
|
||||
class DockerFleet(object):
|
||||
|
||||
def __init__(self, config_overrides = {}):
|
||||
self.config_overrides = config_overrides
|
||||
|
||||
def _get_dockerfiles_for(self, token):
|
||||
"""Retrieves the list of Dockerfiles for a given type of machine.
|
||||
|
||||
The available types are: proxysql and mysql.
|
||||
"""
|
||||
files = {}
|
||||
dockerfiles_path = os.path.dirname(__file__) + "/../docker/images/%s" % token
|
||||
for item in os.listdir(dockerfiles_path):
|
||||
dir_path = dockerfiles_path + os.sep + item
|
||||
if path.isdir(dir_path):
|
||||
dockerfile = dir_path + os.sep + "Dockerfile"
|
||||
if path.isfile(dockerfile):
|
||||
files[item] = {
|
||||
"dir": dir_path,
|
||||
"dockerfile": dockerfile
|
||||
}
|
||||
return files
|
||||
|
||||
def get_dockerfiles_for_proxysql(self):
|
||||
"""Retrieves the list of Dockerfiles available to build ProxySQL."""
|
||||
return self._get_dockerfiles_for('proxysql')
|
||||
|
||||
def get_dockerfiles_for_mysql(self):
|
||||
"""Retrieves the list of Dockerfiles available to build MySQL."""
|
||||
return self._get_dockerfiles_for('mysql')
|
||||
|
||||
def build_images(self):
|
||||
"""Builds docker images for all the Dockerfiles available for
|
||||
ProxySQL/MySQL."""
|
||||
proxysql_dockerfiles = self.get_dockerfiles_for_proxysql()
|
||||
mysql_dockerfiles = self.get_dockerfiles_for_mysql()
|
||||
|
||||
for label, info in proxysql_dockerfiles.iteritems():
|
||||
subprocess.call(["docker", "rmi", "-f", "proxysql:%s" % label])
|
||||
subprocess.call(["docker", "build", "-t", "proxysql:%s" % label, "."], cwd=info["dir"])
|
||||
|
||||
for label, info in mysql_dockerfiles.iteritems():
|
||||
subprocess.call(["docker", "rmi", "-f", "proxysql:%s" % label])
|
||||
subprocess.call(["docker", "build", "-t", "proxysql:%s" % label, "."], cwd=info["dir"])
|
||||
|
||||
def get_docker_images(self, filters):
|
||||
names = filters.pop('names', [])
|
||||
if len(names) == 0:
|
||||
if filters.get('com.proxysql.type') == 'proxysql':
|
||||
config = ProxySQL_Tests_Config(overrides=self.config_overrides)
|
||||
names = config.get('Scenarios', 'default_proxysql_images').split(',')
|
||||
elif filters.get('com.proxysql.type') == 'mysql':
|
||||
config = ProxySQL_Tests_Config(overrides=self.config_overrides)
|
||||
names = config.get('Scenarios', 'default_mysql_images').split(',')
|
||||
|
||||
args = ["docker", "images"]
|
||||
for k, v in filters.iteritems():
|
||||
args.append("--filter")
|
||||
args.append("label=%s=%s" % (k, v))
|
||||
nonemtpy_lines = self._get_stdout_as_lines(args)
|
||||
results = nonemtpy_lines[1:]
|
||||
images = []
|
||||
for (i, r) in enumerate(results):
|
||||
tokens = r.split(' ')
|
||||
nonempty_tokens = [t for t in tokens if len(t.strip()) > 0]
|
||||
image = nonempty_tokens[1]
|
||||
if image in names:
|
||||
images.append(image)
|
||||
return images
|
||||
|
||||
def get_docker_scenario_templates(self, scenarios=[]):
|
||||
"""Retrieve the list of docker templates that will be used to generate
|
||||
scenarios.
|
||||
|
||||
Why do we need templates for scenarios? Because the scenario will be
|
||||
the same, the only difference will be the configuration of the machines
|
||||
involved (for example, it might be a different operating system).
|
||||
"""
|
||||
if len(scenarios) == 0:
|
||||
config = ProxySQL_Tests_Config(overrides=self.config_overrides)
|
||||
scenarios = config.get('Scenarios', 'default_scenarios').split(',')
|
||||
|
||||
files = {}
|
||||
dockercompose_path = os.path.dirname(__file__) + "/../docker/scenarios"
|
||||
for item in os.listdir(dockercompose_path):
|
||||
# Filter based on received default scenarios
|
||||
if item not in scenarios:
|
||||
continue
|
||||
|
||||
dir_path = dockercompose_path + os.sep + item
|
||||
if path.isdir(dir_path):
|
||||
dockercomposefile = dir_path + os.sep + "docker-compose.yml"
|
||||
if path.isfile(dockercomposefile):
|
||||
files[item] = {
|
||||
"dir": dir_path,
|
||||
"dockercomposefile": dockercomposefile
|
||||
}
|
||||
with open (dockercomposefile, "r") as myfile:
|
||||
files[item]["contents"] = data=myfile.read()
|
||||
return files
|
||||
|
||||
def generate_scenarios(self, scenarios=[], proxysql_filters={}, mysql_filters={}):
|
||||
# We have 2 types of docker images - for testing and for packaging.
|
||||
# We will only use the ones for testing, because the others won't have
|
||||
# a running daemon inside.
|
||||
|
||||
local_proxysql_filters = copy.deepcopy(proxysql_filters)
|
||||
local_proxysql_filters['vendor'] = 'proxysql'
|
||||
local_proxysql_filters['com.proxysql.purpose'] = 'testing'
|
||||
local_proxysql_filters['com.proxysql.type'] = 'proxysql'
|
||||
|
||||
local_mysql_filters = copy.deepcopy(mysql_filters)
|
||||
local_mysql_filters['vendor'] = 'proxysql'
|
||||
# TODO(aismail): rebuild MySQL image
|
||||
# TODO(aismail): add flag in order to rebuild images to the test suite
|
||||
# local_mysql_filters['com.proxysql.purpose'] = 'testing'
|
||||
local_mysql_filters['com.proxysql.type'] = 'mysql'
|
||||
|
||||
unique_scenarios = {}
|
||||
|
||||
proxysql_images = self.get_docker_images(local_proxysql_filters)
|
||||
mysql_images = self.get_docker_images(local_mysql_filters)
|
||||
scenario_templates = self.get_docker_scenario_templates(scenarios=scenarios)
|
||||
|
||||
for scenario_label, scenario_info in scenario_templates.iteritems():
|
||||
for proxysql_image in proxysql_images:
|
||||
for mysql_image in mysql_images:
|
||||
template = Template(scenario_info['contents'])
|
||||
scenario = template.render({
|
||||
'proxysql_image': proxysql_image,
|
||||
'mysql_image': mysql_image
|
||||
})
|
||||
m = hashlib.md5()
|
||||
m.update(scenario)
|
||||
digest = m.digest()
|
||||
if digest not in unique_scenarios:
|
||||
unique_scenarios[digest] = {
|
||||
'content': scenario,
|
||||
'proxysql_image': proxysql_image,
|
||||
'mysql_image': mysql_image,
|
||||
'scenario_dir': scenario_info['dir'],
|
||||
'scenario_dockercompose_file': scenario_info['dockercomposefile']
|
||||
}
|
||||
return unique_scenarios.values()
|
||||
|
||||
def _wait_for_daemons_startup(self):
|
||||
# First off, wait for all the MySQL backends to have initialized
|
||||
mysql_credentials = self.get_all_mysql_connection_credentials()
|
||||
for credential in mysql_credentials:
|
||||
self.wait_for_mysql_connection_ok(**credential)
|
||||
|
||||
# Afterwards, wait for the main ProxySQL thread to start responding to
|
||||
# MySQL queries. Note that we have chosen such a query that gets handled
|
||||
# directly by the proxy. That makes sure that tests are running
|
||||
# correctly even when there's something broken inside ProxySQL.
|
||||
proxysql_credentials = self.get_proxysql_connection_credentials()
|
||||
self.wait_for_mysql_connection_ok(**proxysql_credentials)
|
||||
proxysql_admin_credentials = self.get_proxysql_admin_connection_credentials()
|
||||
self.wait_for_mysql_connection_ok(**proxysql_admin_credentials)
|
||||
|
||||
# Extra sleep at the end, because if the test tries to shut down
|
||||
# ProxySQL too close to its startup, problems may arise
|
||||
time.sleep(5)
|
||||
|
||||
def _create_folder_to_share_proxysql_code_with_container(self):
|
||||
try:
|
||||
if os.path.exists('/tmp/proxysql-tests'):
|
||||
shutil.rmtree('/tmp/proxysql-tests/')
|
||||
except:
|
||||
pass
|
||||
|
||||
os.mkdir('/tmp/proxysql-tests')
|
||||
os.system("cp -R " + os.path.dirname(__file__) + "/../* /tmp/proxysql-tests")
|
||||
|
||||
def _delete_folder_with_shared_proxysql_code(self):
|
||||
shutil.rmtree('/tmp/proxysql-tests/')
|
||||
|
||||
def _stop_existing_docker_containers(self):
|
||||
"""Stops any proxysql-related docker containers running on this host.
|
||||
|
||||
Warning: this means that if you are running the tests and using this
|
||||
host to operate a production instance of ProxySQL using Docker, it
|
||||
will be stopped. Unfortunately, there is no easy way to differentiate
|
||||
between the two.
|
||||
"""
|
||||
args = ["docker", "ps", "--filter", "label=vendor=proxysql"]
|
||||
p = subprocess.Popen(args, stdout=subprocess.PIPE)
|
||||
out, _ = p.communicate()
|
||||
lines = out.split('\n')
|
||||
nonemtpy_lines = [l for l in lines if len(l.strip()) > 0]
|
||||
results = nonemtpy_lines[1:]
|
||||
images = []
|
||||
for (i, r) in enumerate(results):
|
||||
tokens = r.split(' ')
|
||||
nonempty_tokens = [t for t in tokens if len(t.strip()) > 0]
|
||||
images.append(nonempty_tokens[0])
|
||||
|
||||
for image in images:
|
||||
subprocess.call(["docker", "kill", image])
|
||||
|
||||
def start_temp_scenario(self, scenario, copy_folder=True):
|
||||
self._stop_existing_docker_containers()
|
||||
if copy_folder:
|
||||
self._create_folder_to_share_proxysql_code_with_container()
|
||||
|
||||
dirname = tempfile.mkdtemp('-proxysql-tests')
|
||||
os.system("cp -R %s/* %s/" % (scenario['scenario_dir'], dirname))
|
||||
filename = "%s/docker-compose.yml" % dirname
|
||||
with open(filename, "wt") as f:
|
||||
f.write(scenario['content'])
|
||||
subprocess.call(["docker-compose", "up", "-d"], cwd=dirname)
|
||||
|
||||
self._wait_for_daemons_startup()
|
||||
self._populate_mysql_containers_with_dump()
|
||||
self._populate_proxy_configuration_with_backends()
|
||||
|
||||
return dirname
|
||||
|
||||
def stop_temp_scenario(self, path_to_scenario, delete_folder=True):
|
||||
# Shut down ProxySQL cleanly
|
||||
try:
|
||||
self.run_query_proxysql_admin("PROXYSQL SHUTDOWN")
|
||||
except:
|
||||
# This will throw an exception because it will forcefully shut down
|
||||
# the connection with the MySQL client.
|
||||
pass
|
||||
|
||||
subprocess.call(["docker-compose", "stop"], cwd=path_to_scenario)
|
||||
if delete_folder:
|
||||
self._delete_folder_with_shared_proxysql_code()
|
||||
|
||||
def get_proxysql_container(self):
|
||||
"""Out of all the started docker containers, select the one which
|
||||
represents the proxy instance.
|
||||
|
||||
Note that this only supports one proxy instance for now. This method
|
||||
relies on interogating the Docker daemon via its REST API.
|
||||
"""
|
||||
|
||||
args = ["docker", "ps",
|
||||
"--filter", "label=vendor=proxysql",
|
||||
"--filter", "label=com.proxysql.type=proxysql",
|
||||
"--format", "{{.ID}}"]
|
||||
return self._get_stdout_as_lines(args)[0]
|
||||
|
||||
def get_mysql_containers(self, hostgroup=None):
|
||||
"""Out of all the started docker containers, select the ones which
|
||||
represent the MySQL backend instances.
|
||||
|
||||
This method relies on interogating the Docker daemon via its REST API.
|
||||
"""
|
||||
|
||||
args = ["docker", "ps",
|
||||
"--filter", "label=vendor=proxysql",
|
||||
"--filter", "label=com.proxysql.type=mysql"]
|
||||
if hostgroup is not None:
|
||||
args.append("--filter")
|
||||
args.append("label=com.proxysql.hostgroup=%d" % hostgroup)
|
||||
args.extend(["--format", "{{.ID}}"])
|
||||
return self._get_stdout_as_lines(args)
|
||||
|
||||
def mysql_connection_ok(self, hostname, port, username, password, schema):
|
||||
"""Checks whether the MySQL server reachable at (hostname, port) is
|
||||
up or not. This is useful for waiting for ProxySQL/MySQL containers to
|
||||
start up correctly (meaning that the daemons running inside them have
|
||||
started to be able to respond to queries).
|
||||
"""
|
||||
try:
|
||||
db = MySQLdb.connect(host=hostname,
|
||||
user=username,
|
||||
passwd=password,
|
||||
db=schema,
|
||||
port=int(port))
|
||||
cursor = db.cursor()
|
||||
cursor.execute("select @@version_comment limit 1")
|
||||
results = cursor.fetchone()
|
||||
# Check if anything at all is returned
|
||||
if results:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
except MySQLdb.Error, e:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
def wait_for_mysql_connection_ok(self, hostname, port, username, password,
|
||||
max_retries=500, time_between_retries=1):
|
||||
|
||||
retries = 0
|
||||
result = False
|
||||
|
||||
while (not result) and (retries < max_retries):
|
||||
result = self.mysql_connection_ok(
|
||||
hostname=hostname,
|
||||
port=port,
|
||||
username=username,
|
||||
password=password,
|
||||
schema="information_schema"
|
||||
)
|
||||
if not result:
|
||||
retries += 1
|
||||
time.sleep(1)
|
||||
print("Trying again to connect to %s:%s (retries=%d)" % (hostname, port, retries))
|
||||
|
||||
return result
|
||||
|
||||
def get_all_mysql_connection_credentials(self, hostgroup=None):
|
||||
# Figure out which are the containers for the specified hostgroup
|
||||
mysql_backend_ids = self.get_mysql_containers(hostgroup=hostgroup)
|
||||
|
||||
config = ProxySQL_Tests_Config(overrides=self.config_overrides)
|
||||
hostname = config.get('ProxySQL', 'hostname')
|
||||
username = config.get('ProxySQL', 'username')
|
||||
password = config.get('ProxySQL', 'password')
|
||||
|
||||
result = []
|
||||
for container_id in mysql_backend_ids:
|
||||
metadata = self.docker_inspect(container_id)
|
||||
mysql_port = metadata.get('NetworkSettings', {})\
|
||||
.get('Ports', {})\
|
||||
.get('3306/tcp', [{}])[0]\
|
||||
.get('HostPort', None)
|
||||
if mysql_port is not None:
|
||||
result.append({
|
||||
'hostname': hostname,
|
||||
'port': mysql_port,
|
||||
'username': username,
|
||||
'password': password
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
def get_mysql_connection_credentials(self, hostgroup=0):
|
||||
|
||||
credentials = self.get_all_mysql_connection_credentials(hostgroup=hostgroup)
|
||||
|
||||
if len(credentials) == 0:
|
||||
raise Exception('No backends with a publicly exposed port were '
|
||||
'found in hostgroup %d' % hostgroup)
|
||||
|
||||
return random.choice(credentials)
|
||||
|
||||
def get_proxysql_connection_credentials(self):
|
||||
config = ProxySQL_Tests_Config(overrides=self.config_overrides)
|
||||
return {
|
||||
"hostname": config.get("ProxySQL", "hostname"),
|
||||
"port": config.get("ProxySQL", "port"),
|
||||
"username": config.get("ProxySQL", "username"),
|
||||
"password": config.get("ProxySQL", "password")
|
||||
}
|
||||
|
||||
def get_proxysql_admin_connection_credentials(self):
|
||||
config = ProxySQL_Tests_Config(overrides=self.config_overrides)
|
||||
return {
|
||||
"hostname": config.get("ProxySQL", "hostname"),
|
||||
"port": config.get("ProxySQL", "admin_port"),
|
||||
"username": config.get("ProxySQL", "admin_username"),
|
||||
"password": config.get("ProxySQL", "admin_password")
|
||||
}
|
||||
|
||||
def _populate_mysql_containers_with_dump(self):
|
||||
"""Populates the started MySQL backend containers with the specified
|
||||
SQL dump file.
|
||||
|
||||
The reason for doing this __after__ the containers are started is
|
||||
because we want to keep them as generic as possible.
|
||||
"""
|
||||
|
||||
mysql_container_ids = self.get_mysql_containers()
|
||||
# We have already added the SQL dump to the container by using
|
||||
# the ADD mysql command in the Dockerfile for mysql -- check it
|
||||
# out. The standard agreed location is at /tmp/schema.sql.
|
||||
#
|
||||
# Unfortunately we can't do this step at runtime due to limitations
|
||||
# on how transfer between host and container is supposed to work by
|
||||
# design. See the Dockerfile for MySQL for more details.
|
||||
for container_id in mysql_container_ids:
|
||||
# /tmp/import_schema.sh might fail because there are some MySQL
|
||||
# instances that we're using (especially for slave servers) that
|
||||
# have no schema.
|
||||
try:
|
||||
subprocess.call(["docker", "exec", container_id, "bash", "/tmp/import_schema.sh"])
|
||||
except:
|
||||
continue
|
||||
|
||||
def _populate_proxy_configuration_with_backends(self):
|
||||
"""Populate ProxySQL's admin information with the MySQL backends
|
||||
and their associated hostgroups.
|
||||
|
||||
This is needed because I do not want to hardcode this into the ProxySQL
|
||||
config file of the test scenario, as it leaves more room for quick
|
||||
iteration.
|
||||
|
||||
In order to configure ProxySQL with the correct backends, we are using
|
||||
the MySQL admin interface of ProxySQL, and inserting rows into the
|
||||
`mysql_servers` table, which contains a list of which servers go into
|
||||
which hostgroup.
|
||||
"""
|
||||
config = ProxySQL_Tests_Config(overrides=self.config_overrides)
|
||||
proxysql_container_id = self.get_proxysql_container()
|
||||
mysql_container_ids = self.get_mysql_containers()
|
||||
environment_variables = self.get_environment_variables_from_container(proxysql_container_id)
|
||||
|
||||
proxy_admin_connection = MySQLdb.connect(config.get('ProxySQL', 'hostname'),
|
||||
config.get('ProxySQL', 'admin_username'),
|
||||
config.get('ProxySQL', 'admin_password'),
|
||||
port=int(config.get('ProxySQL', 'admin_port')))
|
||||
cursor = proxy_admin_connection.cursor()
|
||||
|
||||
for mysql_container_id in mysql_container_ids:
|
||||
metadata = self.docker_inspect(mysql_container_id)
|
||||
container_name = metadata['Name'][1:].upper()
|
||||
port_uri = environment_variables['%s_PORT' % container_name]
|
||||
port_no = self._extract_port_number_from_uri(port_uri)
|
||||
ip = environment_variables['%s_PORT_%d_TCP_ADDR' % (container_name, port_no)]
|
||||
hostgroup = int(metadata.get('Config', {}).get('Labels', {}).get('com.proxysql.hostgroup'))
|
||||
cursor.execute("INSERT INTO mysql_servers(hostgroup_id, hostname, port, status) "
|
||||
"VALUES(%d, '%s', %d, 'ONLINE')" %
|
||||
(hostgroup, ip, port_no))
|
||||
|
||||
cursor.execute("LOAD MYSQL SERVERS TO RUNTIME")
|
||||
cursor.close()
|
||||
proxy_admin_connection.close()
|
||||
|
||||
def _extract_port_number_from_uri(self, uri):
|
||||
"""Given a Docker container URI (exposed as an environment variable by
|
||||
the host linking mechanism), extract the TCP port number from it."""
|
||||
return int(uri.split(':')[2])
|
||||
|
||||
def get_environment_variables_from_container(self, container_id):
|
||||
"""Retrieve the environment variables from the given container.
|
||||
|
||||
This is useful because the host linking mechanism will expose
|
||||
connectivity information to the linked hosts by the use of environment
|
||||
variables.
|
||||
"""
|
||||
lines = self._get_stdout_as_lines(["docker", "exec", container_id, "env"])
|
||||
result = {}
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if len(line) == 0:
|
||||
continue
|
||||
(k, v) = line.split('=')
|
||||
result[k] = v
|
||||
return result
|
||||
|
||||
def run_query_proxysql(self, query, db,
|
||||
hostname=None, port=None,
|
||||
username=None, password=None,
|
||||
return_result=True):
|
||||
"""Run a query against the ProxySQL proxy and optionally return its
|
||||
results as a set of rows."""
|
||||
credentials = self.get_proxysql_connection_credentials()
|
||||
proxy_connection = MySQLdb.connect(hostname or credentials['hostname'],
|
||||
username or credentials['username'],
|
||||
password or credentials['password'],
|
||||
port=int(port or credentials['port']),
|
||||
db=db)
|
||||
cursor = proxy_connection.cursor()
|
||||
cursor.execute(query)
|
||||
if return_result:
|
||||
rows = cursor.fetchall()
|
||||
cursor.close()
|
||||
proxy_connection.close()
|
||||
if return_result:
|
||||
return rows
|
||||
|
||||
def run_query_proxysql_admin(self, query, return_result=True):
|
||||
"""Run a query against the ProxySQL admin.
|
||||
|
||||
Note: we do not need to specify a db for this query, as it's always
|
||||
against the "main" database.
|
||||
TODO(andrei): revisit db assumption once stats databases from ProxySQL
|
||||
are accessible via the MySQL interface.
|
||||
"""
|
||||
credentials = self.get_proxysql_admin_connection_credentials()
|
||||
proxy_connection = MySQLdb.connect(credentials['hostname'],
|
||||
credentials['username'],
|
||||
credentials['password'],
|
||||
port=int(credentials['port']),
|
||||
db='main')
|
||||
cursor = proxy_connection.cursor()
|
||||
cursor.execute(query)
|
||||
if return_result:
|
||||
rows = cursor.fetchall()
|
||||
cursor.close()
|
||||
proxy_connection.close()
|
||||
if return_result:
|
||||
return rows
|
||||
|
||||
def run_query_mysql(self, query, db, return_result=True, hostgroup=0,
|
||||
username=None, password=None):
|
||||
"""Run a query against the MySQL backend and optionally return its
|
||||
results as a set of rows.
|
||||
|
||||
IMPORTANT: since the queries are actually ran against the MySQL backend,
|
||||
that backend needs to expose its MySQL port to the outside through
|
||||
docker compose's port mapping mechanism.
|
||||
|
||||
This will actually parse the docker-compose configuration file to
|
||||
retrieve the available backends and hostgroups and will pick a backend
|
||||
from the specified hostgroup."""
|
||||
|
||||
credentials = self.get_mysql_connection_credentials(hostgroup=hostgroup)
|
||||
mysql_connection = MySQLdb.connect(host=credentials['hostname'],
|
||||
user=username or credentials['username'],
|
||||
passwd=password or credentials['password'],
|
||||
port=int(credentials['port']),
|
||||
db=db)
|
||||
cursor = mysql_connection.cursor()
|
||||
cursor.execute(query)
|
||||
if return_result:
|
||||
rows = cursor.fetchall()
|
||||
cursor.close()
|
||||
mysql_connection.close()
|
||||
if return_result:
|
||||
return rows
|
||||
|
||||
def run_query_mysql_container(self, query, db, container_id, return_result=True):
|
||||
config = ProxySQL_Tests_Config(overrides=self.config_overrides)
|
||||
hostname = config.get('ProxySQL', 'hostname')
|
||||
username = config.get('ProxySQL', 'username')
|
||||
password = config.get('ProxySQL', 'password')
|
||||
|
||||
metadata = self.docker_inspect(container_id)
|
||||
mysql_port = metadata.get('NetworkSettings', {})\
|
||||
.get('Ports', {})\
|
||||
.get('3306/tcp', [{}])[0]\
|
||||
.get('HostPort', None)
|
||||
if mysql_port is not None:
|
||||
mysql_connection = MySQLdb.connect(host=hostname,
|
||||
user=username,
|
||||
passwd=password,
|
||||
port=int(mysql_port),
|
||||
db=db)
|
||||
cursor = mysql_connection.cursor()
|
||||
cursor.execute(query)
|
||||
if return_result:
|
||||
rows = cursor.fetchall()
|
||||
cursor.close()
|
||||
mysql_connection.close()
|
||||
if return_result:
|
||||
return rows
|
||||
else:
|
||||
return None
|
||||
|
||||
def commit_proxysql_image(self, label):
|
||||
"""Given a Docker image used within the ProxySQL tests, commit it."""
|
||||
|
||||
# Run "docker ps" on the proxysql containers and build a
|
||||
# label -> container_id mapping.
|
||||
nonemtpy_lines = self._get_stdout_as_lines(["docker", "ps", "--filter", "label=vendor=proxysql"])
|
||||
|
||||
results = nonemtpy_lines[1:]
|
||||
images = {}
|
||||
for (i, r) in enumerate(results):
|
||||
tokens = r.split(' ')
|
||||
nonempty_tokens = [t for t in tokens if len(t.strip()) > 0]
|
||||
images[nonempty_tokens[1]] = nonempty_tokens[0]
|
||||
|
||||
# Having built the mapping, see if the specified image is in the
|
||||
# currently running images. If it is, `docker commit` it.
|
||||
#
|
||||
# TODO(aismail): improve this to use `docker ps --filter`. Couldn't get
|
||||
# that one to work correctly.
|
||||
key = "proxysql:%s" % label
|
||||
if key in images:
|
||||
args = ["docker", "commit", images[key], key]
|
||||
subprocess.call(args)
|
||||
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def _get_stdout_as_lines(self, args):
|
||||
p = subprocess.Popen(args, stdout=subprocess.PIPE)
|
||||
out, _ = p.communicate()
|
||||
lines = out.split('\n')
|
||||
nonemtpy_lines = [l for l in lines if len(l.strip()) > 0]
|
||||
return nonemtpy_lines
|
||||
|
||||
def docker_inspect(self, id):
|
||||
lines = self._get_stdout_as_lines(["docker", "inspect", id])
|
||||
return json.loads("".join(lines))[0]
|
||||
|
|
@ -4,10 +4,11 @@ from proxysql_base_test import ProxySQLBaseTest
|
|||
|
||||
class OneBackendTest(ProxySQLBaseTest):
|
||||
|
||||
SCENARIO = "./scenarios/1backend"
|
||||
def _test_select_strings_returns_correct_result(self):
|
||||
|
||||
rows = self.run_query_proxysql("SELECT * FROM strings", "test")
|
||||
self.assertEqual(set([row[0] for row in rows]),
|
||||
set(['a', 'ab', 'abc', 'abcd']))
|
||||
|
||||
def test_select_strings_returns_correct_result(self):
|
||||
|
||||
rows = ProxySQLBaseTest.run_query_proxysql("SELECT * FROM strings", "test")
|
||||
self.assertEqual(set([row[0] for row in rows]),
|
||||
set(['a', 'ab', 'abc', 'abcd']))
|
||||
self.run_in_docker_scenarios(self._test_select_strings_returns_correct_result)
|
||||
|
|
@ -1,8 +1,5 @@
|
|||
import os
|
||||
import os.path
|
||||
import random
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from unittest import TestCase
|
||||
|
|
@ -11,6 +8,7 @@ from docker import Client
|
|||
from docker.utils import kwargs_from_env
|
||||
import MySQLdb
|
||||
|
||||
from docker_fleet import DockerFleet
|
||||
from proxysql_ping_thread import ProxySQL_Ping_Thread
|
||||
from proxysql_tests_config import ProxySQL_Tests_Config
|
||||
|
||||
|
|
@ -22,419 +20,28 @@ class ProxySQLBaseTest(TestCase):
|
|||
# Custom, per-test, config overrides
|
||||
CONFIG_OVERRIDES = {}
|
||||
|
||||
@classmethod
|
||||
def _startup_docker_services(cls):
|
||||
"""Start up all the docker services necessary to start this test.
|
||||
def setUp(self):
|
||||
self.docker_fleet = DockerFleet(config_overrides=ProxySQLBaseTest.CONFIG_OVERRIDES)
|
||||
|
||||
They are specified in the docker compose file specified in the variable
|
||||
cls.SCENARIO.
|
||||
"""
|
||||
# TODO(aismail): revive interactive mode
|
||||
#if cls.INTERACTIVE_TEST:
|
||||
# cls._compile_host_proxysql()
|
||||
# cls._connect_gdb_to_proxysql_within_container()
|
||||
#self._start_proxysql_pings()
|
||||
|
||||
# We have to perform docker-compose build + docker-compose up,
|
||||
# instead of just doing the latter because of a bug which will give a
|
||||
# 500 internal error for the Docker bug. When this is fixed, we should
|
||||
# remove this first extra step.
|
||||
subprocess.call(["docker-compose", "build"], cwd=cls.SCENARIO)
|
||||
subprocess.call(["docker-compose", "up", "-d"], cwd=cls.SCENARIO)
|
||||
def tearDown(self):
|
||||
# TODO(aismail): revive interactive mode
|
||||
#if cls.INTERACTIVE_TEST:
|
||||
# cls._gdb_process.wait()
|
||||
|
||||
@classmethod
|
||||
def _shutdown_docker_services(cls):
|
||||
"""Shut down all the docker services necessary to start this test.
|
||||
|
||||
They are specified in the docker compose file specified in the variable
|
||||
cls.SCENARIO.
|
||||
"""
|
||||
|
||||
subprocess.call(["docker-compose", "stop"], cwd=cls.SCENARIO)
|
||||
subprocess.call(["docker-compose", "rm", "-v --force"], cwd=cls.SCENARIO)
|
||||
|
||||
@classmethod
|
||||
def _get_proxysql_container(cls):
|
||||
"""Out of all the started docker containers, select the one which
|
||||
represents the proxy instance.
|
||||
|
||||
Note that this only supports one proxy instance for now. This method
|
||||
relies on interogating the Docker daemon via its REST API.
|
||||
"""
|
||||
|
||||
containers = Client(**kwargs_from_env()).containers()
|
||||
for container in containers:
|
||||
if 'proxysql' in container['Image']:
|
||||
return container
|
||||
|
||||
@classmethod
|
||||
def _get_mysql_containers(cls):
|
||||
"""Out of all the started docker containers, select the ones which
|
||||
represent the MySQL backend instances.
|
||||
|
||||
This method relies on interogating the Docker daemon via its REST API.
|
||||
"""
|
||||
|
||||
result = []
|
||||
containers = Client(**kwargs_from_env()).containers()
|
||||
for container in containers:
|
||||
if 'proxysql' not in container['Image']:
|
||||
result.append(container)
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def _populate_mysql_containers_with_dump(cls):
|
||||
"""Populates the started MySQL backend containers with the specified
|
||||
SQL dump file.
|
||||
|
||||
The reason for doing this __after__ the containers are started is
|
||||
because we want to keep them as generic as possible.
|
||||
"""
|
||||
|
||||
mysql_containers = cls._get_mysql_containers()
|
||||
# We have already added the SQL dump to the container by using
|
||||
# the ADD mysql command in the Dockerfile for mysql -- check it
|
||||
# out. The standard agreed location is at /tmp/schema.sql.
|
||||
#
|
||||
# Unfortunately we can't do this step at runtime due to limitations
|
||||
# on how transfer between host and container is supposed to work by
|
||||
# design. See the Dockerfile for MySQL for more details.
|
||||
for mysql_container in mysql_containers:
|
||||
container_id = mysql_container['Names'][0][1:]
|
||||
subprocess.call(["docker", "exec", container_id, "bash", "/tmp/import_schema.sh"])
|
||||
|
||||
@classmethod
|
||||
def _extract_hostgroup_from_container_name(cls, container_name):
|
||||
"""MySQL backend containers are named using a naming convention:
|
||||
backendXhostgroupY, where X and Y can be multi-digit numbers.
|
||||
This extracts the value of the hostgroup from the container name.
|
||||
|
||||
I made this choice because I wasn't able to find another easy way to
|
||||
associate arbitrary metadata with a Docker container through the
|
||||
docker compose file.
|
||||
"""
|
||||
|
||||
service_name = container_name.split('_')[1]
|
||||
return int(re.search(r'BACKEND(\d+)HOSTGROUP(\d+)', service_name).group(2))
|
||||
|
||||
@classmethod
|
||||
def _extract_port_number_from_uri(cls, uri):
|
||||
"""Given a Docker container URI (exposed as an environment variable by
|
||||
the host linking mechanism), extract the TCP port number from it."""
|
||||
return int(uri.split(':')[2])
|
||||
|
||||
@classmethod
|
||||
def _get_environment_variables_from_container(cls, container_name):
|
||||
"""Retrieve the environment variables from the given container.
|
||||
|
||||
This is useful because the host linking mechanism will expose
|
||||
connectivity information to the linked hosts by the use of environment
|
||||
variables.
|
||||
"""
|
||||
|
||||
output = Client(**kwargs_from_env()).execute(container_name, 'env')
|
||||
result = {}
|
||||
lines = output.split('\n')
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if len(line) == 0:
|
||||
continue
|
||||
(k, v) = line.split('=')
|
||||
result[k] = v
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def _populate_proxy_configuration_with_backends(cls):
|
||||
"""Populate ProxySQL's admin information with the MySQL backends
|
||||
and their associated hostgroups.
|
||||
|
||||
This is needed because I do not want to hardcode this into the ProxySQL
|
||||
config file of the test scenario, as it leaves more room for quick
|
||||
iteration.
|
||||
|
||||
In order to configure ProxySQL with the correct backends, we are using
|
||||
the MySQL admin interface of ProxySQL, and inserting rows into the
|
||||
`mysql_servers` table, which contains a list of which servers go into
|
||||
which hostgroup.
|
||||
"""
|
||||
config = ProxySQL_Tests_Config(overrides=cls.CONFIG_OVERRIDES)
|
||||
proxysql_container = cls._get_proxysql_container()
|
||||
mysql_containers = cls._get_mysql_containers()
|
||||
environment_variables = cls._get_environment_variables_from_container(
|
||||
proxysql_container['Names'][0][1:])
|
||||
|
||||
proxy_admin_connection = MySQLdb.connect(config.get('ProxySQL', 'hostname'),
|
||||
config.get('ProxySQL', 'admin_username'),
|
||||
config.get('ProxySQL', 'admin_password'),
|
||||
port=int(config.get('ProxySQL', 'admin_port')))
|
||||
cursor = proxy_admin_connection.cursor()
|
||||
|
||||
for mysql_container in mysql_containers:
|
||||
container_name = mysql_container['Names'][0][1:].upper()
|
||||
port_uri = environment_variables['%s_PORT' % container_name]
|
||||
port_no = cls._extract_port_number_from_uri(port_uri)
|
||||
ip = environment_variables['%s_PORT_%d_TCP_ADDR' % (container_name, port_no)]
|
||||
hostgroup = cls._extract_hostgroup_from_container_name(container_name)
|
||||
cursor.execute("INSERT INTO mysql_servers(hostgroup_id, hostname, port, status) "
|
||||
"VALUES(%d, '%s', %d, 'ONLINE')" %
|
||||
(hostgroup, ip, port_no))
|
||||
|
||||
cursor.execute("LOAD MYSQL SERVERS TO RUNTIME")
|
||||
cursor.close()
|
||||
proxy_admin_connection.close()
|
||||
|
||||
@classmethod
|
||||
def onerror(cls, function, path, excinfo):
|
||||
print("Error while trying to delete %s: %r" % (path, excinfo))
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# Always shutdown docker services because the previous test might have
|
||||
# left them in limbo.
|
||||
cls._shutdown_docker_services()
|
||||
|
||||
try:
|
||||
if os.path.exists('/tmp/proxysql-tests'):
|
||||
shutil.rmtree('/tmp/proxysql-tests/', onerror=cls.onerror)
|
||||
except:
|
||||
pass
|
||||
|
||||
os.mkdir('/tmp/proxysql-tests')
|
||||
os.system("cp -R " + os.path.dirname(__file__) + "/../* /tmp/proxysql-tests")
|
||||
|
||||
cls._startup_docker_services()
|
||||
|
||||
if cls.INTERACTIVE_TEST:
|
||||
cls._compile_host_proxysql()
|
||||
cls._connect_gdb_to_proxysql_within_container()
|
||||
|
||||
# First, wait for all backend servers to have their internal MySQL
|
||||
# started up
|
||||
mysql_credentials = cls.get_all_mysql_connection_credentials()
|
||||
for credential in mysql_credentials:
|
||||
cls.wait_for_mysql_connection_ok(**credential)
|
||||
proxysql_credentials = cls.get_proxysql_connection_credentials()
|
||||
cls.wait_for_mysql_connection_ok(**proxysql_credentials)
|
||||
proxysql_admin_credentials = cls.get_proxysql_admin_connection_credentials()
|
||||
cls.wait_for_mysql_connection_ok(**proxysql_admin_credentials)
|
||||
|
||||
# admin_test would be failing without this. Basically it means that
|
||||
# ProxySQL doesn't seem to behave well when starting it and stopping it
|
||||
# immediately after that.
|
||||
time.sleep(5)
|
||||
|
||||
cls._populate_mysql_containers_with_dump()
|
||||
cls._populate_proxy_configuration_with_backends()
|
||||
cls._start_proxysql_pings()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
try:
|
||||
cls.run_query_proxysql_admin("PROXYSQL SHUTDOWN")
|
||||
except:
|
||||
# This will throw an exception because it will forcefully shut down
|
||||
# the connection with the MySQL client.
|
||||
pass
|
||||
if cls.INTERACTIVE_TEST:
|
||||
cls._gdb_process.wait()
|
||||
# It's essential that pings are stopped __after__ the gdb process has
|
||||
# finished. This allows them to keep pinging ProxySQL in the background
|
||||
# while it's stuck waiting for user interaction (user interaction needed
|
||||
# in order to debug the problem causing it to crash).
|
||||
cls._stop_proxysql_pings()
|
||||
cls._shutdown_docker_services()
|
||||
#self._stop_proxysql_pings()
|
||||
pass
|
||||
|
||||
shutil.rmtree('/tmp/proxysql-tests/')
|
||||
|
||||
@classmethod
|
||||
def run_query_proxysql(cls, query, db,
|
||||
hostname=None, port=None,
|
||||
username=None, password=None,
|
||||
return_result=True):
|
||||
"""Run a query against the ProxySQL proxy and optionally return its
|
||||
results as a set of rows."""
|
||||
credentials = cls.get_proxysql_connection_credentials()
|
||||
proxy_connection = MySQLdb.connect(hostname or credentials['hostname'],
|
||||
username or credentials['username'],
|
||||
password or credentials['password'],
|
||||
port=int(port or credentials['port']),
|
||||
db=db)
|
||||
cursor = proxy_connection.cursor()
|
||||
cursor.execute(query)
|
||||
if return_result:
|
||||
rows = cursor.fetchall()
|
||||
cursor.close()
|
||||
proxy_connection.close()
|
||||
if return_result:
|
||||
return rows
|
||||
|
||||
@classmethod
|
||||
def run_query_proxysql_admin(cls, query, return_result=True):
|
||||
"""Run a query against the ProxySQL admin.
|
||||
|
||||
Note: we do not need to specify a db for this query, as it's always
|
||||
against the "main" database.
|
||||
TODO(andrei): revisit db assumption once stats databases from ProxySQL
|
||||
are accessible via the MySQL interface.
|
||||
"""
|
||||
credentials = cls.get_proxysql_admin_connection_credentials()
|
||||
proxy_connection = MySQLdb.connect(credentials['hostname'],
|
||||
credentials['username'],
|
||||
credentials['password'],
|
||||
port=int(credentials['port']),
|
||||
db='main')
|
||||
cursor = proxy_connection.cursor()
|
||||
cursor.execute(query)
|
||||
if return_result:
|
||||
rows = cursor.fetchall()
|
||||
cursor.close()
|
||||
proxy_connection.close()
|
||||
if return_result:
|
||||
return rows
|
||||
|
||||
@classmethod
|
||||
def mysql_connection_ok(cls, hostname, port, username, password, schema):
|
||||
"""Checks whether the MySQL server reachable at (hostname, port) is
|
||||
up or not. This is useful for waiting for ProxySQL/MySQL containers to
|
||||
start up correctly (meaning that the daemons running inside them have
|
||||
started to be able to respond to queries).
|
||||
"""
|
||||
try:
|
||||
db = MySQLdb.connect(host=hostname,
|
||||
user=username,
|
||||
passwd=password,
|
||||
db=schema,
|
||||
port=int(port))
|
||||
cursor = db.cursor()
|
||||
cursor.execute("select @@version_comment limit 1")
|
||||
results = cursor.fetchone()
|
||||
# Check if anything at all is returned
|
||||
if results:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
except MySQLdb.Error, e:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def wait_for_mysql_connection_ok(cls, hostname, port, username, password,
|
||||
max_retries=500, time_between_retries=1):
|
||||
|
||||
retries = 0
|
||||
result = False
|
||||
|
||||
while (not result) and (retries < max_retries):
|
||||
result = ProxySQLBaseTest.mysql_connection_ok(
|
||||
hostname=hostname,
|
||||
port=port,
|
||||
username=username,
|
||||
password=password,
|
||||
schema="information_schema"
|
||||
)
|
||||
if not result:
|
||||
retries += 1
|
||||
time.sleep(1)
|
||||
print("Trying again to connect to %s:%s (retries=%d)" % (hostname, port, retries))
|
||||
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def get_all_mysql_connection_credentials(cls, hostgroup=None):
|
||||
# Figure out which are the containers for the specified hostgroup
|
||||
mysql_backends = cls._get_mysql_containers()
|
||||
mysql_backends_in_hostgroup = []
|
||||
for backend in mysql_backends:
|
||||
container_name = backend['Names'][0][1:].upper()
|
||||
backend_hostgroup = cls._extract_hostgroup_from_container_name(container_name)
|
||||
|
||||
mysql_port_exposed=False
|
||||
if not backend.get('Ports'):
|
||||
continue
|
||||
for exposed_port in backend.get('Ports', []):
|
||||
if exposed_port['PrivatePort'] == 3306:
|
||||
mysql_port_exposed = True
|
||||
|
||||
if ((backend_hostgroup == hostgroup) or (hostgroup is None)) and mysql_port_exposed:
|
||||
mysql_backends_in_hostgroup.append(backend)
|
||||
|
||||
config = ProxySQL_Tests_Config(overrides=cls.CONFIG_OVERRIDES)
|
||||
hostname = config.get('ProxySQL', 'hostname')
|
||||
username = config.get('ProxySQL', 'username')
|
||||
password = config.get('ProxySQL', 'password')
|
||||
|
||||
result = []
|
||||
for container in mysql_backends_in_hostgroup:
|
||||
for exposed_port in container.get('Ports', []):
|
||||
if exposed_port['PrivatePort'] == 3306:
|
||||
mysql_port = exposed_port['PublicPort']
|
||||
result.append({
|
||||
'hostname': hostname,
|
||||
'port': mysql_port,
|
||||
'username': username,
|
||||
'password': password
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
@classmethod
|
||||
def get_mysql_connection_credentials(cls, hostgroup=0):
|
||||
|
||||
credentials = cls.get_all_mysql_connection_credentials(hostgroup=hostgroup)
|
||||
|
||||
if len(credentials) == 0:
|
||||
raise Exception('No backends with a publicly exposed port were '
|
||||
'found in hostgroup %d' % hostgroup)
|
||||
|
||||
return random.choice(credentials)
|
||||
|
||||
@classmethod
|
||||
def get_proxysql_connection_credentials(cls):
|
||||
config = ProxySQL_Tests_Config(overrides=cls.CONFIG_OVERRIDES)
|
||||
return {
|
||||
"hostname": config.get("ProxySQL", "hostname"),
|
||||
"port": config.get("ProxySQL", "port"),
|
||||
"username": config.get("ProxySQL", "username"),
|
||||
"password": config.get("ProxySQL", "password")
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def get_proxysql_admin_connection_credentials(cls):
|
||||
config = ProxySQL_Tests_Config(overrides=cls.CONFIG_OVERRIDES)
|
||||
return {
|
||||
"hostname": config.get("ProxySQL", "hostname"),
|
||||
"port": config.get("ProxySQL", "admin_port"),
|
||||
"username": config.get("ProxySQL", "admin_username"),
|
||||
"password": config.get("ProxySQL", "admin_password")
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def run_query_mysql(cls, query, db, return_result=True, hostgroup=0,
|
||||
username=None, password=None):
|
||||
"""Run a query against the MySQL backend and optionally return its
|
||||
results as a set of rows.
|
||||
|
||||
IMPORTANT: since the queries are actually ran against the MySQL backend,
|
||||
that backend needs to expose its MySQL port to the outside through
|
||||
docker compose's port mapping mechanism.
|
||||
|
||||
This will actually parse the docker-compose configuration file to
|
||||
retrieve the available backends and hostgroups and will pick a backend
|
||||
from the specified hostgroup."""
|
||||
|
||||
credentials = ProxySQLBaseTest.get_mysql_connection_credentials()
|
||||
mysql_connection = MySQLdb.connect(host=credentials['hostname'],
|
||||
user=credentials['username'],
|
||||
passwd=credentials['password'],
|
||||
port=int(credentials['port']),
|
||||
db=db)
|
||||
cursor = mysql_connection.cursor()
|
||||
cursor.execute(query)
|
||||
if return_result:
|
||||
rows = cursor.fetchall()
|
||||
cursor.close()
|
||||
mysql_connection.close()
|
||||
if return_result:
|
||||
return rows
|
||||
|
||||
@classmethod
|
||||
def run_sysbench_proxysql(cls, threads=4, time=60, db="test",
|
||||
def run_sysbench_proxysql(self, threads=4, time=60, db="test",
|
||||
username=None, password=None, port=None):
|
||||
"""Runs a sysbench test with the given parameters against the given
|
||||
ProxySQL instance.
|
||||
|
|
@ -444,8 +51,7 @@ class ProxySQLBaseTest(TestCase):
|
|||
container with it.
|
||||
"""
|
||||
|
||||
proxysql_container_id = ProxySQLBaseTest._get_proxysql_container()['Id']
|
||||
config = ProxySQL_Tests_Config(overrides=cls.CONFIG_OVERRIDES)
|
||||
config = ProxySQL_Tests_Config(overrides=ProxySQLBaseTest.CONFIG_OVERRIDES)
|
||||
hostname = config.get('ProxySQL', 'hostname')
|
||||
username = username or config.get('ProxySQL', 'username')
|
||||
password = password or config.get('ProxySQL', 'password')
|
||||
|
|
@ -471,12 +77,11 @@ class ProxySQLBaseTest(TestCase):
|
|||
"--mysql-port=%s" % port
|
||||
]
|
||||
|
||||
cls.run_bash_command_within_proxysql(params + ["prepare"])
|
||||
cls.run_bash_command_within_proxysql(params + ["run"])
|
||||
cls.run_bash_command_within_proxysql(params + ["cleanup"])
|
||||
self.run_bash_command_within_proxysql(params + ["prepare"])
|
||||
self.run_bash_command_within_proxysql(params + ["run"])
|
||||
self.run_bash_command_within_proxysql(params + ["cleanup"])
|
||||
|
||||
@classmethod
|
||||
def run_bash_command_within_proxysql(cls, params):
|
||||
def run_bash_command_within_proxysql(self, params):
|
||||
"""Run a bash command given as an array of tokens within the ProxySQL
|
||||
container.
|
||||
|
||||
|
|
@ -486,12 +91,11 @@ class ProxySQLBaseTest(TestCase):
|
|||
- running various debugging commands against the ProxySQL instance
|
||||
"""
|
||||
|
||||
proxysql_container_id = cls._get_proxysql_container()['Id']
|
||||
proxysql_container_id = self.docker_fleet.get_proxysql_container()
|
||||
exec_params = ["docker", "exec", proxysql_container_id] + params
|
||||
subprocess.call(exec_params)
|
||||
|
||||
@classmethod
|
||||
def _compile_host_proxysql(cls):
|
||||
def _compile_host_proxysql(self):
|
||||
"""Compile ProxySQL on the Docker host from which we're running the
|
||||
tests.
|
||||
|
||||
|
|
@ -504,8 +108,7 @@ class ProxySQLBaseTest(TestCase):
|
|||
subprocess.call(["make", "clean"])
|
||||
subprocess.call(["make"])
|
||||
|
||||
@classmethod
|
||||
def _connect_gdb_to_proxysql_within_container(cls):
|
||||
def _connect_gdb_to_proxysql_within_container(self):
|
||||
"""Connect a local gdb running on the docker host to the remote
|
||||
ProxySQL binary for remote debugging.
|
||||
|
||||
|
|
@ -518,22 +121,86 @@ class ProxySQLBaseTest(TestCase):
|
|||
shut down the process later on.
|
||||
"""
|
||||
|
||||
cls._gdb_process = subprocess.Popen(["gdb", "--command=gdb-commands.txt",
|
||||
"./proxysql"],
|
||||
cwd="./src")
|
||||
self._gdb_process = subprocess.Popen(["gdb", "--command=gdb-commands.txt",
|
||||
"./proxysql"],
|
||||
cwd="./src")
|
||||
|
||||
@classmethod
|
||||
def _start_proxysql_pings(cls):
|
||||
def _start_proxysql_pings(self):
|
||||
"""During the running of the tests, the test suite will continuously
|
||||
monitor the ProxySQL daemon in order to check that it's up.
|
||||
|
||||
This special thread will do exactly that."""
|
||||
config = ProxySQL_Tests_Config(overrides=cls.CONFIG_OVERRIDES)
|
||||
cls.ping_thread = ProxySQL_Ping_Thread(config)
|
||||
cls.ping_thread.start()
|
||||
config = ProxySQL_Tests_Config(overrides=ProxySQLBaseTest.CONFIG_OVERRIDES)
|
||||
self.ping_thread = ProxySQL_Ping_Thread(config)
|
||||
self.ping_thread.start()
|
||||
|
||||
@classmethod
|
||||
def _stop_proxysql_pings(cls):
|
||||
def _stop_proxysql_pings(self):
|
||||
"""Stop the special thread which pings the ProxySQL daemon."""
|
||||
cls.ping_thread.stop()
|
||||
cls.ping_thread.join()
|
||||
self.ping_thread.stop()
|
||||
self.ping_thread.join()
|
||||
|
||||
def run_query_proxysql(self, query, db,
|
||||
hostname=None, port=None,
|
||||
username=None, password=None,
|
||||
return_result=True):
|
||||
return self.docker_fleet.run_query_proxysql(query, db,
|
||||
hostname, port,
|
||||
username, password,
|
||||
return_result)
|
||||
|
||||
def run_query_proxysql_admin(self, query, return_result=True):
|
||||
return self.docker_fleet.run_query_proxysql_admin(query, return_result)
|
||||
|
||||
def run_query_mysql(self, query, db, return_result=True, hostgroup=0,
|
||||
username=None, password=None):
|
||||
return self.docker_fleet.run_query_mysql(query, db, return_result,
|
||||
hostgroup,
|
||||
username, password)
|
||||
|
||||
def run_query_mysql_container(self, query, db, container_id, return_result=True):
|
||||
return self.docker_fleet.run_query_mysql_container(query=query,
|
||||
db=db,
|
||||
container_id=container_id,
|
||||
return_result=return_result)
|
||||
|
||||
def get_proxysql_container(self):
|
||||
return self.docker_fleet.get_proxysql_container()
|
||||
|
||||
def get_mysql_containers(self, hostgroup=0):
|
||||
return self.docker_fleet.get_mysql_containers(hostgroup=hostgroup)
|
||||
|
||||
def get_environment_variables_from_container(self, container_id):
|
||||
return self.docker_fleet.get_environment_variables_from_container(container_id)
|
||||
|
||||
def get_tests_config(self):
|
||||
return ProxySQL_Tests_Config(overrides=ProxySQLBaseTest.CONFIG_OVERRIDES)
|
||||
|
||||
def docker_inspect(self, container_id):
|
||||
return self.docker_fleet.docker_inspect(container_id)
|
||||
|
||||
def run_in_docker_scenarios(self, f, scenarios=[], proxysql_filters={}, mysql_filters={}):
|
||||
"""Runs a function in a number of docker scenarios.
|
||||
|
||||
This is a helper for running your test assertions against different
|
||||
configurations without having to go the extra mile.
|
||||
"""
|
||||
|
||||
scenarios = self.docker_fleet.generate_scenarios(scenarios=scenarios,
|
||||
proxysql_filters=proxysql_filters,
|
||||
mysql_filters=mysql_filters)
|
||||
committed_images = set()
|
||||
copy_folder = True
|
||||
delete_folder = True
|
||||
for (i, scenario) in enumerate(scenarios):
|
||||
copy_folder = (i == 0)
|
||||
delete_folder = (i == len(scenarios) - 1)
|
||||
|
||||
folder = self.docker_fleet.start_temp_scenario(scenario, copy_folder)
|
||||
f()
|
||||
if scenario['proxysql_image'] not in committed_images:
|
||||
self.docker_fleet.commit_proxysql_image(scenario['proxysql_image'])
|
||||
committed_images.add(scenario['proxysql_image'])
|
||||
if scenario['mysql_image'] not in committed_images:
|
||||
self.docker_fleet.commit_proxysql_image(scenario['mysql_image'])
|
||||
committed_images.add(scenario['mysql_image'])
|
||||
self.docker_fleet.stop_temp_scenario(folder, delete_folder)
|
||||
|
|
@ -0,0 +1,180 @@
|
|||
import MySQLdb
|
||||
from MySQLdb import OperationalError
|
||||
from nose.tools import raises
|
||||
import random
|
||||
import time
|
||||
|
||||
from proxysql_base_test import ProxySQLBaseTest
|
||||
|
||||
class ReplicationTopologyAwareness(ProxySQLBaseTest):
|
||||
|
||||
def _test_insert_sent_through_proxysql_is_visible_in_slave_servers(self):
|
||||
self._start_replication()
|
||||
|
||||
random_string = ''.join(random.choice(['a', 'b', 'c', 'd', 'e']) for _ in xrange(10))
|
||||
q = "INSERT INTO strings(value) VALUES('%s')" % random_string
|
||||
self.run_query_proxysql(q, "test")
|
||||
|
||||
# Give slaves the time to catch up
|
||||
time.sleep(5)
|
||||
|
||||
slave_containers = self.get_mysql_containers(hostgroup=1)
|
||||
for slave_container_id in slave_containers:
|
||||
q = "SELECT * FROM strings"
|
||||
rows = self.run_query_mysql_container("SELECT * FROM strings",
|
||||
"test",
|
||||
slave_container_id)
|
||||
self.assertEqual(set([row[0] for row in rows]),
|
||||
set(['a', 'ab', 'abc', 'abcd', random_string]))
|
||||
|
||||
def test_insert_sent_through_proxysql_is_visible_in_slave_servers(self):
|
||||
self.run_in_docker_scenarios(self._test_insert_sent_through_proxysql_is_visible_in_slave_servers,
|
||||
scenarios=['5backends-replication'])
|
||||
|
||||
def _test_promote_slave_to_master_reflected_in_proxysql_admin_tables(self):
|
||||
self._start_replication()
|
||||
time.sleep(5)
|
||||
self._promote_first_slave_to_master()
|
||||
self._check_slave_promotion_reflected_in_proxysql_admin()
|
||||
|
||||
def test_promote_slave_to_master_reflected_in_proxysql_admin_tables(self):
|
||||
self.run_in_docker_scenarios(self._test_promote_slave_to_master_reflected_in_proxysql_admin_tables,
|
||||
scenarios=['5backends-replication'])
|
||||
|
||||
def _wait_for_slave_to_catch_up(self, slave_container_id):
|
||||
# Wait for the slave to catch up with the master
|
||||
slave_caught_up = False
|
||||
while not slave_caught_up:
|
||||
slave_status = self.run_query_mysql_container(
|
||||
'SHOW SLAVE STATUS',
|
||||
'information_schema',
|
||||
slave_container_id
|
||||
)
|
||||
slave_caught_up = slave_status[0][44].startswith(
|
||||
'Slave has read all relay log')
|
||||
|
||||
def _start_replication(self):
|
||||
master_container = self.get_mysql_containers(hostgroup=0)[0]
|
||||
slave_containers = self.get_mysql_containers(hostgroup=1)
|
||||
proxysql_container = self.get_proxysql_container()
|
||||
|
||||
# connect the slaves to the master
|
||||
for slave_container_id in slave_containers:
|
||||
env = self.get_environment_variables_from_container(slave_container_id)
|
||||
# master will have a local (hostname, port) address using which the
|
||||
# slave can access it.
|
||||
master_local_hostname = env['BACKEND1HOSTGROUP0_PORT_3306_TCP_ADDR']
|
||||
master_local_port = env['BACKEND1HOSTGROUP0_PORT_3306_TCP_PORT']
|
||||
config = self.get_tests_config()
|
||||
username = config.get('ProxySQL', 'username')
|
||||
password = config.get('ProxySQL', 'password')
|
||||
args = (master_local_hostname, master_local_port, username, password)
|
||||
q = "CHANGE MASTER TO MASTER_HOST='%s', MASTER_PORT=%s, MASTER_USER='%s', MASTER_PASSWORD='%s', MASTER_AUTO_POSITION = 1" % args
|
||||
self.run_query_mysql_container(q, 'information_schema', slave_container_id)
|
||||
self.run_query_mysql_container('START SLAVE', 'information_schema', slave_container_id)
|
||||
self.run_query_mysql_container('SET GLOBAL read_only=ON', 'information_schema', slave_container_id)
|
||||
self._wait_for_slave_to_catch_up(slave_container_id)
|
||||
|
||||
# Let ProxySQL know that:
|
||||
# - the readers (slaves) are in hostgroup 1
|
||||
# - the writer (master) is in hostgroup 0
|
||||
self.run_query_proxysql_admin("INSERT INTO mysql_replication_hostgroups(writer_hostgroup, reader_hostgroup) VALUES(0, 1)")
|
||||
|
||||
def _promote_first_slave_to_master(self):
|
||||
# Following steps from https://dev.mysql.com/doc/refman/5.6/en/replication-solutions-switch.html
|
||||
|
||||
# Send STOP SLAVE IO_THREAD to all slave
|
||||
master_container = self.get_mysql_containers(hostgroup=0)[0]
|
||||
slave_containers = self.get_mysql_containers(hostgroup=1)
|
||||
for slave_container_id in slave_containers:
|
||||
self.run_query_mysql_container('STOP SLAVE IO_THREAD',
|
||||
'information_schema',
|
||||
slave_container_id)
|
||||
self._wait_for_slave_to_catch_up(slave_container_id)
|
||||
|
||||
# Find out from the metadata which of the slaves is the one to be
|
||||
# promoted as a master
|
||||
first_slave = None
|
||||
for slave_container_id in slave_containers:
|
||||
meta = self.docker_inspect(slave_container_id)
|
||||
if 'NEW_MASTER=True' in meta['Config']['Env']:
|
||||
first_slave = slave_container_id
|
||||
first_slave_ip = meta['NetworkSettings']['IPAddress']
|
||||
|
||||
# Promote slave1 to a master
|
||||
self.run_query_mysql_container('SET GLOBAL read_only=OFF',
|
||||
'information_schema',
|
||||
first_slave)
|
||||
self.run_query_mysql_container('STOP SLAVE',
|
||||
'information_schema',
|
||||
first_slave)
|
||||
self.run_query_mysql_container('RESET MASTER',
|
||||
'information_schema',
|
||||
first_slave)
|
||||
|
||||
# Point the other slaves to the newly elected master
|
||||
for slave_container_id in slave_containers:
|
||||
if slave_container_id == first_slave:
|
||||
continue
|
||||
|
||||
self.run_query_mysql_container('STOP SLAVE',
|
||||
'information_schema',
|
||||
slave_container_id)
|
||||
self.run_query_mysql_container("CHANGE MASTER TO MASTER_HOST = '%s'" % first_slave_ip,
|
||||
'information_schema',
|
||||
slave_container_id)
|
||||
self.run_query_mysql_container('START SLAVE',
|
||||
'information_schema',
|
||||
slave_container_id)
|
||||
|
||||
# Point the old master to the new master (that was previously a slave)
|
||||
config = self.get_tests_config()
|
||||
username = config.get('ProxySQL', 'username')
|
||||
password = config.get('ProxySQL', 'password')
|
||||
self.run_query_mysql_container('SET GLOBAL read_only=ON',
|
||||
'information_schema',
|
||||
master_container)
|
||||
q = "CHANGE MASTER TO MASTER_HOST = '%s', MASTER_PORT = 3306, MASTER_USER = '%s', MASTER_PASSWORD = '%s', MASTER_AUTO_POSITION = 1"
|
||||
self.run_query_mysql_container(q % (first_slave_ip, username, password),
|
||||
'information_schema',
|
||||
master_container)
|
||||
self.run_query_mysql_container('RESET SLAVE',
|
||||
'information_schema',
|
||||
master_container)
|
||||
self.run_query_mysql_container('START SLAVE',
|
||||
'information_schema',
|
||||
master_container)
|
||||
|
||||
# Wait for the slaves to catch up with the new master
|
||||
new_slaves = set(slave_containers)
|
||||
new_slaves.remove(first_slave)
|
||||
new_slaves.add(master_container)
|
||||
for slave_container_id in new_slaves:
|
||||
self._wait_for_slave_to_catch_up(slave_container_id)
|
||||
|
||||
def _check_slave_promotion_reflected_in_proxysql_admin(self):
|
||||
# Determine mapping from container IDs to IPs
|
||||
old_master_container = self.get_mysql_containers(hostgroup=0)[0]
|
||||
old_slave_containers = self.get_mysql_containers(hostgroup=1)
|
||||
slave_ips = set()
|
||||
old_master_ip = self.docker_inspect(old_master_container)['NetworkSettings']['IPAddress']
|
||||
slave_ips.add(old_master_ip)
|
||||
for slave_container_id in old_slave_containers:
|
||||
meta = self.docker_inspect(slave_container_id)
|
||||
slave_ip = meta['NetworkSettings']['IPAddress']
|
||||
if 'NEW_MASTER=True' in meta['Config']['Env']:
|
||||
master_ip = slave_ip
|
||||
else:
|
||||
slave_ips.add(slave_ip)
|
||||
|
||||
rows = self.run_query_proxysql_admin("SELECT * FROM mysql_servers")
|
||||
hostgroups = {}
|
||||
for row in rows:
|
||||
hostgroups[row[1]] = int(row[0])
|
||||
|
||||
# First slave is now a master, thus should be present in the writer hostgroup
|
||||
self.assertEqual(hostgroups[master_ip], 0)
|
||||
|
||||
# The old master, and the other slaves should still be in the slave hostgroup
|
||||
for ip in slave_ips:
|
||||
self.assertEqual(hostgroups[ip], 1)
|
||||
|
|
@ -2,7 +2,8 @@ from proxysql_base_test import ProxySQLBaseTest
|
|||
|
||||
class SysBenchTest(ProxySQLBaseTest):
|
||||
|
||||
SCENARIO = "./scenarios/1backend"
|
||||
def _test_proxy_doesnt_crash_under_mild_sysbench_load(self):
|
||||
self.run_sysbench_proxysql()
|
||||
|
||||
def test_proxy_doesnt_crash_under_mild_sysbench_load(self):
|
||||
ProxySQLBaseTest.run_sysbench_proxysql()
|
||||
self.run_in_docker_scenarios(self._test_proxy_doesnt_crash_under_mild_sysbench_load)
|
||||
Загрузка…
Ссылка в новой задаче