diff --git a/examples/compose/README.md b/examples/compose/README.md index 82528bbde4..b4f77cd7b4 100644 --- a/examples/compose/README.md +++ b/examples/compose/README.md @@ -5,6 +5,30 @@ To understand it better, you can run it. First you will need to [install docker-compose](https://docs.docker.com/compose/install/). +### Programatically create Vitess configuration for Docker +To create a configuration to your specifications, run vtcompose. Creates corresponding docker-compose file, vschema files per keyspace, and loads schemas. +``` +vitess/examples/compose$ go run vtcompose.go +``` + +Use `-h` or `--help` to get list of flags with descriptions. + +Flags available: +* **baseDockerComposeFile** - Specifies starting docker-compose yaml file. +* **baseVschemaFile** - Specifies starting vschema json file. +* **topologyFlags** - Specifies Vitess topology flags config +* **webPort** - Specifies web port to be used. +* **gRpcPort** - Specifies gRPC port to be used. +* **mySqlPort** - Specifies mySql port to be used. +* **cell** - `Specifies Vitess cell name to be used. +* **keyspaceData** - List of `keyspace_name:num_of_shards:num_of_replica_tablets:schema_file_names:lookup_keyspace_name` separated by ' '. + * This is where you specify most of the data for the program to build your vSchema and docker-compose files. + * Use `0` for `num_of_shards` to specify an unsharded keyspace + ``` + e.g "test_keyspace:2:1:create_messages.sql,create_tokens.sql:lookup_keyspace lookup_keyspace:1:1:create_tokens_token_lookup.sql,create_messages_message_lookup.sql" + ``` + + ### Start the cluster To start Consul(which saves the topology config), vtctld, vtgate and a few vttablets with MySQL running on them. ``` @@ -19,15 +43,17 @@ vitess/examples/compose$ docker-compose logs -f vtgate ``` ### Load the schema -We need to create a few tables into our new cluster. To do that, we can run the `ApplySchema` command. -``` -vitess/examples/compose$ ./lvtctl.sh ApplySchema -sql "$(cat create_test_table.sql)" test_keyspace +***Note: Should not be needed if VtCompose was used.*** + +We need to create a few tables into our new cluster. To do that, we can run the `ApplySchema` command. +``` +vitess/examples/compose$ ./lvtctl.sh ApplySchema -sql "$(cat create_test_table.sql)" test_keyspace ``` -### Create Vschema -Create Vschema -``` -vitess/examples/compose$ ./lvtctl.sh ApplyVschema -vschema '{"tables": {"messages": {} } }' test_keyspace +### Create Vschema +Create Vschema (should not be needed if VtCompose was used) +``` +vitess/examples/compose$ ./lvtctl.sh ApplyVschema -vschema '{"tables": {"messages": {} } }' test_keyspace ``` ### Run the client to insert and read some data @@ -40,7 +66,7 @@ vitess/examples/compose$ ./client.sh ### Connect to vgate and run queries vtgate responds to the MySQL protocol, so we can connect to it using the default MySQL client command line. ``` -vitess/examples/compose$ ./lmysql.sh --port=15306 --host=127.0.0.1 +vitess/examples/compose$ mysql --port=15306 --host=127.0.0.1 ``` @@ -113,15 +139,15 @@ vitess/examples/compose$ ./lfixrepl.sh ``` ### Apply Vschema -Apply Vschema for the unsharded keyspace -``` -vitess/examples/compose$ ./lvtctl.sh ApplyVschema -vschema '{"sharded":false, "tables": {"*": {} } }' external_db_name +Apply Vschema for the unsharded keyspace +``` +vitess/examples/compose$ ./lvtctl.sh ApplyVschema -vschema '{"sharded":false, "tables": {"*": {} } }' external_db_name ``` ### Connect to vgate and run queries vtgate responds to the MySQL protocol, so we can connect to it using the default MySQL client command line. ```sh -vitess/examples/compose$ ./lmysql.sh --port=15306 --host= +vitess/examples/compose$ mysql --port=15306 --host= mysql> show databases; +--------------------+ @@ -157,7 +183,6 @@ mysql> show tables; ## Helper Scripts The following helper scripts are included to help you perform various actions easily * vitess/examples/compose/lvtctl.sh -* vitess/examples/compose/lmysql.sh * vitess/examples/compose/lfixrepl.sh You may run them as below diff --git a/examples/compose/lmysql.sh b/examples/compose/lmysql.sh deleted file mode 100644 index fb67c619d9..0000000000 --- a/examples/compose/lmysql.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Enable tty for Windows users using git-bash or cygwin -if [[ "$OSTYPE" == "msys" ]]; then - # Lightweight shell and GNU utilities compiled for Windows (part of MinGW) - tty=winpty -fi - -# This is a convenience script to run mysql client against the local example. -exec $tty docker-compose exec ${CS:-vttablet1} mysql "$@" diff --git a/examples/compose/lvtctl.sh b/examples/compose/lvtctl.sh index 83d9a7c3c6..d95f1b75d4 100755 --- a/examples/compose/lvtctl.sh +++ b/examples/compose/lvtctl.sh @@ -21,4 +21,4 @@ if [[ "$OSTYPE" == "msys" ]]; then fi # This is a convenience script to run vtctlclient against the local example. -exec $tty docker-compose exec ${CS:-vttablet1} vtctlclient -server vtctld:15999 "$@" +exec docker-compose run vtctld vtctlclient -server vtctld:15999 "$@" diff --git a/examples/compose/tables/create_messages.sql b/examples/compose/tables/create_messages.sql index 9fc19f50b6..44c4cc16e8 100644 --- a/examples/compose/tables/create_messages.sql +++ b/examples/compose/tables/create_messages.sql @@ -1,6 +1,6 @@ CREATE TABLE messages ( page BIGINT(20) UNSIGNED, - time_created_ns timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - message VARCHAR(1000), + time_created_ns BIGINT(20) UNSIGNED, + message VARCHAR(10000), PRIMARY KEY (page, time_created_ns) ) ENGINE=InnoDB; \ No newline at end of file diff --git a/examples/compose/tables/create_tokens.sql b/examples/compose/tables/create_tokens.sql index 3300ea69b0..69f1be03de 100644 --- a/examples/compose/tables/create_tokens.sql +++ b/examples/compose/tables/create_tokens.sql @@ -1,6 +1,6 @@ CREATE TABLE tokens ( page BIGINT(20) UNSIGNED, - time_created_ns timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + time_created_ns BIGINT(20) UNSIGNED, token VARCHAR(255) DEFAULT NULL, PRIMARY KEY (page, time_created_ns) ) ENGINE=InnoDB; \ No newline at end of file diff --git a/examples/compose/vtcompose.go b/examples/compose/vtcompose.go index 31a84fcd60..127c190d4d 100644 --- a/examples/compose/vtcompose.go +++ b/examples/compose/vtcompose.go @@ -19,7 +19,7 @@ import ( var ( tabletsUsed = 0 - baseYamlFile = flag.String("base_yaml", "docker-compose.base.yml", "Starting docker-compose yaml") + baseDockerComposeFile = flag.String("base_yaml", "docker-compose.base.yml", "Starting docker-compose yaml") baseVschemaFile = flag.String("base_vschema", "base_vschema.json", "Starting vschema json") topologyFlags = flag.String("topologyFlags", @@ -29,7 +29,7 @@ var ( gRpcPort = flag.String("gRpcPort", "15999", "gRPC port to be used") mySqlPort = flag.String("mySqlPort", "15306", "mySql port to be used") cell = flag.String("cell", "test", "Vitess Cell name") - keyspaceData = flag.String("keyspaces", "test_keyspace:2:1:create_messages.sql,create_tokens.sql:lookup_keyspace unsharded_keyspace:0:0:create_dinosaurs.sql,create_eggs.sql", "List of keyspace_name:num_of_shards:num_of_replica_tablets:schema_files:lookup_keyspace_name") + keyspaceData = flag.String("keyspaces", "test_keyspace:2:1:create_messages.sql,create_tokens.sql unsharded_keyspace:0:0:create_dinosaurs.sql,create_eggs.sql", "List of keyspace_name:num_of_shards:num_of_replica_tablets:schema_files:lookup_keyspace_name seperated by ' '") ) type keyspaceInfo struct { @@ -96,9 +96,9 @@ func main() { } // Docker Compose File Patches - dockerFile := readFile(*baseYamlFile) - dockerFile = applyDockerComposePatches(dockerFile, keyspaceInfoMap) - writeFile(dockerFile, "docker-compose.yml") + dockerComposeFile := readFile(*baseDockerComposeFile) + dockerComposeFile = applyDockerComposePatches(dockerComposeFile, keyspaceInfoMap) + writeFile(dockerComposeFile, "docker-compose.yml") } func applyFilePatch(dockerYaml []byte, patchFile string) []byte { @@ -287,7 +287,7 @@ func writeFile (file []byte, fileName string) { } } -func applyKeyspaceDependentPatches(dockerFile []byte, keyspaceData keyspaceInfo) []byte { +func applyKeyspaceDependentPatches(dockerComposeFile []byte, keyspaceData keyspaceInfo) []byte { tabAlias := 0 + tabletsUsed*100 shard := "-" var masterTablets []string @@ -302,12 +302,12 @@ func applyKeyspaceDependentPatches(dockerFile []byte, keyspaceData keyspaceInfo) masterTablets = append(masterTablets, strconv.Itoa((i+1)*100+1)) } - dockerFile = applyInMemoryPatch(dockerFile, generateSchemaload(masterTablets, "", keyspaceData.keyspace)) + dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateSchemaload(masterTablets, "", keyspaceData.keyspace)) // Append Master and Replica Tablets if keyspaceData.shards < 2 { tabAlias = tabAlias + 100 - dockerFile = applyTabletPatches(dockerFile, tabAlias, shard, keyspaceData) + dockerComposeFile = applyTabletPatches(dockerComposeFile, tabAlias, shard, keyspaceData) } else { // Determine shard range for i:=0; i < keyspaceData.shards; i++ { @@ -319,37 +319,37 @@ func applyKeyspaceDependentPatches(dockerFile []byte, keyspaceData keyspaceInfo) shard = fmt.Sprintf("%x-%x", interval*(i) ,interval*(i+1)) } tabAlias = tabAlias + 100 - dockerFile = applyTabletPatches(dockerFile, tabAlias, shard, keyspaceData) + dockerComposeFile = applyTabletPatches(dockerComposeFile, tabAlias, shard, keyspaceData) } } tabletsUsed += len(masterTablets) - return dockerFile + return dockerComposeFile } -func applyDefaultDockerPatches(dockerFile []byte) []byte { - dockerFile = applyInMemoryPatch(dockerFile, generateVtctld()) - dockerFile = applyInMemoryPatch(dockerFile, generateVtgate()) - dockerFile = applyInMemoryPatch(dockerFile, generateVtwork()) - return dockerFile +func applyDefaultDockerPatches(dockerComposeFile []byte) []byte { + dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtctld()) + dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtgate()) + dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtwork()) + return dockerComposeFile } -func applyDockerComposePatches(dockerFile []byte, keyspaceInfoMap map[string]keyspaceInfo) []byte { +func applyDockerComposePatches(dockerComposeFile []byte, keyspaceInfoMap map[string]keyspaceInfo) []byte { //Vtctld, vtgate, vtwork, schemaload patches - dockerFile = applyDefaultDockerPatches(dockerFile) + dockerComposeFile = applyDefaultDockerPatches(dockerComposeFile) for _, keyspaceData := range keyspaceInfoMap { - dockerFile = applyKeyspaceDependentPatches(dockerFile, keyspaceData) + dockerComposeFile = applyKeyspaceDependentPatches(dockerComposeFile, keyspaceData) } - return dockerFile + return dockerComposeFile } -func applyTabletPatches(dockerFile []byte, tabAlias int, shard string, keyspaceData keyspaceInfo) []byte { - dockerFile = applyInMemoryPatch(dockerFile, generateDefaultTablet(strconv.Itoa(tabAlias+1), shard, "master", keyspaceData.keyspace)) +func applyTabletPatches(dockerComposeFile []byte, tabAlias int, shard string, keyspaceData keyspaceInfo) []byte { + dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateDefaultTablet(strconv.Itoa(tabAlias+1), shard, "master", keyspaceData.keyspace)) for i:=0; i < keyspaceData.replicaTablets; i++ { - dockerFile = applyInMemoryPatch(dockerFile, generateDefaultTablet(strconv.Itoa(tabAlias+ 2 + i), shard, "replica", keyspaceData.keyspace)) + dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateDefaultTablet(strconv.Itoa(tabAlias+ 2 + i), shard, "replica", keyspaceData.keyspace)) } - return dockerFile + return dockerComposeFile } // Default Tablet