зеркало из https://github.com/github/vitess-gh.git
Updated README to include vtcompose instructions, and changed a variable name
Signed-off-by: JavierR14 <javier@squareup.com>
This commit is contained in:
JavierR14
Родитель
098695259b
Коммит
5a2725537f
|
|
@ -5,6 +5,30 @@ To understand it better, you can run it.
|
|||
|
||||
First you will need to [install docker-compose](https://docs.docker.com/compose/install/).
|
||||
|
||||
### Programatically create Vitess configuration for Docker
|
||||
To create a configuration to your specifications, run vtcompose. Creates corresponding docker-compose file, vschema files per keyspace, and loads schemas.
|
||||
```
|
||||
vitess/examples/compose$ go run vtcompose.go
|
||||
```
|
||||
|
||||
Use `-h` or `--help` to get list of flags with descriptions.
|
||||
|
||||
Flags available:
|
||||
* **baseDockerComposeFile** - Specifies starting docker-compose yaml file.
|
||||
* **baseVschemaFile** - Specifies starting vschema json file.
|
||||
* **topologyFlags** - Specifies Vitess topology flags config
|
||||
* **webPort** - Specifies web port to be used.
|
||||
* **gRpcPort** - Specifies gRPC port to be used.
|
||||
* **mySqlPort** - Specifies mySql port to be used.
|
||||
* **cell** - `Specifies Vitess cell name to be used.
|
||||
* **keyspaceData** - List of `keyspace_name:num_of_shards:num_of_replica_tablets:schema_file_names:<optional>lookup_keyspace_name` separated by ' '.
|
||||
* This is where you specify most of the data for the program to build your vSchema and docker-compose files.
|
||||
* Use `0` for `num_of_shards` to specify an unsharded keyspace
|
||||
```
|
||||
e.g "test_keyspace:2:1:create_messages.sql,create_tokens.sql:lookup_keyspace lookup_keyspace:1:1:create_tokens_token_lookup.sql,create_messages_message_lookup.sql"
|
||||
```
|
||||
|
||||
|
||||
### Start the cluster
|
||||
To start Consul(which saves the topology config), vtctld, vtgate and a few vttablets with MySQL running on them.
|
||||
```
|
||||
|
|
@ -19,15 +43,17 @@ vitess/examples/compose$ docker-compose logs -f vtgate
|
|||
```
|
||||
|
||||
### Load the schema
|
||||
We need to create a few tables into our new cluster. To do that, we can run the `ApplySchema` command.
|
||||
```
|
||||
vitess/examples/compose$ ./lvtctl.sh ApplySchema -sql "$(cat create_test_table.sql)" test_keyspace
|
||||
***Note: Should not be needed if VtCompose was used.***
|
||||
|
||||
We need to create a few tables into our new cluster. To do that, we can run the `ApplySchema` command.
|
||||
```
|
||||
vitess/examples/compose$ ./lvtctl.sh ApplySchema -sql "$(cat create_test_table.sql)" test_keyspace
|
||||
```
|
||||
|
||||
### Create Vschema
|
||||
Create Vschema
|
||||
```
|
||||
vitess/examples/compose$ ./lvtctl.sh ApplyVschema -vschema '{"tables": {"messages": {} } }' test_keyspace
|
||||
### Create Vschema
|
||||
Create Vschema (should not be needed if VtCompose was used)
|
||||
```
|
||||
vitess/examples/compose$ ./lvtctl.sh ApplyVschema -vschema '{"tables": {"messages": {} } }' test_keyspace
|
||||
```
|
||||
|
||||
### Run the client to insert and read some data
|
||||
|
|
@ -40,7 +66,7 @@ vitess/examples/compose$ ./client.sh
|
|||
### Connect to vgate and run queries
|
||||
vtgate responds to the MySQL protocol, so we can connect to it using the default MySQL client command line.
|
||||
```
|
||||
vitess/examples/compose$ ./lmysql.sh --port=15306 --host=127.0.0.1
|
||||
vitess/examples/compose$ mysql --port=15306 --host=127.0.0.1
|
||||
```
|
||||
|
||||
|
||||
|
|
@ -113,15 +139,15 @@ vitess/examples/compose$ ./lfixrepl.sh
|
|||
```
|
||||
|
||||
### Apply Vschema
|
||||
Apply Vschema for the unsharded keyspace
|
||||
```
|
||||
vitess/examples/compose$ ./lvtctl.sh ApplyVschema -vschema '{"sharded":false, "tables": {"*": {} } }' external_db_name
|
||||
Apply Vschema for the unsharded keyspace
|
||||
```
|
||||
vitess/examples/compose$ ./lvtctl.sh ApplyVschema -vschema '{"sharded":false, "tables": {"*": {} } }' external_db_name
|
||||
```
|
||||
|
||||
### Connect to vgate and run queries
|
||||
vtgate responds to the MySQL protocol, so we can connect to it using the default MySQL client command line.
|
||||
```sh
|
||||
vitess/examples/compose$ ./lmysql.sh --port=15306 --host=<host of machine containers are running in e.g. 127.0.0.1, docker-machine ip e.t.c>
|
||||
vitess/examples/compose$ mysql --port=15306 --host=<host of machine containers are running in e.g. 127.0.0.1, docker-machine ip e.t.c>
|
||||
|
||||
mysql> show databases;
|
||||
+--------------------+
|
||||
|
|
@ -157,7 +183,6 @@ mysql> show tables;
|
|||
## Helper Scripts
|
||||
The following helper scripts are included to help you perform various actions easily
|
||||
* vitess/examples/compose/lvtctl.sh
|
||||
* vitess/examples/compose/lmysql.sh
|
||||
* vitess/examples/compose/lfixrepl.sh
|
||||
|
||||
You may run them as below
|
||||
|
|
|
|||
|
|
@ -1,24 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2019 The Vitess Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Enable tty for Windows users using git-bash or cygwin
|
||||
if [[ "$OSTYPE" == "msys" ]]; then
|
||||
# Lightweight shell and GNU utilities compiled for Windows (part of MinGW)
|
||||
tty=winpty
|
||||
fi
|
||||
|
||||
# This is a convenience script to run mysql client against the local example.
|
||||
exec $tty docker-compose exec ${CS:-vttablet1} mysql "$@"
|
||||
|
|
@ -21,4 +21,4 @@ if [[ "$OSTYPE" == "msys" ]]; then
|
|||
fi
|
||||
|
||||
# This is a convenience script to run vtctlclient against the local example.
|
||||
exec $tty docker-compose exec ${CS:-vttablet1} vtctlclient -server vtctld:15999 "$@"
|
||||
exec docker-compose run vtctld vtctlclient -server vtctld:15999 "$@"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
CREATE TABLE messages (
|
||||
page BIGINT(20) UNSIGNED,
|
||||
time_created_ns timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
message VARCHAR(1000),
|
||||
time_created_ns BIGINT(20) UNSIGNED,
|
||||
message VARCHAR(10000),
|
||||
PRIMARY KEY (page, time_created_ns)
|
||||
) ENGINE=InnoDB;
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
CREATE TABLE tokens (
|
||||
page BIGINT(20) UNSIGNED,
|
||||
time_created_ns timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
time_created_ns BIGINT(20) UNSIGNED,
|
||||
token VARCHAR(255) DEFAULT NULL,
|
||||
PRIMARY KEY (page, time_created_ns)
|
||||
) ENGINE=InnoDB;
|
||||
|
|
@ -19,7 +19,7 @@ import (
|
|||
|
||||
var (
|
||||
tabletsUsed = 0
|
||||
baseYamlFile = flag.String("base_yaml", "docker-compose.base.yml", "Starting docker-compose yaml")
|
||||
baseDockerComposeFile = flag.String("base_yaml", "docker-compose.base.yml", "Starting docker-compose yaml")
|
||||
baseVschemaFile = flag.String("base_vschema", "base_vschema.json", "Starting vschema json")
|
||||
|
||||
topologyFlags = flag.String("topologyFlags",
|
||||
|
|
@ -29,7 +29,7 @@ var (
|
|||
gRpcPort = flag.String("gRpcPort", "15999", "gRPC port to be used")
|
||||
mySqlPort = flag.String("mySqlPort", "15306", "mySql port to be used")
|
||||
cell = flag.String("cell", "test", "Vitess Cell name")
|
||||
keyspaceData = flag.String("keyspaces", "test_keyspace:2:1:create_messages.sql,create_tokens.sql:lookup_keyspace unsharded_keyspace:0:0:create_dinosaurs.sql,create_eggs.sql", "List of keyspace_name:num_of_shards:num_of_replica_tablets:schema_files:<optional>lookup_keyspace_name")
|
||||
keyspaceData = flag.String("keyspaces", "test_keyspace:2:1:create_messages.sql,create_tokens.sql unsharded_keyspace:0:0:create_dinosaurs.sql,create_eggs.sql", "List of keyspace_name:num_of_shards:num_of_replica_tablets:schema_files:<optional>lookup_keyspace_name seperated by ' '")
|
||||
)
|
||||
|
||||
type keyspaceInfo struct {
|
||||
|
|
@ -96,9 +96,9 @@ func main() {
|
|||
}
|
||||
|
||||
// Docker Compose File Patches
|
||||
dockerFile := readFile(*baseYamlFile)
|
||||
dockerFile = applyDockerComposePatches(dockerFile, keyspaceInfoMap)
|
||||
writeFile(dockerFile, "docker-compose.yml")
|
||||
dockerComposeFile := readFile(*baseDockerComposeFile)
|
||||
dockerComposeFile = applyDockerComposePatches(dockerComposeFile, keyspaceInfoMap)
|
||||
writeFile(dockerComposeFile, "docker-compose.yml")
|
||||
}
|
||||
|
||||
func applyFilePatch(dockerYaml []byte, patchFile string) []byte {
|
||||
|
|
@ -287,7 +287,7 @@ func writeFile (file []byte, fileName string) {
|
|||
}
|
||||
}
|
||||
|
||||
func applyKeyspaceDependentPatches(dockerFile []byte, keyspaceData keyspaceInfo) []byte {
|
||||
func applyKeyspaceDependentPatches(dockerComposeFile []byte, keyspaceData keyspaceInfo) []byte {
|
||||
tabAlias := 0 + tabletsUsed*100
|
||||
shard := "-"
|
||||
var masterTablets []string
|
||||
|
|
@ -302,12 +302,12 @@ func applyKeyspaceDependentPatches(dockerFile []byte, keyspaceData keyspaceInfo)
|
|||
masterTablets = append(masterTablets, strconv.Itoa((i+1)*100+1))
|
||||
}
|
||||
|
||||
dockerFile = applyInMemoryPatch(dockerFile, generateSchemaload(masterTablets, "", keyspaceData.keyspace))
|
||||
dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateSchemaload(masterTablets, "", keyspaceData.keyspace))
|
||||
|
||||
// Append Master and Replica Tablets
|
||||
if keyspaceData.shards < 2 {
|
||||
tabAlias = tabAlias + 100
|
||||
dockerFile = applyTabletPatches(dockerFile, tabAlias, shard, keyspaceData)
|
||||
dockerComposeFile = applyTabletPatches(dockerComposeFile, tabAlias, shard, keyspaceData)
|
||||
} else {
|
||||
// Determine shard range
|
||||
for i:=0; i < keyspaceData.shards; i++ {
|
||||
|
|
@ -319,37 +319,37 @@ func applyKeyspaceDependentPatches(dockerFile []byte, keyspaceData keyspaceInfo)
|
|||
shard = fmt.Sprintf("%x-%x", interval*(i) ,interval*(i+1))
|
||||
}
|
||||
tabAlias = tabAlias + 100
|
||||
dockerFile = applyTabletPatches(dockerFile, tabAlias, shard, keyspaceData)
|
||||
dockerComposeFile = applyTabletPatches(dockerComposeFile, tabAlias, shard, keyspaceData)
|
||||
}
|
||||
}
|
||||
|
||||
tabletsUsed += len(masterTablets)
|
||||
return dockerFile
|
||||
return dockerComposeFile
|
||||
}
|
||||
|
||||
func applyDefaultDockerPatches(dockerFile []byte) []byte {
|
||||
dockerFile = applyInMemoryPatch(dockerFile, generateVtctld())
|
||||
dockerFile = applyInMemoryPatch(dockerFile, generateVtgate())
|
||||
dockerFile = applyInMemoryPatch(dockerFile, generateVtwork())
|
||||
return dockerFile
|
||||
func applyDefaultDockerPatches(dockerComposeFile []byte) []byte {
|
||||
dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtctld())
|
||||
dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtgate())
|
||||
dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtwork())
|
||||
return dockerComposeFile
|
||||
}
|
||||
|
||||
func applyDockerComposePatches(dockerFile []byte, keyspaceInfoMap map[string]keyspaceInfo) []byte {
|
||||
func applyDockerComposePatches(dockerComposeFile []byte, keyspaceInfoMap map[string]keyspaceInfo) []byte {
|
||||
//Vtctld, vtgate, vtwork, schemaload patches
|
||||
dockerFile = applyDefaultDockerPatches(dockerFile)
|
||||
dockerComposeFile = applyDefaultDockerPatches(dockerComposeFile)
|
||||
for _, keyspaceData := range keyspaceInfoMap {
|
||||
dockerFile = applyKeyspaceDependentPatches(dockerFile, keyspaceData)
|
||||
dockerComposeFile = applyKeyspaceDependentPatches(dockerComposeFile, keyspaceData)
|
||||
}
|
||||
|
||||
return dockerFile
|
||||
return dockerComposeFile
|
||||
}
|
||||
|
||||
func applyTabletPatches(dockerFile []byte, tabAlias int, shard string, keyspaceData keyspaceInfo) []byte {
|
||||
dockerFile = applyInMemoryPatch(dockerFile, generateDefaultTablet(strconv.Itoa(tabAlias+1), shard, "master", keyspaceData.keyspace))
|
||||
func applyTabletPatches(dockerComposeFile []byte, tabAlias int, shard string, keyspaceData keyspaceInfo) []byte {
|
||||
dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateDefaultTablet(strconv.Itoa(tabAlias+1), shard, "master", keyspaceData.keyspace))
|
||||
for i:=0; i < keyspaceData.replicaTablets; i++ {
|
||||
dockerFile = applyInMemoryPatch(dockerFile, generateDefaultTablet(strconv.Itoa(tabAlias+ 2 + i), shard, "replica", keyspaceData.keyspace))
|
||||
dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateDefaultTablet(strconv.Itoa(tabAlias+ 2 + i), shard, "replica", keyspaceData.keyspace))
|
||||
}
|
||||
return dockerFile
|
||||
return dockerComposeFile
|
||||
}
|
||||
|
||||
// Default Tablet
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче