Code re-org to make shellout execution logic more generic
Split exec.go into 2 files: ddlstatement.go (code related specifically to DDL execution) and shellout.go (code related specifically to shelling out to external processes). Added new ShellOut struct accordingly. Extract connect-options parse logic into its own helper function, which is also now called from RealConnectOptions to fix a prior bug. connect-options no longer allows bareword booleans, since drivers don't support this anyway. Added several new unit tests relating to external command execution, as well as connect-options parsing. Move several option-related helper methods from various files to new file config.go. Now that the number of source files is growing, renamed all subcommand files to have "cmd_" prefix. At some future point they may be moved to a separate package, but for now this provides easier visual identification of which files define subcommands.
This commit is contained in:
Evan Elias
Родитель
614bf85f76
Коммит
be1c9f8070
|
|
@ -0,0 +1,212 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
log "github.com/Sirupsen/logrus"
|
||||
"github.com/skeema/mycli"
|
||||
"golang.org/x/crypto/ssh/terminal"
|
||||
)
|
||||
|
||||
// This file contains misc functions relating to configuration or option
|
||||
// handling.
|
||||
|
||||
// AddGlobalConfigFiles takes the mycli.Config generated from the CLI and adds
|
||||
// global option files as sources. It also handles special processing for a few
|
||||
// options. Generally, subcommand handlers should call AddGlobalConfigFiles at
|
||||
// the top of the method.
|
||||
func AddGlobalConfigFiles(cfg *mycli.Config) {
|
||||
globalFilePaths := []string{"/etc/skeema", "/usr/local/etc/skeema"}
|
||||
home := filepath.Clean(os.Getenv("HOME"))
|
||||
if home != "" {
|
||||
globalFilePaths = append(globalFilePaths, path.Join(home, ".my.cnf"), path.Join(home, ".skeema"))
|
||||
}
|
||||
for _, path := range globalFilePaths {
|
||||
f := mycli.NewFile(path)
|
||||
if !f.Exists() {
|
||||
continue
|
||||
}
|
||||
if err := f.Read(); err != nil {
|
||||
log.Warnf("Ignoring global option file %s due to read error: %s", f.Path(), err)
|
||||
continue
|
||||
}
|
||||
if strings.HasSuffix(path, ".my.cnf") {
|
||||
f.IgnoreUnknownOptions = true
|
||||
}
|
||||
if err := f.Parse(cfg); err != nil {
|
||||
log.Warnf("Ignoring global option file %s due to parse error: %s", f.Path(), err)
|
||||
continue
|
||||
}
|
||||
if strings.HasSuffix(path, ".my.cnf") {
|
||||
_ = f.UseSection("skeema", "client") // safe to ignore error (doesn't matter if section doesn't exist)
|
||||
} else {
|
||||
_ = f.UseSection(cfg.Get("environment")) // safe to ignore error (doesn't matter if section doesn't exist)
|
||||
}
|
||||
|
||||
cfg.AddSource(f)
|
||||
}
|
||||
|
||||
// The host and schema options are special -- most commands only expect
|
||||
// to find them when recursively crawling directory configs. So if these
|
||||
// options have been set globally (via CLI or a global config file), and
|
||||
// the current subcommand hasn't explicitly overridden these options (as
|
||||
// init and add-environment do), silently ignore the value.
|
||||
for _, name := range []string{"host", "schema"} {
|
||||
if cfg.Changed(name) && cfg.FindOption(name) == CommandSuite.Options()[name] {
|
||||
cfg.CLI.OptionValues[name] = ""
|
||||
cfg.MarkDirty()
|
||||
}
|
||||
}
|
||||
|
||||
// Special handling for password option: supplying it with no value prompts on STDIN
|
||||
if cfg.Get("password") == "" {
|
||||
var err error
|
||||
cfg.CLI.OptionValues["password"], err = PromptPassword()
|
||||
if err != nil {
|
||||
Exit(NewExitValue(CodeNoInput, err.Error()))
|
||||
}
|
||||
cfg.MarkDirty()
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
if cfg.GetBool("debug") {
|
||||
log.SetLevel(log.DebugLevel)
|
||||
}
|
||||
}
|
||||
|
||||
// PromptPassword reads a password from STDIN without echoing the typed
|
||||
// characters. Requires that STDIN is a TTY.
|
||||
func PromptPassword() (string, error) {
|
||||
stdin := int(syscall.Stdin)
|
||||
if !terminal.IsTerminal(stdin) {
|
||||
return "", errors.New("STDIN must be a TTY to read password")
|
||||
}
|
||||
fmt.Printf("Enter password: ")
|
||||
bytePassword, err := terminal.ReadPassword(stdin)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(bytePassword), nil
|
||||
}
|
||||
|
||||
// SplitConnectOptions takes a string containing a comma-separated list of
|
||||
// connection options (typically obtained from the "connect-options" option)
|
||||
// and splits them into a map of individual key: value strings. This function
|
||||
// understands single-quoted values may contain commas, and will properly
|
||||
// treat them not as delimiters. Single-quoted values may also include escaped
|
||||
// single quotes, and values in general may contain escaped commas; these are
|
||||
// all also treated properly.
|
||||
func SplitConnectOptions(connectOpts string) (map[string]string, error) {
|
||||
result := make(map[string]string)
|
||||
if len(connectOpts) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
if connectOpts[len(connectOpts)-1] == '\\' {
|
||||
return result, fmt.Errorf("Trailing backslash in connect-options \"%s\"", connectOpts)
|
||||
}
|
||||
|
||||
var startToken int
|
||||
var name string
|
||||
var inQuote, escapeNext bool
|
||||
for n, c := range connectOpts + "," {
|
||||
if escapeNext {
|
||||
escapeNext = false
|
||||
continue
|
||||
}
|
||||
if inQuote && c != '\'' && c != '\\' {
|
||||
continue
|
||||
}
|
||||
switch c {
|
||||
case '\'':
|
||||
if name == "" {
|
||||
return result, fmt.Errorf("Invalid quote character in option name at byte offset %d in connect-options \"%s\"", n, connectOpts)
|
||||
}
|
||||
inQuote = !inQuote
|
||||
case '\\':
|
||||
escapeNext = true
|
||||
case '=':
|
||||
if name == "" {
|
||||
name = connectOpts[startToken:n]
|
||||
startToken = n + 1
|
||||
} else {
|
||||
return result, fmt.Errorf("Invalid equals-sign character in option value at byte offset %d in connect-options \"%s\"", n, connectOpts)
|
||||
}
|
||||
case ',':
|
||||
if startToken == n { // comma directly after equals sign, comma, or start of string
|
||||
return result, fmt.Errorf("Invalid comma placement in option value at byte offset %d in connect-options \"%s\"", n, connectOpts)
|
||||
}
|
||||
if name == "" {
|
||||
return result, fmt.Errorf("Option %s is missing a value at byte offset %d in connect-options \"%s\"", connectOpts[startToken:n], n, connectOpts)
|
||||
}
|
||||
if _, already := result[name]; already {
|
||||
// Disallow this since it's inherently ordering-dependent, and would
|
||||
// further complicate RealConnectOptions logic
|
||||
return result, fmt.Errorf("Option %s is set multiple times in connect-options \"%s\"", name, connectOpts)
|
||||
}
|
||||
result[name] = connectOpts[startToken:n]
|
||||
name = ""
|
||||
startToken = n + 1
|
||||
}
|
||||
}
|
||||
|
||||
if inQuote {
|
||||
return result, fmt.Errorf("Unterminated quote in connect-options \"%s\"", connectOpts)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// RealConnectOptions takes a comma-separated string of connection options,
|
||||
// strips any Go driver-specific ones, and then returns the new string which
|
||||
// is now suitable for passing to an external tool.
|
||||
func RealConnectOptions(connectOpts string) (string, error) {
|
||||
// list of lowercased versions of all go-sql-driver/mysql special params
|
||||
ignored := map[string]bool{
|
||||
"allowallfiles": true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
|
||||
"allowcleartextpasswords": true,
|
||||
"allownativepasswords": true,
|
||||
"allowoldpasswords": true,
|
||||
"charset": true,
|
||||
"clientfoundrows": true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
|
||||
"collation": true,
|
||||
"columnswithalias": true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
|
||||
"interpolateparams": true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
|
||||
"loc": true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
|
||||
"maxallowedpacket": true,
|
||||
"multistatements": true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
|
||||
"parsetime": true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
|
||||
"readtimeout": true,
|
||||
"strict": true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
|
||||
"timeout": true,
|
||||
"tls": true,
|
||||
"writetimeout": true,
|
||||
}
|
||||
|
||||
options, err := SplitConnectOptions(connectOpts)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Iterate through the returned map, and remove any driver-specific options.
|
||||
// This is done via regular expressions substitution in order to keep the
|
||||
// string in its original order.
|
||||
for name, value := range options {
|
||||
if ignored[strings.ToLower(name)] {
|
||||
re, err := regexp.Compile(fmt.Sprintf(`%s=%s(,|$)`, regexp.QuoteMeta(name), regexp.QuoteMeta(value)))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
connectOpts = re.ReplaceAllString(connectOpts, "")
|
||||
}
|
||||
}
|
||||
if len(connectOpts) > 0 && connectOpts[len(connectOpts)-1] == ',' {
|
||||
connectOpts = connectOpts[0 : len(connectOpts)-1]
|
||||
}
|
||||
return connectOpts, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSplitConnectOptions(t *testing.T) {
|
||||
assertConnectOpts := func(connectOptions string, expectedPair ...string) {
|
||||
result, err := SplitConnectOptions(connectOptions)
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error from SplitConnectOptions(\"%s\"): %s", connectOptions, err)
|
||||
}
|
||||
expected := make(map[string]string, len(expectedPair))
|
||||
for _, pair := range expectedPair {
|
||||
tokens := strings.SplitN(pair, "=", 2)
|
||||
expected[tokens[0]] = tokens[1]
|
||||
}
|
||||
if !reflect.DeepEqual(expected, result) {
|
||||
t.Errorf("Expected SplitConnectOptions(\"%s\") to return %v, instead received %v", connectOptions, expected, result)
|
||||
}
|
||||
}
|
||||
assertConnectOpts("")
|
||||
assertConnectOpts("foo='bar'", "foo='bar'")
|
||||
assertConnectOpts("bool=true,quotes='yes,no'", "bool=true", "quotes='yes,no'")
|
||||
assertConnectOpts(`escaped=we\'re ok`, `escaped=we\'re ok`)
|
||||
assertConnectOpts(`escquotes='we\'re still quoted',this=that`, `escquotes='we\'re still quoted'`, "this=that")
|
||||
|
||||
expectError := []string{
|
||||
"foo=bar,'bip'=bap",
|
||||
"flip=flap=flarb",
|
||||
"foo=,yes=no",
|
||||
"too_many_commas=1,,between_these='yeah'",
|
||||
"one=true,two=false,",
|
||||
",bad=true",
|
||||
",",
|
||||
"unterminated='yep",
|
||||
"trailingBackSlash=true\\",
|
||||
"bareword",
|
||||
"start=1,bareword",
|
||||
}
|
||||
for _, connOpts := range expectError {
|
||||
if _, err := SplitConnectOptions(connOpts); err == nil {
|
||||
t.Errorf("Did not get expected error from SplitConnectOptions(\"%s\")", connOpts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRealConnectOptions(t *testing.T) {
|
||||
assertResult := func(input, expected string) {
|
||||
actual, err := RealConnectOptions(input)
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error result from RealConnectOptions(\"%s\"): %s", input, err)
|
||||
} else if actual != expected {
|
||||
t.Errorf("Expected RealConnectOptions(\"%s\") to return \"%s\", instead found \"%s\"", input, expected, actual)
|
||||
}
|
||||
}
|
||||
assertResult("", "")
|
||||
assertResult("foo=1", "foo=1")
|
||||
assertResult("allowAllFiles=true", "")
|
||||
assertResult("foo='ok,cool',multiStatements=true", "foo='ok,cool'")
|
||||
assertResult("timeout=1s,bar=123", "bar=123")
|
||||
assertResult("strict=1,foo=2,charset='utf8mb4,utf8'", "foo=2")
|
||||
assertResult("timeout=10ms,TIMEOUT=20ms,timeOut=30ms", "")
|
||||
}
|
||||
|
|
@ -1,16 +1,12 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
log "github.com/Sirupsen/logrus"
|
||||
"github.com/skeema/mycli"
|
||||
"github.com/skeema/tengo"
|
||||
)
|
||||
|
||||
|
|
@ -25,8 +21,9 @@ type DDLStatement struct {
|
|||
// command)
|
||||
Err error
|
||||
|
||||
stmt string
|
||||
isExec bool
|
||||
stmt string
|
||||
shellOut *ShellOut
|
||||
|
||||
instance *tengo.Instance
|
||||
schemaName string
|
||||
}
|
||||
|
|
@ -137,14 +134,20 @@ func NewDDLStatement(diff tengo.TableDiff, mods tengo.StatementModifiers, target
|
|||
ddl.setErr(fmt.Errorf("TableDiff type %T not yet supported", diff))
|
||||
}
|
||||
|
||||
ddl.isExec = true
|
||||
ddl.stmt, err = InterpolateExec(wrapper, target.Dir, extras)
|
||||
ddl.shellOut, err = NewInterpolatedShellOut(wrapper, target.Dir, extras)
|
||||
ddl.setErr(err)
|
||||
}
|
||||
|
||||
return ddl
|
||||
}
|
||||
|
||||
// IsShellOut returns true if the DDL is to be executed via shelling out to an
|
||||
// external binary, or false if the DDL represents SQL to be executed directly
|
||||
// via a standard database connection.
|
||||
func (ddl *DDLStatement) IsShellOut() bool {
|
||||
return (ddl.shellOut != nil)
|
||||
}
|
||||
|
||||
// String returns a string representation of ddl. If an external command is in
|
||||
// use, the returned string will be prefixed with "\!", the MySQL CLI command
|
||||
// shortcut for "system" shellout. If ddl.Err is non-nil, the returned string
|
||||
|
|
@ -154,8 +157,8 @@ func (ddl *DDLStatement) String() string {
|
|||
return ""
|
||||
}
|
||||
var stmt string
|
||||
if ddl.isExec {
|
||||
stmt = fmt.Sprintf("\\! %s", ddl.stmt)
|
||||
if ddl.IsShellOut() {
|
||||
stmt = fmt.Sprintf("\\! %s", ddl.shellOut)
|
||||
} else {
|
||||
stmt = fmt.Sprintf("%s;", ddl.stmt)
|
||||
}
|
||||
|
|
@ -174,13 +177,12 @@ func (ddl *DDLStatement) Execute() error {
|
|||
} else if ddl.Err != nil {
|
||||
return ddl.Err
|
||||
}
|
||||
if ddl.isExec {
|
||||
cmd := exec.Command("/bin/sh", "-c", ddl.stmt)
|
||||
cmd.Stdin = os.Stdin
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
ddl.Err = cmd.Run()
|
||||
if ddl.IsShellOut() {
|
||||
ddl.Err = ddl.shellOut.Run()
|
||||
} else {
|
||||
if ddl.stmt == "" {
|
||||
return errors.New("Attempted to execute empty DDL statement")
|
||||
}
|
||||
if db, err := ddl.instance.Connect(ddl.schemaName, ""); err != nil {
|
||||
ddl.Err = err
|
||||
} else {
|
||||
|
|
@ -210,119 +212,3 @@ func (ddl *DDLStatement) getTableSize(target Target, table *tengo.Table) (int64,
|
|||
}
|
||||
return target.Instance.TableSize(target.SchemaFromInstance, table)
|
||||
}
|
||||
|
||||
// InterpolateExec takes a shell command-line containing variables of format
|
||||
// {VARNAME}, and performs substitution on them based on the supplied directory
|
||||
// and its configuration, as well as any additional values provided in the
|
||||
// extra map.
|
||||
//
|
||||
// The following variables are supplied as-is from the dir's configuration,
|
||||
// UNLESS the variable value itself contains backticks, in which case it is
|
||||
// not available in this context:
|
||||
// {USER}, {PASSWORD}, {SCHEMA}, {HOST}, {PORT}
|
||||
//
|
||||
// The following variables supply the *base name* (relative name) of whichever
|
||||
// directory had a .skeema file defining the variable:
|
||||
// {HOSTDIR}, {SCHEMADIR}
|
||||
// For example, if dir is /opt/schemas/myhost/someschema, usually the host will
|
||||
// be defined in /opt/schemas/myhost/.skeema (so HOSTDIR="myhost") and the
|
||||
// schema defined in /opt/schemas/myhost/someschema/.skeema (so
|
||||
// SCHEMADIR="someschema"). These variables are typically useful for passing to
|
||||
// service discovery.
|
||||
//
|
||||
// Vars are case-insensitive, but all-caps is recommended for visual reasons.
|
||||
// If any unknown variable is contained in the command string, a non-nil error
|
||||
// will be returned and the unknown variable will not be interpolated.
|
||||
func InterpolateExec(command string, dir *Dir, extra map[string]string) (string, error) {
|
||||
var err error
|
||||
re := regexp.MustCompile(`{([^}]*)}`)
|
||||
values := make(map[string]string, 7+len(extra))
|
||||
|
||||
asis := []string{"user", "password", "schema", "host", "port"}
|
||||
for _, name := range asis {
|
||||
value := dir.Config.Get(strings.ToLower(name))
|
||||
// any value containing shell exec will itself need be run thru
|
||||
// InterpolateExec at some point, so not available for interpolation
|
||||
if !strings.ContainsRune(value, '`') {
|
||||
values[strings.ToUpper(name)] = value
|
||||
}
|
||||
}
|
||||
|
||||
hostSource := dir.Config.Source("host")
|
||||
if file, ok := hostSource.(*mycli.File); ok {
|
||||
values["HOSTDIR"] = path.Base(file.Dir)
|
||||
}
|
||||
schemaSource := dir.Config.Source("schema")
|
||||
if file, ok := schemaSource.(*mycli.File); ok {
|
||||
values["SCHEMADIR"] = path.Base(file.Dir)
|
||||
}
|
||||
values["DIRNAME"] = path.Base(dir.Path)
|
||||
values["DIRPARENT"] = path.Base(path.Dir(dir.Path))
|
||||
values["DIRPATH"] = dir.Path
|
||||
values["CONNOPTS"] = realConnOptions(dir.Config.Get("connect-options"))
|
||||
|
||||
// Add in extras *after*, to allow them to override if desired
|
||||
for name, val := range extra {
|
||||
values[strings.ToUpper(name)] = val
|
||||
}
|
||||
|
||||
replacer := func(input string) string {
|
||||
input = strings.ToUpper(input[1 : len(input)-1])
|
||||
if value, ok := values[input]; ok {
|
||||
return escapeExecValue(value)
|
||||
}
|
||||
err = fmt.Errorf("Unknown variable {%s}", input)
|
||||
return fmt.Sprintf("{%s}", input)
|
||||
}
|
||||
|
||||
result := re.ReplaceAllStringFunc(command, replacer)
|
||||
return result, err
|
||||
}
|
||||
|
||||
var noQuotesNeeded = regexp.MustCompile(`^[\w/@%=:.,+-]*$`)
|
||||
|
||||
func escapeExecValue(value string) string {
|
||||
if noQuotesNeeded.MatchString(value) {
|
||||
return value
|
||||
}
|
||||
return fmt.Sprintf("'%s'", strings.Replace(value, "'", `'"'"'`, -1))
|
||||
}
|
||||
|
||||
// realConnOptions takes a comma-separated string of connection options, strips
|
||||
// any Go driver-specific ones, and then returns the new string which is now
|
||||
// suitable for passing to an external tool.
|
||||
func realConnOptions(origValue string) string {
|
||||
// list of lowercased versions of all go-sql-driver/mysql special params
|
||||
ignored := map[string]bool{
|
||||
"allowallfiles": true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
|
||||
"allowcleartextpasswords": true,
|
||||
"allownativepasswords": true,
|
||||
"allowoldpasswords": true,
|
||||
"charset": true,
|
||||
"collation": true,
|
||||
"clientfoundrows": true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
|
||||
"columnswithalias": true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
|
||||
"interpolateparams": true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
|
||||
"loc": true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
|
||||
"maxallowedpacket": true,
|
||||
"multistatements": true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
|
||||
"parsetime": true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
|
||||
"readtimeout": true,
|
||||
"strict": true, // banned in Dir.InstanceDefaultParams, listed here for sake of completeness
|
||||
"timeout": true,
|
||||
"tls": true,
|
||||
"writetimeout": true,
|
||||
}
|
||||
|
||||
var keep []string
|
||||
for _, keyAndValue := range strings.Split(origValue, ",") {
|
||||
if keyAndValue == "" {
|
||||
continue
|
||||
}
|
||||
tokens := strings.SplitN(keyAndValue, "=", 2)
|
||||
if !ignored[strings.ToLower(tokens[0])] {
|
||||
keep = append(keep, keyAndValue)
|
||||
}
|
||||
}
|
||||
return strings.Join(keep, ",")
|
||||
}
|
||||
66
dir.go
66
dir.go
|
|
@ -208,63 +208,17 @@ func (dir *Dir) InstanceDefaultParams() (string, error) {
|
|||
"foreign_key_checks": true, // always disabled explicitly later in this method
|
||||
}
|
||||
|
||||
v := url.Values{}
|
||||
|
||||
// Parse connect-options rune-by-rune. Split on commas, but NOT commas inside
|
||||
// of a single-quoted string.
|
||||
connectOpts := dir.Config.Get("connect-options")
|
||||
var startToken int
|
||||
var name string
|
||||
var inQuote, escapeNext bool
|
||||
if len(connectOpts) > 0 {
|
||||
if connectOpts[len(connectOpts)-1] == '\\' {
|
||||
return "", fmt.Errorf("Trailing backslash in connect-options \"%s\"", connectOpts)
|
||||
}
|
||||
for n, c := range connectOpts + "," {
|
||||
if escapeNext {
|
||||
escapeNext = false
|
||||
continue
|
||||
}
|
||||
if inQuote && c != '\'' && c != '\\' {
|
||||
continue
|
||||
}
|
||||
switch c {
|
||||
case '\'':
|
||||
if name == "" {
|
||||
return "", fmt.Errorf("Invalid quote character in option name at byte offset %d in connect-options \"%s\"", n, connectOpts)
|
||||
}
|
||||
inQuote = !inQuote
|
||||
case '\\':
|
||||
escapeNext = true
|
||||
case '=':
|
||||
if name == "" {
|
||||
name = connectOpts[startToken:n]
|
||||
startToken = n + 1
|
||||
} else {
|
||||
return "", fmt.Errorf("Invalid equals-sign character in option value at byte offset %d in connect-options \"%s\"", n, connectOpts)
|
||||
}
|
||||
case ',':
|
||||
var value string
|
||||
if startToken == n { // comma directly after equals sign, comma, or start of string
|
||||
return "", fmt.Errorf("Invalid comma placement in option value at byte offset %d in connect-options \"%s\"", n, connectOpts)
|
||||
}
|
||||
if name == "" {
|
||||
name = connectOpts[startToken:n]
|
||||
value = "1"
|
||||
} else {
|
||||
value = connectOpts[startToken:n]
|
||||
}
|
||||
if banned[strings.ToLower(name)] {
|
||||
return "", fmt.Errorf("connect-options is not allowed to contain %s", name)
|
||||
}
|
||||
v.Set(name, value)
|
||||
name = ""
|
||||
startToken = n + 1
|
||||
}
|
||||
}
|
||||
options, err := SplitConnectOptions(dir.Config.Get("connect-options"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if inQuote {
|
||||
return "", fmt.Errorf("Unterminated quote in connect-options \"%s\"", connectOpts)
|
||||
|
||||
v := url.Values{}
|
||||
for name, value := range options {
|
||||
if banned[strings.ToLower(name)] {
|
||||
return "", fmt.Errorf("connect-options is not allowed to contain %s", name)
|
||||
}
|
||||
v.Set(name, value)
|
||||
}
|
||||
|
||||
v.Set("interpolateParams", "true")
|
||||
|
|
|
|||
27
dir_test.go
27
dir_test.go
|
|
@ -56,35 +56,20 @@ func TestInstanceDefaultParams(t *testing.T) {
|
|||
"bool=true,quotes='yes,no'": "interpolateParams=true&foreign_key_checks=0&bool=true"es=%27yes,no%27",
|
||||
`escaped=we\'re ok`: "interpolateParams=true&foreign_key_checks=0&escaped=we%5C%27re ok",
|
||||
`escquotes='we\'re still quoted',this=that`: "interpolateParams=true&foreign_key_checks=0&escquotes=%27we%5C%27re still quoted%27&this=that",
|
||||
"bareword": "interpolateParams=true&foreign_key_checks=0&bareword=1",
|
||||
"start=1,bareword": "interpolateParams=true&foreign_key_checks=0&start=1&bareword=1",
|
||||
"bareword,end='yes'": "interpolateParams=true&foreign_key_checks=0&bareword=1&end=%27yes%27",
|
||||
}
|
||||
for connOpts, expected := range expectParams {
|
||||
assertDefaultParams(connOpts, expected)
|
||||
}
|
||||
|
||||
assertDefaultParamsErr := func(connectOptions string) {
|
||||
dir := getDir(connectOptions)
|
||||
_, err := dir.InstanceDefaultParams()
|
||||
if err == nil {
|
||||
t.Errorf("Did not get expected error from connect-options=\"%s\"", connectOptions)
|
||||
}
|
||||
}
|
||||
expectError := []string{
|
||||
"foo=bar,'bip'=bap",
|
||||
"flip=flap=flarb",
|
||||
"foo=,yes=no",
|
||||
"too_many_commas=1,,between_these='yeah'",
|
||||
"one=true,two=false,",
|
||||
",bad=true",
|
||||
",",
|
||||
"totally_benign=1,allowAllFiles=true",
|
||||
"FOREIGN_key_CHECKS",
|
||||
"unterminated='yep",
|
||||
"trailingBackSlash=false\\",
|
||||
"FOREIGN_key_CHECKS='on'",
|
||||
"bad_parse",
|
||||
}
|
||||
for _, connOpts := range expectError {
|
||||
assertDefaultParamsErr(connOpts)
|
||||
dir := getDir(connOpts)
|
||||
if _, err := dir.InstanceDefaultParams(); err == nil {
|
||||
t.Errorf("Did not get expected error from connect-options=\"%s\"", connOpts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,134 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/skeema/mycli"
|
||||
)
|
||||
|
||||
// varPlaceholder is a regexp for detecting placeholders in format "{VARNAME}"
|
||||
var varPlaceholder = regexp.MustCompile(`{([^}]*)}`)
|
||||
|
||||
// noQuotesNeeded is a regexp for detecting which variable values do not require
|
||||
// escaping and quote-wrapping
|
||||
var noQuotesNeeded = regexp.MustCompile(`^[\w/@%=:.,+-]*$`)
|
||||
|
||||
// ShellOut represents a command-line for an external command, executed via sh -c
|
||||
type ShellOut struct {
|
||||
Command string
|
||||
}
|
||||
|
||||
func (s *ShellOut) String() string {
|
||||
return s.Command
|
||||
}
|
||||
|
||||
// Run shells out to the external command and blocks until it completes. It
|
||||
// returns an error if one occurred. STDIN, STDOUT, and STDERR will be
|
||||
// redirected to those of the parent process.
|
||||
func (s *ShellOut) Run() error {
|
||||
if s.Command == "" {
|
||||
return errors.New("Attempted to shell out to an empty command string")
|
||||
}
|
||||
cmd := exec.Command("/bin/sh", "-c", s.Command)
|
||||
cmd.Stdin = os.Stdin
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
// NewShellOut takes a shell command-line string and returns a ShellOut, without
|
||||
// performing any variable interpolation.
|
||||
func NewShellOut(command string) *ShellOut {
|
||||
return &ShellOut{
|
||||
Command: command,
|
||||
}
|
||||
}
|
||||
|
||||
// NewInterpolatedShellOut takes a shell command-line containing variables of
|
||||
// format {VARNAME}, and performs substitution on them based on the supplied
|
||||
// directory and its configuration, as well as any additional values provided
|
||||
// in the extra map.
|
||||
//
|
||||
// The following variables are supplied as-is from the dir's configuration,
|
||||
// UNLESS the variable value itself contains backticks, in which case it is
|
||||
// not available in this context:
|
||||
// {USER}, {PASSWORD}, {SCHEMA}, {HOST}, {PORT}
|
||||
//
|
||||
// The following variables supply the *base name* (relative name) of whichever
|
||||
// directory had a .skeema file defining the variable:
|
||||
// {HOSTDIR}, {SCHEMADIR}
|
||||
// For example, if dir is /opt/schemas/myhost/someschema, usually the host will
|
||||
// be defined in /opt/schemas/myhost/.skeema (so HOSTDIR="myhost") and the
|
||||
// schema defined in /opt/schemas/myhost/someschema/.skeema (so
|
||||
// SCHEMADIR="someschema"). These variables are typically useful for passing to
|
||||
// service discovery.
|
||||
//
|
||||
// Vars are case-insensitive, but all-caps is recommended for visual reasons.
|
||||
// If any unknown variable is contained in the command string, a non-nil error
|
||||
// will be returned and the unknown variable will not be interpolated.
|
||||
func NewInterpolatedShellOut(command string, dir *Dir, extra map[string]string) (*ShellOut, error) {
|
||||
var err error
|
||||
values := make(map[string]string, 7+len(extra))
|
||||
|
||||
asis := []string{"user", "password", "schema", "host", "port"}
|
||||
for _, name := range asis {
|
||||
value := dir.Config.Get(strings.ToLower(name))
|
||||
// any value containing shell exec will itself need be run thru
|
||||
// NewInterpolatedShellOut at some point, so not available for interpolation
|
||||
// here, to avoid recursive shellouts. They can still be supplied via the
|
||||
// extra map instead; that's handled later.
|
||||
if !strings.ContainsRune(value, '`') {
|
||||
values[strings.ToUpper(name)] = value
|
||||
}
|
||||
}
|
||||
|
||||
hostSource := dir.Config.Source("host")
|
||||
if file, ok := hostSource.(*mycli.File); ok {
|
||||
values["HOSTDIR"] = path.Base(file.Dir)
|
||||
}
|
||||
schemaSource := dir.Config.Source("schema")
|
||||
if file, ok := schemaSource.(*mycli.File); ok {
|
||||
values["SCHEMADIR"] = path.Base(file.Dir)
|
||||
}
|
||||
values["DIRNAME"] = path.Base(dir.Path)
|
||||
values["DIRPARENT"] = path.Base(path.Dir(dir.Path))
|
||||
values["DIRPATH"] = dir.Path
|
||||
|
||||
if values["CONNOPTS"], err = RealConnectOptions(dir.Config.Get("connect-options")); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Add in extras *after*, to allow them to override previous vars if desired
|
||||
for name, val := range extra {
|
||||
values[strings.ToUpper(name)] = val
|
||||
}
|
||||
|
||||
replacer := func(input string) string {
|
||||
input = strings.ToUpper(input[1 : len(input)-1])
|
||||
if value, ok := values[input]; ok {
|
||||
return escapeVarValue(value)
|
||||
}
|
||||
err = fmt.Errorf("Unknown variable {%s}", input)
|
||||
return fmt.Sprintf("{%s}", input)
|
||||
}
|
||||
|
||||
result := varPlaceholder.ReplaceAllStringFunc(command, replacer)
|
||||
return NewShellOut(result), err
|
||||
}
|
||||
|
||||
// escapeVarValue takes a string, and wraps it in single-quotes so that it will
|
||||
// be interpretted as a single arg in a shell-out command line. If the value
|
||||
// already contained any single-quotes, they will be escaped in a way that will
|
||||
// cause /bin/sh -c to still interpret them as part of a single arg.
|
||||
func escapeVarValue(value string) string {
|
||||
if noQuotesNeeded.MatchString(value) {
|
||||
return value
|
||||
}
|
||||
return fmt.Sprintf("'%s'", strings.Replace(value, "'", `'"'"'`, -1))
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewInterpolatedShellOut(t *testing.T) {
|
||||
getDir := func(path string, pairs ...string) *Dir {
|
||||
optValues := make(map[string]string)
|
||||
for _, pair := range pairs {
|
||||
tokens := strings.SplitN(pair, "=", 2)
|
||||
optValues[tokens[0]] = tokens[1]
|
||||
}
|
||||
return &Dir{
|
||||
Path: path,
|
||||
Config: getConfig(optValues), // see dir_test.go
|
||||
section: "production",
|
||||
}
|
||||
}
|
||||
dir := getDir("/var/schemas/somehost/someschema", "host=ahost", "schema=aschema", "user=someone", "password=", "port=3306", `connect-options=sql_mode='STRICT_ALL_TABLES,ALLOW_INVALID_DATES'`)
|
||||
assertShellOut := func(command, expected string, extraPairs ...string) {
|
||||
extra := make(map[string]string)
|
||||
for _, pair := range extraPairs {
|
||||
tokens := strings.SplitN(pair, "=", 2)
|
||||
extra[tokens[0]] = tokens[1]
|
||||
}
|
||||
s, err := NewInterpolatedShellOut(command, dir, extra)
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error from NewInterpolatedShellOut on %s: %s", command, err)
|
||||
} else if s.Command != expected {
|
||||
t.Errorf("Expected NewInterpolatedShellOut to return ShellOut.Command of %s, instead found %s", expected, s.Command)
|
||||
}
|
||||
}
|
||||
|
||||
assertShellOut("/bin/echo {HOST} {SCHEMA} {user} {PASSWORD} {DirName} {DIRPARENT} {DIRPATH}", "/bin/echo ahost aschema someone someschema somehost /var/schemas/somehost/someschema")
|
||||
assertShellOut("/bin/echo {HOST} {SOMETHING}", "/bin/echo 'overridden value' new_value", "host=overridden value", "something=new_value")
|
||||
assertShellOut("/bin/echo {connopts}", `/bin/echo 'sql_mode='"'"'STRICT_ALL_TABLES,ALLOW_INVALID_DATES'"'"''`)
|
||||
|
||||
s, err := NewInterpolatedShellOut("/bin/echo {HOST} {iNvAlId} {SCHEMA}", dir, nil)
|
||||
if err == nil {
|
||||
t.Error("Expected NewInterpolatedShellOut to return an error when invalid variable used, but it did not")
|
||||
} else if s == nil || s.Command != "/bin/echo ahost {INVALID} aschema" {
|
||||
t.Errorf("Unexpected result from NewInterpolatedShellOut when an invalid variable was present: %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEscapeVarValue(t *testing.T) {
|
||||
values := map[string]string{
|
||||
`has space`: `'has space'`,
|
||||
`has "double quote"`: `'has "double quote"'`,
|
||||
`\`: `'\'`,
|
||||
`/etc/*`: `'/etc/*'`,
|
||||
`has 'single quoted'`: `'has '"'"'single quoted'"'"''`,
|
||||
}
|
||||
for input, expected := range values {
|
||||
if actual := escapeVarValue(input); actual != expected {
|
||||
t.Errorf("Expected escapeVarValue(`%s`) to return `%s`, instead found `%s`", input, expected, actual)
|
||||
}
|
||||
}
|
||||
|
||||
fineAsIs := []string{
|
||||
"",
|
||||
"just-words",
|
||||
"this@that,1=1:no_spaces-so/we.r+ok",
|
||||
}
|
||||
for _, val := range fineAsIs {
|
||||
if actual := escapeVarValue(val); actual != val {
|
||||
t.Errorf("Expected \"%s\" to not need escaping, but escapeVarValue returned: %s", val, actual)
|
||||
}
|
||||
}
|
||||
}
|
||||
84
skeema.go
84
skeema.go
|
|
@ -1,18 +1,12 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
log "github.com/Sirupsen/logrus"
|
||||
"github.com/skeema/mycli"
|
||||
"golang.org/x/crypto/ssh/terminal"
|
||||
)
|
||||
|
||||
const version = "0.1 (pre-release)"
|
||||
|
|
@ -23,84 +17,6 @@ schema to the filesystem, and apply online schema changes by modifying files.`
|
|||
// added to it via init() functions in each subcommand's source file.
|
||||
var CommandSuite = mycli.NewCommandSuite("skeema", version, rootDesc)
|
||||
|
||||
// AddGlobalConfigFiles takes the mycli.Config generated from the CLI and adds
|
||||
// global option files as sources. It also handles special processing for a few
|
||||
// options. Generally, subcommand handlers should call AddGlobalConfigFiles at
|
||||
// the top of the method.
|
||||
func AddGlobalConfigFiles(cfg *mycli.Config) {
|
||||
globalFilePaths := []string{"/etc/skeema", "/usr/local/etc/skeema"}
|
||||
home := filepath.Clean(os.Getenv("HOME"))
|
||||
if home != "" {
|
||||
globalFilePaths = append(globalFilePaths, path.Join(home, ".my.cnf"), path.Join(home, ".skeema"))
|
||||
}
|
||||
for _, path := range globalFilePaths {
|
||||
f := mycli.NewFile(path)
|
||||
if !f.Exists() {
|
||||
continue
|
||||
}
|
||||
if err := f.Read(); err != nil {
|
||||
log.Warnf("Ignoring global option file %s due to read error: %s", f.Path(), err)
|
||||
continue
|
||||
}
|
||||
if strings.HasSuffix(path, ".my.cnf") {
|
||||
f.IgnoreUnknownOptions = true
|
||||
}
|
||||
if err := f.Parse(cfg); err != nil {
|
||||
log.Warnf("Ignoring global option file %s due to parse error: %s", f.Path(), err)
|
||||
continue
|
||||
}
|
||||
if strings.HasSuffix(path, ".my.cnf") {
|
||||
_ = f.UseSection("skeema", "client") // safe to ignore error (doesn't matter if section doesn't exist)
|
||||
} else {
|
||||
_ = f.UseSection(cfg.Get("environment")) // safe to ignore error (doesn't matter if section doesn't exist)
|
||||
}
|
||||
|
||||
cfg.AddSource(f)
|
||||
}
|
||||
|
||||
// The host and schema options are special -- most commands only expect
|
||||
// to find them when recursively crawling directory configs. So if these
|
||||
// options have been set globally (via CLI or a global config file), and
|
||||
// the current subcommand hasn't explicitly overridden these options (as
|
||||
// init and add-environment do), silently ignore the value.
|
||||
for _, name := range []string{"host", "schema"} {
|
||||
if cfg.Changed(name) && cfg.FindOption(name) == CommandSuite.Options()[name] {
|
||||
cfg.CLI.OptionValues[name] = ""
|
||||
cfg.MarkDirty()
|
||||
}
|
||||
}
|
||||
|
||||
// Special handling for password option: supplying it with no value prompts on STDIN
|
||||
if cfg.Get("password") == "" {
|
||||
var err error
|
||||
cfg.CLI.OptionValues["password"], err = PromptPassword()
|
||||
if err != nil {
|
||||
Exit(NewExitValue(CodeNoInput, err.Error()))
|
||||
}
|
||||
cfg.MarkDirty()
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
if cfg.GetBool("debug") {
|
||||
log.SetLevel(log.DebugLevel)
|
||||
}
|
||||
}
|
||||
|
||||
// PromptPassword reads a password from STDIN without echoing the typed
|
||||
// characters. Requires that STDIN is a TTY.
|
||||
func PromptPassword() (string, error) {
|
||||
stdin := int(syscall.Stdin)
|
||||
if !terminal.IsTerminal(stdin) {
|
||||
return "", errors.New("STDIN must be a TTY to read password")
|
||||
}
|
||||
fmt.Printf("Enter password: ")
|
||||
bytePassword, err := terminal.ReadPassword(stdin)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(bytePassword), nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
// Add global options. Sub-commands may override these when needed.
|
||||
CommandSuite.AddOption(mycli.StringOption("host", 0, "", "Database hostname or IP address").Hidden())
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче