Merge branch 'partitioning'
This commit is contained in:
Evan Elias
Коммит
8a4862d684
|
|
@ -51,7 +51,7 @@ Tagged releases are tested against the following databases, all running on Linux
|
|||
|
||||
Outside of a tagged release, every commit to the master branch is automatically tested against MySQL 5.6 and 5.7.
|
||||
|
||||
A few uncommon MySQL features -- such as partitioned tables and spatial types -- are not supported yet. Skeema is able to *create* or *drop* tables using these features, but not *alter* them. The output of `skeema diff` and `skeema push` clearly displays when this is the case. You may still make such alters directly/manually (outside of Skeema), and then update the corresponding CREATE TABLE files via `skeema pull`.
|
||||
A few uncommon MySQL features -- such as spatial indexes and subpartitioning -- are not supported yet. Skeema is able to *create* or *drop* tables using these features, but not *alter* them. The output of `skeema diff` and `skeema push` clearly displays when this is the case. You may still make such alters directly/manually (outside of Skeema), and then update the corresponding CREATE TABLE files via `skeema pull`.
|
||||
|
||||
## Credits
|
||||
|
||||
|
|
@ -68,6 +68,8 @@ Additional [contributions](https://github.com/skeema/skeema/graphs/contributors)
|
|||
|
||||
Support for stored procedures and functions generously sponsored by [Psyonix](https://psyonix.com).
|
||||
|
||||
Support for partitioned tables generously sponsored by [Etsy](https://www.etsy.com).
|
||||
|
||||
## License
|
||||
|
||||
**Copyright 2019 Skeema LLC**
|
||||
|
|
|
|||
|
|
@ -75,10 +75,7 @@ func applyTarget(t *Target, printer *Printer) (Result, error) {
|
|||
}
|
||||
|
||||
t.logApplyStart()
|
||||
diff := tengo.NewSchemaDiff(schemaFromInstance, t.SchemaFromDir())
|
||||
if err := VerifyDiff(diff, t); err != nil {
|
||||
return result, err
|
||||
}
|
||||
schemaFromDir := t.SchemaFromDir()
|
||||
|
||||
// Obtain StatementModifiers based on the dir's config
|
||||
mods, err := StatementModifiersForDir(t.Dir)
|
||||
|
|
@ -86,6 +83,24 @@ func applyTarget(t *Target, printer *Printer) (Result, error) {
|
|||
return result, ConfigError(err.Error())
|
||||
}
|
||||
mods.Flavor = t.Instance.Flavor()
|
||||
if mods.Partitioning == tengo.PartitioningRemove {
|
||||
// With partitioning=remove, forcibly treat all filesystem definitions as if
|
||||
// they didn't have a partitioning clause. This is designed to aid in the
|
||||
// use-case of not running any partition management in a dev environment; if
|
||||
// a table somehow manages to be partitioned there anyway by mistake, we
|
||||
// intentionally want to de-partition it.
|
||||
for _, table := range schemaFromDir.Tables {
|
||||
if table.Partitioning != nil {
|
||||
table.CreateStatement = table.UnpartitionedCreateStatement(mods.Flavor)
|
||||
table.Partitioning = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
diff := tengo.NewSchemaDiff(schemaFromInstance, schemaFromDir)
|
||||
if err := VerifyDiff(diff, t); err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Build DDLStatements for each ObjectDiff, handling pre-execution errors
|
||||
// accordingly. Also track ObjectKeys for modified objects, for subsequent
|
||||
|
|
@ -189,6 +204,16 @@ func StatementModifiersForDir(dir *fs.Dir) (mods tengo.StatementModifiers, err e
|
|||
if mods.IgnoreTable, err = dir.Config.GetRegexp("ignore-table"); err != nil {
|
||||
return
|
||||
}
|
||||
var partitioning string
|
||||
if partitioning, err = dir.Config.GetEnum("partitioning", "keep", "remove", "modify"); err != nil {
|
||||
return
|
||||
}
|
||||
partMap := map[string]tengo.PartitioningMode{
|
||||
"keep": tengo.PartitioningKeep,
|
||||
"remove": tengo.PartitioningRemove,
|
||||
"modify": tengo.PartitioningPermissive,
|
||||
}
|
||||
mods.Partitioning = partMap[partitioning]
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ func VerifyDiff(diff *tengo.SchemaDiff, t *Target) error {
|
|||
StrictIndexOrder: true, // needed since we must get the SHOW CREATE TABLEs to match
|
||||
StrictForeignKeyNaming: true, // ditto
|
||||
AllowUnsafe: true, // needed since we're just running against the temp schema
|
||||
SkipPreDropAlters: true, // needed to ignore DROP PARTITION generated just to speed up DROP TABLE
|
||||
Flavor: t.Instance.Flavor(),
|
||||
}
|
||||
if major, minor, _ := t.Instance.Version(); major > 5 || minor > 5 {
|
||||
|
|
@ -78,10 +79,27 @@ func VerifyDiff(diff *tengo.SchemaDiff, t *Target) error {
|
|||
return fmt.Errorf("Diff verification failure: %s", err.Error())
|
||||
}
|
||||
|
||||
// Compare the create statements of the "to" side of the diff with the create
|
||||
// statements from the workspace. In doing so we must ignore differences in
|
||||
// next-auto-inc value (which intentionally is often not updated) as well as
|
||||
// the entirety of the partitioning clause (since the partition list is
|
||||
// intentionally never modified).
|
||||
actualTables := wsSchema.TablesByName()
|
||||
for name, toTable := range expected {
|
||||
expectCreate, _ := tengo.ParseCreateAutoInc(toTable.CreateStatement)
|
||||
actualCreate, _ := tengo.ParseCreateAutoInc(actualTables[name].CreateStatement)
|
||||
// Simply compare partitioning *status*
|
||||
expectPartitioned := (toTable.Partitioning != nil)
|
||||
actualPartitioned := (actualTables[name].Partitioning != nil)
|
||||
if expectPartitioned != actualPartitioned {
|
||||
return fmt.Errorf("Diff verification failure on table %s\nEXPECTED PARTITIONING STATUS POST-ALTER: %t\nACTUAL PARTITIONING STATUS POST-ALTER: %t\nRun command again with --skip-verify if this discrepancy is safe to ignore", name, expectPartitioned, actualPartitioned)
|
||||
}
|
||||
expectCreate := toTable.CreateStatement
|
||||
actualCreate := actualTables[name].CreateStatement
|
||||
if expectPartitioned {
|
||||
expectCreate = toTable.UnpartitionedCreateStatement(mods.Flavor)
|
||||
actualCreate = actualTables[name].UnpartitionedCreateStatement(mods.Flavor)
|
||||
}
|
||||
expectCreate, _ = tengo.ParseCreateAutoInc(expectCreate)
|
||||
actualCreate, _ = tengo.ParseCreateAutoInc(actualCreate)
|
||||
if expectCreate != actualCreate {
|
||||
return fmt.Errorf("Diff verification failure on table %s\n\nEXPECTED POST-ALTER:\n%s\n\nACTUAL POST-ALTER:\n%s\n\nRun command again with --skip-verify if this discrepancy is safe to ignore", name, expectCreate, actualCreate)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ top of the file. If no environment name is supplied, the default is
|
|||
cmd.AddOption(mybase.StringOption("ddl-wrapper", 'X', "", "Like --alter-wrapper, but applies to all DDL types (CREATE, DROP, ALTER)"))
|
||||
cmd.AddOption(mybase.StringOption("safe-below-size", 0, "0", "Always permit destructive operations for tables below this size in bytes"))
|
||||
cmd.AddOption(mybase.StringOption("concurrent-instances", 'c', "1", "Perform operations on this number of instances concurrently"))
|
||||
cmd.AddOption(mybase.StringOption("partitioning", 0, "keep", `Specify handling of partitioning status on the database side (valid values: "keep", "remove", "modify")`))
|
||||
linter.AddCommandOptions(cmd)
|
||||
cmd.AddArg("environment", "production", false)
|
||||
CommandSuite.AddSubCommand(cmd)
|
||||
|
|
|
|||
|
|
@ -34,10 +34,11 @@ Destructive operations only occur when specifically requested via the [allow-uns
|
|||
The following operations are considered unsafe:
|
||||
|
||||
* Dropping a table
|
||||
* Altering a table to drop an existing column
|
||||
* Altering a table to change the type of an existing column in a way that potentially causes data loss, length truncation, or reduction in precision
|
||||
* Altering a table to change the character set of an existing column
|
||||
* Altering a table to drop a normal column or stored (non-virtual) generated column
|
||||
* Altering a table to modify an existing column in a way that potentially causes data loss, length truncation, or reduction in precision
|
||||
* Altering a table to modify the character set of an existing column
|
||||
* Altering a table to change its storage engine
|
||||
* Dropping a stored procedure or function (even if just to [re-create it with a modified definition](requirements.md#routines))
|
||||
|
||||
Note that `skeema diff` also has the same safety logic as `skeema push`, even though `skeema diff` never actually modifies tables. This behavior exists so that `skeema diff` can serve as a safe dry-run that exactly matches the logic for `skeema push`. If unsafe operations are not explicitly allowed, `skeema diff` will display unsafe operations as commented-out DDL.
|
||||
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ This document is a reference, describing all options supported by Skeema. To lea
|
|||
* [lint-pk](#lint-pk)
|
||||
* [my-cnf](#my-cnf)
|
||||
* [new-schemas](#new-schemas)
|
||||
* [partitioning](#partitioning)
|
||||
* [password](#password)
|
||||
* [port](#port)
|
||||
* [reuse-temp-schema](#reuse-temp-schema)
|
||||
|
|
@ -138,11 +139,11 @@ If set to the default of false, `skeema push` refuses to run any DDL on a databa
|
|||
The following operations are considered unsafe:
|
||||
|
||||
* Dropping a table
|
||||
* Altering a table to drop a column
|
||||
* Altering a table to drop a normal column or stored (non-virtual) generated column
|
||||
* Altering a table to modify an existing column in a way that potentially causes data loss, length truncation, or reduction in precision
|
||||
* Altering a table to modify the character set of an existing column
|
||||
* Altering a table to change its storage engine
|
||||
* Dropping a stored procedure or function (even if just to [re-create it with a modified definition](requirements.md#edge-cases-for-routines))
|
||||
* Dropping a stored procedure or function (even if just to [re-create it with a modified definition](requirements.md#routines))
|
||||
|
||||
If [allow-unsafe](#allow-unsafe) is set to true, these operations are fully permitted, for all tables. It is not recommended to enable this setting in an option file, especially in the production environment. It is safer to require users to supply it manually on the command-line on an as-needed basis, to serve as a confirmation step for unsafe operations.
|
||||
|
||||
|
|
@ -844,6 +845,30 @@ If true, `skeema pull` will look for schemas (databases) that exist on the insta
|
|||
|
||||
When using a workflow that involves running `skeema pull development` regularly, it may be useful to disable this option. For example, if the development environment tends to contain various extra schemas for testing purposes, set `skip-new-schemas` in a global or top-level .skeema file's `[development]` section to avoid storing these testing schemas in the filesystem.
|
||||
|
||||
### partitioning
|
||||
|
||||
Commands | diff, push
|
||||
--- | :---
|
||||
**Default** | "keep"
|
||||
**Type** | enum
|
||||
**Restrictions** | Requires one of these values: "keep", "remove", "modify"
|
||||
|
||||
Skeema v1.4.0 added diff support for partitioned tables. This option affects how DDL involving partitioned tables is generated or executed via `skeema diff` and `skeema push`.
|
||||
|
||||
With the default value of "keep", tables may be partitioned (through the filesystem `CREATE TABLE` containing a `PARTITON BY` clause, either initially or one subsequently being added), but will never be de-partitioned or re-partitioned. In other words, once a table is partitioned in a database, with `partitioning=keep` Skeema suppresses further modifications to the partitioning clause for the table.
|
||||
|
||||
With a value of "remove", tables will not be partitioned, and any already-partitioned tables will be de-partitioned. If any filesystem `CREATE TABLE` statements contain a `PARTITION BY` clause, it will effectively be ignored. Any already-partitioned tables in a database will automatically have DDL generated to de-partition them via `ALTER TABLE ... REMOVE PARTITIONING`.
|
||||
|
||||
With a value of "modify", partitioning clauses are handled permissively. Tables will be partitioned, re-partitioned, or de-partitioned based on the presence of a `PARTITION BY` clause in the filesystem `CREATE TABLE` statement.
|
||||
|
||||
Overall, the intended use of the [partitioning](#partitioning) option is as follows:
|
||||
|
||||
* If you use partitioning in production but not in development (for example), place `partitioning=remove` in a `[development]` section of a top-level .skeema file. This will ensure that tables in your development databases are never partitioned, removing the need to run partition-management scripts in dev.
|
||||
* The default of `partitioning=keep` is useful in all environments where partitioning is actually in-use; it prevents accidental re-partitioning or de-partitioning. For example, if someone runs `skeema pull development` and development is using `partitioning=remove`, this will remove the PARTITION BY clause from the filesystem `CREATE TABLE` statements; nonetheless, a subsequent `skeema push production` won't remove partitioning from any already-partitioned tables as long as `partitioning=keep` is enabled there.
|
||||
* For one-off situations where you intentionally want to re-partition or de-partition an existing partitioned table, you can use `skeema push --partitioning=modify` as a command-line override.
|
||||
|
||||
Regardless of this option, modifications to just the *partition list* of a partitioned table are always ignored for RANGE and LIST partitioning methods, and are unsupported for HASH and KEY methods. Skeema will not add or remove partitions from an already-partitioned table, regardless of differences between the filesystem `CREATE TABLE` and the table in a live database. The intended workflow is to use an external tool/cron for managing the partition list, e.g. to remove old time-based RANGE partitions and add new ones.
|
||||
|
||||
### password
|
||||
|
||||
Commands | *all*
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
## Requirements
|
||||
|
||||
This document contains important notes regarding which environments are supported by Skeema.
|
||||
|
||||
### MySQL version and flavor
|
||||
|
||||
Skeema currently supports the following databases:
|
||||
|
|
@ -10,7 +12,7 @@ Skeema currently supports the following databases:
|
|||
|
||||
Testing is performed with the database server running on Linux only. Other operating systems likely work without issue, although there is one [known incompatibility regarding case-insensitive filesystems](https://github.com/skeema/skeema/issues/65#issuecomment-478048414), e.g. when the database server is running on Windows or MacOS, if any schema names or table names use uppercase characters.
|
||||
|
||||
Some MySQL features -- such as partitioned tables and spatial types -- are [not supported yet](requirements.md#unsupported-for-alter-table) in Skeema's diff operations. Additionally, only the InnoDB storage engine is primarily supported at this time. Other storage engines are often perfectly functional in Skeema, but it depends on whether any esoteric features of the engine are used.
|
||||
Some MySQL features -- such as spatial indexes and subpartitioning -- are [not supported yet](requirements.md#unsupported-for-alter-table) in Skeema's diff operations. Additionally, only the InnoDB storage engine is primarily supported at this time. Other storage engines are often perfectly functional in Skeema, but it depends on whether any esoteric features of the engine are used.
|
||||
|
||||
In all cases, Skeema's safety mechanisms will detect when a table is using unsupported features, and will alert you to this fact in `skeema diff` or `skeema push`. There is no risk of generating or executing an incorrect diff. If Skeema does not yet support a table/column feature that you need, please [open a GitHub issue](https://github.com/skeema/skeema/issues/new) so that the work can be prioritized appropriately.
|
||||
|
||||
|
|
@ -93,12 +95,11 @@ The following object types are completely ignored by Skeema. Their presence won'
|
|||
|
||||
Skeema can CREATE or DROP tables using these features, but cannot ALTER them. The output of `skeema diff` and `skeema push` will note that it cannot generate or run ALTER TABLE for tables using these features, so the affected table(s) will be skipped, but the rest of the operation will proceed as normal.
|
||||
|
||||
* partitioned tables
|
||||
* sub-partitioning (two levels of partitioning in the same table)
|
||||
* some features of non-InnoDB storage engines
|
||||
* spatial types
|
||||
* spatial indexes
|
||||
* column-level compression, with or without predefined dictionary (Percona Server 5.6.33+)
|
||||
* CHECK constraints (MySQL 8.0.16+ / MariaDB 10.2+)
|
||||
|
||||
* CHECK constraints (MySQL 8.0.16+ / Percona Server 8.0.16+ / MariaDB 10.2+)
|
||||
|
||||
You can still ALTER these tables externally from Skeema (e.g., direct invocation of `ALTER TABLE` or `pt-online-schema-change`). Afterwards, you can update your schema repo using `skeema pull`, which will work properly even on these tables.
|
||||
|
||||
|
|
@ -114,7 +115,9 @@ Note that for empty tables as a special-case, a rename is technically equivalent
|
|||
|
||||
For tables with data, the work-around to handle renames is to run the appropriate `ALTER TABLE` manually (outside of Skeema) on all relevant databases. You can update your schema repo afterwards by running `skeema pull`.
|
||||
|
||||
#### Edge-cases for routines
|
||||
### Implementation notes and special cases
|
||||
|
||||
#### Routines
|
||||
|
||||
Skeema v1.2.0 added support for MySQL routines (stored procedures and functions). This support generally handles all common usage patterns, but there a few edge-cases to be aware of:
|
||||
|
||||
|
|
@ -126,3 +129,15 @@ Skeema v1.2.0 added support for MySQL routines (stored procedures and functions)
|
|||
* Skeema does not support management of [native UDFs](https://dev.mysql.com/doc/refman/8.0/en/create-function-udf.html), which are typically written in C or C++ and compiled into shared libraries.
|
||||
* MariaDB 10.3's Oracle-style routine PACKAGEs are not supported.
|
||||
|
||||
#### Partitioned tables
|
||||
|
||||
Skeema v1.4.0 added support for partitioned tables. The diff/push functionality fully supports changes to partitioning *status*: initially partitioning a previously-unpartitioned table; removing partitioning from an already-partitioned table; changing the partitioning method or expression of an already-partitioned table. The [partitioning option](options.md#partitioning) controls behavior of DDL involving these operations. With its default value of "keep", tables can be initially partitioned, but won't subsequently be de-partitioned or re-partitioned.
|
||||
|
||||
Skeema intentionally ignores changes to the *list of partitions* for an already-partitioned table using RANGE or LIST partitioning methods; the assumption is that an external partition management script/cron is responsible for handling this, outside of the scope of the schema repository. Meanwhile, for HASH or KEY partitioning methods, attempting to change the partition count causes an unsupported diff error, skipping the affected table. Future versions of Skeema may add additional options controlling these behaviors.
|
||||
|
||||
Whenever a RANGE or LIST partitioned table is being dropped, Skeema will generate a series of `ALTER TABLE ... DROP PARTITION` clauses to drop all but 1 partition prior to generating the `DROP TABLE`. This avoids having a single excessively-long `DROP TABLE` operation, which could be disruptive to other queries since it holds MySQL's dict_sys mutex.
|
||||
|
||||
Sub-partitioning (two levels of partitioning in the same table) is not supported for diff operations yet, as this feature adds complexity and is infrequently used.
|
||||
|
||||
When running `skeema pull` against an environment that uses `partitioning=remove`, please be aware that since the environment has no partitions, the *.sql files will be rewritten such that the `CREATE TABLE` statements lose their `PARTITION BY` clauses. By design this won't interfere with other environments that use `partitioning=keep`, however it is cosmetically undesirable and potentially confusing in SCM history.
|
||||
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -12,7 +12,7 @@ require (
|
|||
github.com/opencontainers/runc v1.0.0-rc5 // indirect
|
||||
github.com/sirupsen/logrus v1.4.2
|
||||
github.com/skeema/mybase v1.0.8
|
||||
github.com/skeema/tengo v0.8.20-0.20191115154500-6ca952692106
|
||||
github.com/skeema/tengo v0.9.0
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
|
||||
golang.org/x/lint v0.0.0-20190409202823-959b441ac422
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -68,8 +68,8 @@ github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4
|
|||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/skeema/mybase v1.0.8 h1:eqtxi5FfphYhosEEeHBsYf/9oXfwjCZ8fMnpGJ+AdxI=
|
||||
github.com/skeema/mybase v1.0.8/go.mod h1:09Uz3MIoXTNCUZWBeKDeO8SUHlQNjIEocXbc1DFvEKQ=
|
||||
github.com/skeema/tengo v0.8.20-0.20191115154500-6ca952692106 h1:r7T+RbE9wdIfJ5WhRPqptJhmB7N+UhiV81gxiFTKuDs=
|
||||
github.com/skeema/tengo v0.8.20-0.20191115154500-6ca952692106/go.mod h1:7ahmzzEKjeOzHEqq0okxccPFozsDyGmZORUfF25GRYc=
|
||||
github.com/skeema/tengo v0.9.0 h1:/hfT3PBD60E7p//GttldM9jeFpX1Dldxgs3umJsJTUI=
|
||||
github.com/skeema/tengo v0.9.0/go.mod h1:7ahmzzEKjeOzHEqq0okxccPFozsDyGmZORUfF25GRYc=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ schema to the filesystem, and apply online schema changes by modifying files.`
|
|||
|
||||
// Globals overridden by GoReleaser's ldflags
|
||||
var (
|
||||
version = "1.3.1-dev"
|
||||
version = "1.4.0"
|
||||
commit = "unknown"
|
||||
date = "unknown"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1404,3 +1404,87 @@ END`
|
|||
s.handleCommand(t, CodeSuccess, ".", "skeema push --temp-schema-binlog=off")
|
||||
pos = assertLogged(pos)
|
||||
}
|
||||
|
||||
func (s SkeemaIntegrationSuite) TestPartitioning(t *testing.T) {
|
||||
s.handleCommand(t, CodeSuccess, ".", "skeema init --dir mydb -h %s -P %d", s.d.Instance.Host, s.d.Instance.Port)
|
||||
|
||||
contentsNoPart := fs.ReadTestFile(t, "mydb/analytics/activity.sql")
|
||||
contents2Part := strings.Replace(contentsNoPart, ";\n",
|
||||
"\nPARTITION BY RANGE (ts)\n(PARTITION p0 VALUES LESS THAN (1571678000),\n PARTITION pN VALUES LESS THAN MAXVALUE);\n",
|
||||
1)
|
||||
contents3Part := strings.Replace(contentsNoPart, ";\n",
|
||||
"\nPARTITION BY RANGE (ts)\n(PARTITION p0 VALUES LESS THAN (1571678000),\n PARTITION p1 VALUES LESS THAN (1571679000),\n PARTITION pN VALUES LESS THAN MAXVALUE);\n",
|
||||
1)
|
||||
contents3PartPlusNewCol := strings.Replace(contents3Part, " `target_id`", " `somenewcol` int,\n `target_id`", 1)
|
||||
contentsHashPart := strings.Replace(contentsNoPart, ";\n",
|
||||
"\nPARTITION BY HASH (action_id) PARTITIONS 4;\n",
|
||||
1)
|
||||
|
||||
// Rewrite activity.sql to be partitioned by range with 2 partitions, and then
|
||||
// test diff behavior with each value of partitioning option
|
||||
fs.WriteTestFile(t, "mydb/analytics/activity.sql", contents2Part)
|
||||
s.handleCommand(t, CodeSuccess, "mydb/analytics", "skeema diff --partitioning=remove")
|
||||
s.handleCommand(t, CodeDifferencesFound, "mydb/analytics", "skeema diff --partitioning=keep")
|
||||
s.handleCommand(t, CodeDifferencesFound, "mydb/analytics", "skeema diff") // default is keep
|
||||
s.handleCommand(t, CodeDifferencesFound, "mydb/analytics", "skeema diff --partitioning=modify")
|
||||
s.handleCommand(t, CodeBadConfig, "mydb/analytics", "skeema diff --partitioning=invalid")
|
||||
|
||||
// Push to execute the ALTER to partition by range with 2 partitions.
|
||||
// Confirm no differences with keep, but some differences with remove.
|
||||
s.handleCommand(t, CodeSuccess, "mydb/analytics", "skeema push") // default is keep
|
||||
s.handleCommand(t, CodeSuccess, "mydb/analytics", "skeema diff")
|
||||
s.handleCommand(t, CodeDifferencesFound, "mydb/analytics", "skeema diff --partitioning=remove")
|
||||
|
||||
// Rewrite activity.sql to now have 3 partitions, still by range. This should
|
||||
// not show differences for keep or modify.
|
||||
fs.WriteTestFile(t, "mydb/analytics/activity.sql", contents3Part)
|
||||
s.handleCommand(t, CodeSuccess, "mydb/analytics", "skeema diff --partitioning=keep")
|
||||
s.handleCommand(t, CodeSuccess, "mydb/analytics", "skeema diff --partitioning=modify")
|
||||
// Note: didn't push the above change
|
||||
|
||||
// Rewrite activity.sql to be unpartitioned. This should not show differences
|
||||
// for keep, but should for remove or modify.
|
||||
fs.WriteTestFile(t, "mydb/analytics/activity.sql", contentsNoPart)
|
||||
s.handleCommand(t, CodeSuccess, "mydb/analytics", "skeema diff --partitioning=keep")
|
||||
s.handleCommand(t, CodeDifferencesFound, "mydb/analytics", "skeema diff --partitioning=remove")
|
||||
s.handleCommand(t, CodeDifferencesFound, "mydb/analytics", "skeema diff --partitioning=modify")
|
||||
// Note: didn't push the above change
|
||||
|
||||
// Rewrite activity.sql to have 3 partitions, still by range, as well as a new
|
||||
// column. Pushing this with remove should add the new column but remove the
|
||||
// partitioning.
|
||||
fs.WriteTestFile(t, "mydb/analytics/activity.sql", contents3PartPlusNewCol)
|
||||
s.handleCommand(t, CodeDifferencesFound, "mydb/analytics", "skeema diff --partitioning=remove")
|
||||
s.handleCommand(t, CodeSuccess, "mydb/analytics", "skeema push --partitioning=remove")
|
||||
s.handleCommand(t, CodeSuccess, "mydb/analytics", "skeema pull")
|
||||
newContents := fs.ReadTestFile(t, "mydb/analytics/activity.sql")
|
||||
if strings.Contains(newContents, "PARTITION BY") || !strings.Contains(newContents, "somenewcol") {
|
||||
t.Errorf("Previous push did not have intended effect; current table structure: %s", newContents)
|
||||
}
|
||||
|
||||
// Remove the new col and restore partitioning
|
||||
fs.WriteTestFile(t, "mydb/analytics/activity.sql", contents3Part)
|
||||
s.handleCommand(t, CodeSuccess, "mydb/analytics", "skeema push --allow-unsafe --partitioning=keep")
|
||||
s.handleCommand(t, CodeSuccess, "mydb/analytics", "skeema diff --partitioning=keep")
|
||||
|
||||
// Rewrite activity.sql to be partitioned by hash. This should be ignored with
|
||||
// keep, repartition with modify, or departition with remove. If pushed with
|
||||
// remove and then pulled, files should be back to initial state.
|
||||
fs.WriteTestFile(t, "mydb/analytics/activity.sql", contentsHashPart)
|
||||
s.handleCommand(t, CodeSuccess, "mydb/analytics", "skeema diff --partitioning=keep")
|
||||
s.handleCommand(t, CodeDifferencesFound, "mydb/analytics", "skeema diff --partitioning=modify")
|
||||
s.handleCommand(t, CodeDifferencesFound, "mydb/analytics", "skeema diff --partitioning=remove")
|
||||
s.handleCommand(t, CodeSuccess, "mydb/analytics", "skeema push --partitioning=remove")
|
||||
cfg := s.handleCommand(t, CodeSuccess, "mydb/analytics", "skeema pull")
|
||||
s.verifyFiles(t, cfg, "../golden/init")
|
||||
|
||||
// Repartition with 2 partitions and push. Confirm that dropping the table
|
||||
// works correctly regardless of partitioning option.
|
||||
for _, value := range []string{"keep", "modify", "remove"} {
|
||||
fs.WriteTestFile(t, "mydb/analytics/activity.sql", contents2Part)
|
||||
s.handleCommand(t, CodeSuccess, "mydb/analytics", "skeema push") // default is keep
|
||||
fs.RemoveTestFile(t, "mydb/analytics/activity.sql")
|
||||
s.handleCommand(t, CodeDifferencesFound, "mydb/analytics", "skeema diff --allow-unsafe --partitioning=%s", value)
|
||||
s.handleCommand(t, CodeSuccess, "mydb/analytics", "skeema push --allow-unsafe --partitioning=%s", value)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,5 +8,7 @@ CREATE TABLE `subscriptions` (
|
|||
KEY `sub_id_user` (`subscription_id`,`user_id`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=latin1
|
||||
PARTITION BY RANGE (`user_id`)
|
||||
SUBPARTITION BY HASH (`post_id`)
|
||||
SUBPARTITIONS 2
|
||||
(PARTITION `p0` VALUES LESS THAN (123) ENGINE = InnoDB,
|
||||
PARTITION `p1` VALUES LESS THAN MAXVALUE ENGINE = InnoDB);
|
||||
|
|
|
|||
|
|
@ -8,5 +8,7 @@ CREATE TABLE `subscriptions` (
|
|||
KEY `sub_id_user` (`subscription_id`,`user_id`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=latin1
|
||||
/*!50100 PARTITION BY RANGE (user_id)
|
||||
SUBPARTITION BY HASH (post_id)
|
||||
SUBPARTITIONS 2
|
||||
(PARTITION p0 VALUES LESS THAN (123) ENGINE = InnoDB,
|
||||
PARTITION p1 VALUES LESS THAN MAXVALUE ENGINE = InnoDB) */;
|
||||
|
|
|
|||
|
|
@ -8,5 +8,7 @@ CREATE TABLE `subscriptions` (
|
|||
KEY `sub_id_user` (`subscription_id`,`user_id`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=latin1
|
||||
/*!50100 PARTITION BY RANGE (`user_id`)
|
||||
SUBPARTITION BY HASH (`post_id`)
|
||||
SUBPARTITIONS 2
|
||||
(PARTITION p0 VALUES LESS THAN (123) ENGINE = InnoDB,
|
||||
PARTITION p1 VALUES LESS THAN MAXVALUE ENGINE = InnoDB) */;
|
||||
|
|
|
|||
|
|
@ -8,5 +8,7 @@ CREATE TABLE `subscriptions` (
|
|||
KEY `sub_id_user` (`subscription_id`,`user_id`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=latin1
|
||||
/*!50100 PARTITION BY RANGE (user_id)
|
||||
SUBPARTITION BY HASH (post_id)
|
||||
SUBPARTITIONS 2
|
||||
(PARTITION p0 VALUES LESS THAN (123) ENGINE = InnoDB,
|
||||
PARTITION p1 VALUES LESS THAN MAXVALUE ENGINE = InnoDB) */;
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@ ALTER TABLE subscriptions
|
|||
ADD COLUMN subscription_id int(10) unsigned NOT NULL AUTO_INCREMENT FIRST,
|
||||
ADD KEY sub_id_user (subscription_id, user_id),
|
||||
AUTO_INCREMENT=456
|
||||
PARTITION BY RANGE (user_id) (
|
||||
PARTITION BY RANGE (user_id)
|
||||
SUBPARTITION BY HASH(post_id)
|
||||
SUBPARTITIONS 2 (
|
||||
PARTITION p0 VALUES LESS THAN (123),
|
||||
PARTITION p1 VALUES LESS THAN MAXVALUE
|
||||
)
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ The `tengo.Instance` struct models a single database instance. It keeps track of
|
|||
|
||||
## Status
|
||||
|
||||
This is beta software. The API is subject to change. Backwards-incompatible changes are generally avoided, but no guarantees are made yet. Documentation and usage examples have not yet been completed.
|
||||
This is package is intended for production use. The release numbering is still pre-1.0 though as the API is subject to minor changes. Backwards-incompatible changes are generally avoided whenever possible, but no guarantees are made yet.
|
||||
|
||||
### Supported databases
|
||||
|
||||
|
|
@ -33,21 +33,29 @@ Tagged releases are tested against the following databases, all running on Linux
|
|||
|
||||
Outside of a tagged release, every commit to the master branch is automatically tested against MySQL 5.6 and 5.7.
|
||||
|
||||
### Unsupported in diffs
|
||||
### Unsupported in table diffs
|
||||
|
||||
Go La Tengo **cannot** diff tables containing any of the following MySQL features yet:
|
||||
|
||||
* partitioned tables
|
||||
* triggers
|
||||
* spatial types
|
||||
* special features of non-InnoDB storage engines
|
||||
* column-level compression, with or without predefined dictionary (Percona Server 5.6.33+)
|
||||
* spatial indexes
|
||||
* sub-partitioning (two levels of partitioning in the same table)
|
||||
* CHECK constraints (MySQL 8.0.16+ / Percona Server 8.0.16+ / MariaDB 10.2+)
|
||||
* column-level compression, with or without predefined dictionary (Percona Server 5.6.33+)
|
||||
* special features of non-InnoDB storage engines
|
||||
|
||||
This list is not necessarily exhaustive. Many of these will be implemented in subsequent releases.
|
||||
|
||||
Go La Tengo also does not yet support rename operations, e.g. column renames or table renames.
|
||||
|
||||
### Ignored object types
|
||||
|
||||
The following object types are completely ignored by this package. Their presence won't break anything, but they will not be introspected or represented by the structs in this package.
|
||||
|
||||
* views
|
||||
* triggers
|
||||
* events
|
||||
* grants / users / roles
|
||||
|
||||
## External Dependencies
|
||||
|
||||
* http://github.com/go-sql-driver/mysql (Mozilla Public License 2.0)
|
||||
|
|
@ -70,6 +78,8 @@ Additional [contributions](https://github.com/skeema/tengo/graphs/contributors)
|
|||
|
||||
Support for stored procedures and functions generously sponsored by [Psyonix](https://psyonix.com).
|
||||
|
||||
Support for partitioned tables generously sponsored by [Etsy](https://www.etsy.com).
|
||||
|
||||
## License
|
||||
|
||||
**Copyright 2019 Skeema LLC**
|
||||
|
|
|
|||
|
|
@ -63,9 +63,10 @@ func (dc DropColumn) Clause(_ StatementModifiers) string {
|
|||
}
|
||||
|
||||
// Unsafe returns true if this clause is potentially destructive of data.
|
||||
// DropColumn is always unsafe.
|
||||
// DropColumn is always unsafe, unless it's a virtual column (which is easy to
|
||||
// roll back; there's no inherent data loss from dropping a virtual column).
|
||||
func (dc DropColumn) Unsafe() bool {
|
||||
return true
|
||||
return !dc.Column.Virtual
|
||||
}
|
||||
|
||||
///// AddIndex /////////////////////////////////////////////////////////////////
|
||||
|
|
@ -199,6 +200,10 @@ func (mc ModifyColumn) Clause(mods StatementModifiers) string {
|
|||
// increasing the size of a varchar is safe, but changing decreasing the size or
|
||||
// changing the column type entirely is considered unsafe.
|
||||
func (mc ModifyColumn) Unsafe() bool {
|
||||
if mc.OldColumn.Virtual {
|
||||
return false
|
||||
}
|
||||
|
||||
if mc.OldColumn.CharSet != mc.NewColumn.CharSet {
|
||||
return true
|
||||
}
|
||||
|
|
@ -524,3 +529,75 @@ func (cse ChangeStorageEngine) Clause(_ StatementModifiers) string {
|
|||
func (cse ChangeStorageEngine) Unsafe() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
///// PartitionBy //////////////////////////////////////////////////////////////
|
||||
|
||||
// PartitionBy represents initially partitioning a previously-unpartitioned
|
||||
// table, or changing the partitioning method and/or expression on an already-
|
||||
// partitioned table. It satisfies the TableAlterClause interface.
|
||||
type PartitionBy struct {
|
||||
Partitioning *TablePartitioning
|
||||
RePartition bool // true if changing partitioning on already-partitioned table
|
||||
}
|
||||
|
||||
// Clause returns a clause of an ALTER TABLE statement that partitions a
|
||||
// previously-unpartitioned table.
|
||||
func (pb PartitionBy) Clause(mods StatementModifiers) string {
|
||||
if mods.Partitioning == PartitioningRemove || (pb.RePartition && mods.Partitioning == PartitioningKeep) {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(pb.Partitioning.Definition(mods.Flavor))
|
||||
}
|
||||
|
||||
///// RemovePartitioning ///////////////////////////////////////////////////////
|
||||
|
||||
// RemovePartitioning represents de-partitioning a previously-partitioned table.
|
||||
// It satisfies the TableAlterClause interface.
|
||||
type RemovePartitioning struct{}
|
||||
|
||||
// Clause returns a clause of an ALTER TABLE statement that partitions a
|
||||
// previously-unpartitioned table.
|
||||
func (rp RemovePartitioning) Clause(mods StatementModifiers) string {
|
||||
if mods.Partitioning == PartitioningKeep {
|
||||
return ""
|
||||
}
|
||||
return "REMOVE PARTITIONING"
|
||||
}
|
||||
|
||||
///// ModifyPartitions /////////////////////////////////////////////////////////
|
||||
|
||||
// ModifyPartitions represents a change to the partition list for a table using
|
||||
// RANGE, RANGE COLUMNS, LIST, or LIST COLUMNS partitioning. Generation of this
|
||||
// clause is only partially supported at this time.
|
||||
type ModifyPartitions struct {
|
||||
Add []*Partition
|
||||
Drop []*Partition
|
||||
ForDropTable bool
|
||||
}
|
||||
|
||||
// Clause currently returns an empty string when a partition list difference
|
||||
// is present in a table that exists in both "from" and "to" sides of the diff;
|
||||
// in that situation, ModifyPartitions is just used as a placeholder to indicate
|
||||
// that a difference was detected.
|
||||
// ModifyPartitions currently returns a non-empty clause string only for the
|
||||
// use-case of dropping individual partitions before dropping a table entirely,
|
||||
// which reduces the amount of time the dict_sys mutex is held when dropping the
|
||||
// table.
|
||||
func (mp ModifyPartitions) Clause(mods StatementModifiers) string {
|
||||
if !mp.ForDropTable || len(mp.Drop) == 0 {
|
||||
return ""
|
||||
}
|
||||
if mp.ForDropTable && mods.SkipPreDropAlters {
|
||||
return ""
|
||||
}
|
||||
var names []string
|
||||
for _, p := range mp.Drop {
|
||||
names = append(names, p.Name)
|
||||
}
|
||||
return fmt.Sprintf("DROP PARTITION %s", strings.Join(names, ", "))
|
||||
}
|
||||
|
||||
// Unsafe returns true if this clause is potentially destructive of data.
|
||||
func (mp ModifyPartitions) Unsafe() bool {
|
||||
return len(mp.Drop) > 0
|
||||
}
|
||||
|
|
|
|||
|
|
@ -56,20 +56,33 @@ const (
|
|||
NextAutoIncAlways // always include auto-inc value in diff
|
||||
)
|
||||
|
||||
// PartitioningMode enumerates ways of handling partitioning status -- that is,
|
||||
// presence or lack of a PARTITION BY clause.
|
||||
type PartitioningMode int
|
||||
|
||||
// Constants for how to handle partitioning status differences.
|
||||
const (
|
||||
PartitioningPermissive PartitioningMode = iota // don't negate any partitioning-related clauses
|
||||
PartitioningRemove // negate PARTITION BY clauses from DDL
|
||||
PartitioningKeep // negate REMOVE PARTITIONING clauses from ALTERs
|
||||
)
|
||||
|
||||
// StatementModifiers are options that may be applied to adjust the DDL emitted
|
||||
// for a particular table, and/or generate errors if certain clauses are
|
||||
// present.
|
||||
type StatementModifiers struct {
|
||||
NextAutoInc NextAutoIncMode // How to handle differences in next-auto-inc values
|
||||
AllowUnsafe bool // Whether to allow potentially-destructive DDL (drop table, drop column, modify col type, etc)
|
||||
LockClause string // Include a LOCK=[value] clause in generated ALTER TABLE
|
||||
AlgorithmClause string // Include an ALGORITHM=[value] clause in generated ALTER TABLE
|
||||
IgnoreTable *regexp.Regexp // Generate blank DDL if table name matches this regexp
|
||||
StrictIndexOrder bool // If true, maintain index order even in cases where there is no functional difference
|
||||
StrictForeignKeyNaming bool // If true, maintain foreign key names even if no functional difference in definition
|
||||
CompareMetadata bool // If true, compare creation-time sql_mode and db collation for funcs, procs (and eventually events, triggers)
|
||||
VirtualColValidation bool // If true, add WITH VALIDATION clause for ALTER TABLE affecting virtual columns
|
||||
Flavor Flavor // Adjust generated DDL to match vendor/version. Zero value is FlavorUnknown which makes no adjustments.
|
||||
NextAutoInc NextAutoIncMode // How to handle differences in next-auto-inc values
|
||||
Partitioning PartitioningMode // How to handle differences in partitioning status
|
||||
AllowUnsafe bool // Whether to allow potentially-destructive DDL (drop table, drop column, modify col type, etc)
|
||||
LockClause string // Include a LOCK=[value] clause in generated ALTER TABLE
|
||||
AlgorithmClause string // Include an ALGORITHM=[value] clause in generated ALTER TABLE
|
||||
IgnoreTable *regexp.Regexp // Generate blank DDL if table name matches this regexp
|
||||
StrictIndexOrder bool // If true, maintain index order even in cases where there is no functional difference
|
||||
StrictForeignKeyNaming bool // If true, maintain foreign key names even if no functional difference in definition
|
||||
CompareMetadata bool // If true, compare creation-time sql_mode and db collation for funcs, procs (and eventually events, triggers)
|
||||
VirtualColValidation bool // If true, add WITH VALIDATION clause for ALTER TABLE affecting virtual columns
|
||||
SkipPreDropAlters bool // If true, skip ALTERs that were only generated to make DROP TABLE faster
|
||||
Flavor Flavor // Adjust generated DDL to match vendor/version. Zero value is FlavorUnknown which makes no adjustments.
|
||||
}
|
||||
|
||||
///// SchemaDiff ///////////////////////////////////////////////////////////////
|
||||
|
|
@ -107,6 +120,7 @@ func compareTables(from, to *Schema) []*TableDiff {
|
|||
for name, fromTable := range fromByName {
|
||||
toTable, stillExists := toByName[name]
|
||||
if !stillExists {
|
||||
tableDiffs = append(tableDiffs, PreDropAlters(fromTable)...)
|
||||
tableDiffs = append(tableDiffs, NewDropTable(fromTable))
|
||||
continue
|
||||
}
|
||||
|
|
@ -367,6 +381,39 @@ func NewDropTable(table *Table) *TableDiff {
|
|||
}
|
||||
}
|
||||
|
||||
// PreDropAlters returns a slice of *TableDiff to run prior to dropping a
|
||||
// table. For tables partitioned with RANGE or LIST partitioning, this returns
|
||||
// ALTERs to drop all partitions but one. In all other cases, this returns nil.
|
||||
func PreDropAlters(table *Table) []*TableDiff {
|
||||
if table.Partitioning == nil || table.Partitioning.SubMethod != "" {
|
||||
return nil
|
||||
}
|
||||
// Only RANGE, RANGE COLUMNS, LIST, LIST COLUMNS support ALTER TABLE...DROP
|
||||
// PARTITION clause
|
||||
if !strings.HasPrefix(table.Partitioning.Method, "RANGE") && !strings.HasPrefix(table.Partitioning.Method, "LIST") {
|
||||
return nil
|
||||
}
|
||||
|
||||
fakeTo := &Table{}
|
||||
*fakeTo = *table
|
||||
fakeTo.Partitioning = nil
|
||||
var result []*TableDiff
|
||||
for _, p := range table.Partitioning.Partitions[0 : len(table.Partitioning.Partitions)-1] {
|
||||
clause := ModifyPartitions{
|
||||
Drop: []*Partition{p},
|
||||
ForDropTable: true,
|
||||
}
|
||||
result = append(result, &TableDiff{
|
||||
Type: DiffTypeAlter,
|
||||
From: table,
|
||||
To: fakeTo,
|
||||
alterClauses: []TableAlterClause{clause},
|
||||
supported: true,
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// SplitAddForeignKeys looks through a TableDiff's alterClauses and pulls out
|
||||
// any AddForeignKey clauses into a separate TableDiff. The first returned
|
||||
// TableDiff is guaranteed to contain no AddForeignKey clauses, and the second
|
||||
|
|
@ -435,6 +482,9 @@ func (td *TableDiff) Statement(mods StatementModifiers) (string, error) {
|
|||
switch td.Type {
|
||||
case DiffTypeCreate:
|
||||
stmt := td.To.CreateStatement
|
||||
if td.To.Partitioning != nil && mods.Partitioning == PartitioningRemove {
|
||||
stmt = td.To.UnpartitionedCreateStatement(mods.Flavor)
|
||||
}
|
||||
if td.To.HasAutoIncrement() && (mods.NextAutoInc == NextAutoIncIgnore || mods.NextAutoInc == NextAutoIncIfAlready) {
|
||||
stmt, _ = ParseCreateAutoInc(stmt)
|
||||
}
|
||||
|
|
@ -508,6 +558,7 @@ func (td *TableDiff) alterStatement(mods StatementModifiers) (string, error) {
|
|||
}
|
||||
|
||||
clauseStrings := make([]string, 0, len(td.alterClauses))
|
||||
var partitionClauseString string
|
||||
var err error
|
||||
for _, clause := range td.alterClauses {
|
||||
if err == nil && !mods.AllowUnsafe {
|
||||
|
|
@ -519,10 +570,23 @@ func (td *TableDiff) alterStatement(mods StatementModifiers) (string, error) {
|
|||
}
|
||||
}
|
||||
if clauseString := clause.Clause(mods); clauseString != "" {
|
||||
clauseStrings = append(clauseStrings, clauseString)
|
||||
switch clause.(type) {
|
||||
case PartitionBy, RemovePartitioning:
|
||||
// Adding or removing partitioning must occur at the end of the ALTER
|
||||
// TABLE, and oddly *without* a preceeding comma
|
||||
partitionClauseString = clauseString
|
||||
case ModifyPartitions:
|
||||
// Other partitioning-related clauses cannot appear alongside any other
|
||||
// clauses, including ALGORITHM or LOCK clauses
|
||||
mods.LockClause = ""
|
||||
mods.AlgorithmClause = ""
|
||||
clauseStrings = append(clauseStrings, clauseString)
|
||||
default:
|
||||
clauseStrings = append(clauseStrings, clauseString)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(clauseStrings) == 0 {
|
||||
if len(clauseStrings) == 0 && partitionClauseString == "" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
|
|
@ -549,7 +613,10 @@ func (td *TableDiff) alterStatement(mods StatementModifiers) (string, error) {
|
|||
}
|
||||
}
|
||||
|
||||
stmt := fmt.Sprintf("%s %s", td.From.AlterStatement(), strings.Join(clauseStrings, ", "))
|
||||
if len(clauseStrings) > 0 && partitionClauseString != "" {
|
||||
partitionClauseString = fmt.Sprintf(" %s", partitionClauseString)
|
||||
}
|
||||
stmt := fmt.Sprintf("%s %s%s", td.From.AlterStatement(), strings.Join(clauseStrings, ", "), partitionClauseString)
|
||||
if fde, isForbiddenDiff := err.(*ForbiddenDiffError); isForbiddenDiff {
|
||||
fde.Statement = stmt
|
||||
}
|
||||
|
|
|
|||
|
|
@ -600,9 +600,10 @@ func (instance *Instance) AlterSchema(schema string, opts SchemaCreationOptions)
|
|||
|
||||
// BulkDropOptions controls how objects are dropped in bulk.
|
||||
type BulkDropOptions struct {
|
||||
OnlyIfEmpty bool // If true, when dropping tables, error if any have rows
|
||||
MaxConcurrency int // Max objects to drop at once
|
||||
SkipBinlog bool // If true, use session sql_log_bin=0 (requires superuser)
|
||||
OnlyIfEmpty bool // If true, when dropping tables, error if any have rows
|
||||
MaxConcurrency int // Max objects to drop at once
|
||||
SkipBinlog bool // If true, use session sql_log_bin=0 (requires superuser)
|
||||
PartitionsFirst bool // If true, drop RANGE/LIST partitioned tables one partition at a time
|
||||
}
|
||||
|
||||
func (opts BulkDropOptions) params() string {
|
||||
|
|
@ -628,41 +629,45 @@ func (instance *Instance) DropTablesInSchema(schema string, opts BulkDropOptions
|
|||
return err
|
||||
}
|
||||
|
||||
// Obtain table names directly; faster than going through instance.Schema(schema)
|
||||
// since we don't need other info besides the names
|
||||
var names []string
|
||||
query := `
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = ?
|
||||
AND table_type = 'BASE TABLE'`
|
||||
if err := db.Select(&names, query, schema); err != nil {
|
||||
// Obtain table and partition names
|
||||
tableMap, err := tablesToPartitions(db, schema)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if len(names) == 0 {
|
||||
} else if len(tableMap) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// If requested, confirm tables are empty
|
||||
if opts.OnlyIfEmpty {
|
||||
names := make([]string, 0, len(tableMap))
|
||||
for tableName := range tableMap {
|
||||
names = append(names, tableName)
|
||||
}
|
||||
if err := confirmTablesEmpty(db, names); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
th := throttler.New(opts.Concurrency(), len(names))
|
||||
retries := make(chan string, len(names))
|
||||
for _, name := range names {
|
||||
go func(name string) {
|
||||
_, err := db.Exec(fmt.Sprintf("DROP TABLE %s", EscapeIdentifier(name)))
|
||||
// With the new data dictionary added in MySQL 8.0, attempting to
|
||||
// concurrently drop two tables that have a foreign key constraint between
|
||||
// them can deadlock.
|
||||
if IsDatabaseError(err, mysqlerr.ER_LOCK_DEADLOCK) {
|
||||
retries <- name
|
||||
err = nil
|
||||
th := throttler.New(opts.Concurrency(), len(tableMap))
|
||||
retries := make(chan string, len(tableMap))
|
||||
for name, partitions := range tableMap {
|
||||
go func(name string, partitions []string) {
|
||||
var err error
|
||||
if len(partitions) > 1 && opts.PartitionsFirst {
|
||||
err = dropPartitions(db, name, partitions[0:len(partitions)-1])
|
||||
}
|
||||
if err == nil {
|
||||
_, err := db.Exec(fmt.Sprintf("DROP TABLE %s", EscapeIdentifier(name)))
|
||||
// With the new data dictionary added in MySQL 8.0, attempting to
|
||||
// concurrently drop two tables that have a foreign key constraint between
|
||||
// them can deadlock.
|
||||
if IsDatabaseError(err, mysqlerr.ER_LOCK_DEADLOCK) {
|
||||
retries <- name
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
th.Done(err)
|
||||
}(name)
|
||||
}(name, partitions)
|
||||
th.Throttle()
|
||||
}
|
||||
close(retries)
|
||||
|
|
@ -714,6 +719,60 @@ func (instance *Instance) DropRoutinesInSchema(schema string, opts BulkDropOptio
|
|||
return nil
|
||||
}
|
||||
|
||||
// tablesToPartitions returns a map whose keys are all tables in the schema
|
||||
// (whether partitioned or not), and values are either nil (if unpartitioned or
|
||||
// partitioned in a way that doesn't support DROP PARTITION) or a slice of
|
||||
// partition names (if using RANGE or LIST partitioning). Views are excluded
|
||||
// from the result.
|
||||
func tablesToPartitions(db *sqlx.DB, schema string) (map[string][]string, error) {
|
||||
// information_schema.partitions contains all tables (not just partitioned)
|
||||
// and excludes views (which we don't want here anyway)
|
||||
var rawNames []struct {
|
||||
TableName string `db:"table_name"`
|
||||
PartitionName sql.NullString `db:"partition_name"`
|
||||
Method sql.NullString `db:"partition_method"`
|
||||
SubMethod sql.NullString `db:"subpartition_method"`
|
||||
Position sql.NullInt64 `db:"partition_ordinal_position"`
|
||||
}
|
||||
// Explicit AS clauses needed for compatibility with MySQL 8 data dictionary,
|
||||
// otherwise results come back with uppercase col names, breaking Select
|
||||
query := `
|
||||
SELECT p.table_name AS table_name, p.partition_name AS partition_name,
|
||||
p.partition_method AS partition_method,
|
||||
p.subpartition_method AS subpartition_method,
|
||||
p.partition_ordinal_position AS partition_ordinal_position
|
||||
FROM information_schema.partitions p
|
||||
WHERE p.table_schema = ?
|
||||
ORDER BY p.table_name, p.partition_ordinal_position`
|
||||
if err := db.Select(&rawNames, query, schema); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
partitions := make(map[string][]string)
|
||||
for _, rn := range rawNames {
|
||||
if !rn.Position.Valid || rn.Position.Int64 == 1 {
|
||||
partitions[rn.TableName] = nil
|
||||
}
|
||||
if rn.Method.Valid && !rn.SubMethod.Valid &&
|
||||
(strings.HasPrefix(rn.Method.String, "RANGE") || strings.HasPrefix(rn.Method.String, "LIST")) {
|
||||
partitions[rn.TableName] = append(partitions[rn.TableName], rn.PartitionName.String)
|
||||
}
|
||||
}
|
||||
return partitions, nil
|
||||
}
|
||||
|
||||
func dropPartitions(db *sqlx.DB, table string, partitions []string) error {
|
||||
for _, partName := range partitions {
|
||||
_, err := db.Exec(fmt.Sprintf("ALTER TABLE %s DROP PARTITION %s",
|
||||
EscapeIdentifier(table),
|
||||
EscapeIdentifier(partName)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultCharSetAndCollation returns the instance's default character set and
|
||||
// collation
|
||||
func (instance *Instance) DefaultCharSetAndCollation() (serverCharSet, serverCollation string, err error) {
|
||||
|
|
@ -823,6 +882,7 @@ func (instance *Instance) querySchemaTables(schema string) ([]*Table, error) {
|
|||
return []*Table{}, nil
|
||||
}
|
||||
tables := make([]*Table, len(rawTables))
|
||||
var havePartitions bool
|
||||
for n, rawTable := range rawTables {
|
||||
tables[n] = &Table{
|
||||
Name: rawTable.Name,
|
||||
|
|
@ -835,16 +895,16 @@ func (instance *Instance) querySchemaTables(schema string) ([]*Table, error) {
|
|||
if rawTable.AutoIncrement.Valid {
|
||||
tables[n].NextAutoIncrement = uint64(rawTable.AutoIncrement.Int64)
|
||||
}
|
||||
if rawTable.CreateOptions.Valid && rawTable.CreateOptions.String != "" && rawTable.CreateOptions.String != "PARTITIONED" {
|
||||
// information_schema.tables.create_options annoyingly contains "partitioned"
|
||||
// if the table is partitioned, despite this not being present as-is in the
|
||||
// table table definition. All other create_options are present verbatim.
|
||||
// Currently in mysql-server/sql/sql_show.cc, it's always at the *end* of
|
||||
// create_options... but just to code defensively we handle any location.
|
||||
if strings.HasPrefix(rawTable.CreateOptions.String, "PARTITIONED ") {
|
||||
tables[n].CreateOptions = strings.Replace(rawTable.CreateOptions.String, "PARTITIONED ", "", 1)
|
||||
} else {
|
||||
tables[n].CreateOptions = strings.Replace(rawTable.CreateOptions.String, " PARTITIONED", "", 1)
|
||||
if rawTable.CreateOptions.Valid && rawTable.CreateOptions.String != "" {
|
||||
tables[n].CreateOptions = rawTable.CreateOptions.String
|
||||
|
||||
// information_schema.tables.create_options contains "partitioned" if the
|
||||
// table is partitioned, despite this not being present as-is in the table
|
||||
// definition. All other create_options are present verbatim.
|
||||
if strings.Contains(tables[n].CreateOptions, "PARTITIONED") {
|
||||
tables[n].CreateOptions = strings.Replace(tables[n].CreateOptions, "PARTITIONED", "", 1)
|
||||
tables[n].CreateOptions = strings.TrimSpace(strings.Replace(tables[n].CreateOptions, " ", " ", 1))
|
||||
havePartitions = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1077,6 +1137,68 @@ func (instance *Instance) querySchemaTables(schema string) ([]*Table, error) {
|
|||
t.ForeignKeys = foreignKeysByTableName[t.Name]
|
||||
}
|
||||
|
||||
// Obtain partitioning information, if at least one table was partitioned
|
||||
if havePartitions {
|
||||
var rawPartitioning []struct {
|
||||
TableName string `db:"table_name"`
|
||||
PartitionName string `db:"partition_name"`
|
||||
SubName sql.NullString `db:"subpartition_name"`
|
||||
Method string `db:"partition_method"`
|
||||
SubMethod sql.NullString `db:"subpartition_method"`
|
||||
Expression sql.NullString `db:"partition_expression"`
|
||||
SubExpression sql.NullString `db:"subpartition_expression"`
|
||||
Values sql.NullString `db:"partition_description"`
|
||||
Comment string `db:"partition_comment"`
|
||||
}
|
||||
query := `
|
||||
SELECT p.table_name AS table_name, p.partition_name AS partition_name,
|
||||
p.subpartition_name AS subpartition_name,
|
||||
p.partition_method AS partition_method,
|
||||
p.subpartition_method AS subpartition_method,
|
||||
p.partition_expression AS partition_expression,
|
||||
p.subpartition_expression AS subpartition_expression,
|
||||
p.partition_description AS partition_description,
|
||||
p.partition_comment AS partition_comment
|
||||
FROM partitions p
|
||||
WHERE p.table_schema = ?
|
||||
AND p.partition_name IS NOT NULL
|
||||
ORDER BY p.table_name, p.partition_ordinal_position,
|
||||
p.subpartition_ordinal_position`
|
||||
if err := db.Select(&rawPartitioning, query, schema); err != nil {
|
||||
return nil, fmt.Errorf("Error querying information_schema.partitions for schema %s: %s", schema, err)
|
||||
}
|
||||
|
||||
partitioningByTableName := make(map[string]*TablePartitioning)
|
||||
for _, rawPart := range rawPartitioning {
|
||||
p, ok := partitioningByTableName[rawPart.TableName]
|
||||
if !ok {
|
||||
p = &TablePartitioning{
|
||||
Method: rawPart.Method,
|
||||
SubMethod: rawPart.SubMethod.String,
|
||||
Expression: rawPart.Expression.String,
|
||||
SubExpression: rawPart.SubExpression.String,
|
||||
Partitions: make([]*Partition, 0),
|
||||
}
|
||||
partitioningByTableName[rawPart.TableName] = p
|
||||
}
|
||||
p.Partitions = append(p.Partitions, &Partition{
|
||||
Name: rawPart.PartitionName,
|
||||
SubName: rawPart.SubName.String,
|
||||
Values: rawPart.Values.String,
|
||||
Comment: rawPart.Comment,
|
||||
method: rawPart.Method,
|
||||
})
|
||||
}
|
||||
for _, t := range tables {
|
||||
if p, ok := partitioningByTableName[t.Name]; ok {
|
||||
for _, part := range p.Partitions {
|
||||
part.engine = t.Engine
|
||||
}
|
||||
t.Partitioning = p
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Obtain actual SHOW CREATE TABLE output and store in each table. Since
|
||||
// there's no way in MySQL to bulk fetch this for multiple tables at once,
|
||||
// use multiple goroutines to make this faster.
|
||||
|
|
@ -1095,6 +1217,9 @@ func (instance *Instance) querySchemaTables(schema string) ([]*Table, error) {
|
|||
if t.Engine == "InnoDB" {
|
||||
t.CreateStatement = NormalizeCreateOptions(t.CreateStatement)
|
||||
}
|
||||
if t.Partitioning != nil {
|
||||
fixPartitioningEdgeCases(t, flavor)
|
||||
}
|
||||
// Index order is unpredictable with new MySQL 8 data dictionary, so reorder
|
||||
// indexes based on parsing SHOW CREATE TABLE if needed
|
||||
if flavor.HasDataDictionary() && len(t.SecondaryIndexes) > 1 {
|
||||
|
|
@ -1230,6 +1355,51 @@ func fixGenerationExpr(t *Table, flavor Flavor) {
|
|||
}
|
||||
}
|
||||
|
||||
// fixPartitioningEdgeCases handles situations that are reflected in SHOW CREATE
|
||||
// TABLE, but missing (or difficult to obtain) in information_schema.
|
||||
func fixPartitioningEdgeCases(t *Table, flavor Flavor) {
|
||||
// Handle edge cases for how partitions are expressed in HASH or KEY methods:
|
||||
// typically this will just be a PARTITIONS N clause, but it could also be
|
||||
// nothing at all, or an explicit list of partitions, depending on how the
|
||||
// partitioning was originally created.
|
||||
if strings.HasSuffix(t.Partitioning.Method, "HASH") || strings.HasSuffix(t.Partitioning.Method, "KEY") {
|
||||
countClause := fmt.Sprintf("\nPARTITIONS %d", len(t.Partitioning.Partitions))
|
||||
if strings.Contains(t.CreateStatement, countClause) {
|
||||
t.Partitioning.forcePartitionList = partitionListCount
|
||||
} else if strings.Contains(t.CreateStatement, "\n(PARTITION ") {
|
||||
t.Partitioning.forcePartitionList = partitionListExplicit
|
||||
} else if len(t.Partitioning.Partitions) == 1 {
|
||||
t.Partitioning.forcePartitionList = partitionListNone
|
||||
}
|
||||
}
|
||||
|
||||
// KEY methods support an optional ALGORITHM clause, which is present in SHOW
|
||||
// CREATE TABLE but not anywhere in information_schema
|
||||
if strings.HasSuffix(t.Partitioning.Method, "KEY") && strings.Contains(t.CreateStatement, "ALGORITHM") {
|
||||
re := regexp.MustCompile(fmt.Sprintf(`PARTITION BY %s ([^(]*)\(`, t.Partitioning.Method))
|
||||
if matches := re.FindStringSubmatch(t.CreateStatement); matches != nil {
|
||||
t.Partitioning.algoClause = matches[1]
|
||||
}
|
||||
}
|
||||
|
||||
// Process DATA DIRECTORY clauses, which are easier to parse from SHOW CREATE
|
||||
// TABLE instead of information_schema.innodb_sys_tablespaces.
|
||||
if (t.Partitioning.forcePartitionList == partitionListDefault || t.Partitioning.forcePartitionList == partitionListExplicit) &&
|
||||
strings.Contains(t.CreateStatement, " DATA DIRECTORY = ") {
|
||||
for _, p := range t.Partitioning.Partitions {
|
||||
name := p.Name
|
||||
if flavor.VendorMinVersion(VendorMariaDB, 10, 2) {
|
||||
name = EscapeIdentifier(name)
|
||||
}
|
||||
name = regexp.QuoteMeta(name)
|
||||
re := regexp.MustCompile(fmt.Sprintf(`PARTITION %s .*DATA DIRECTORY = '((?:\\\\|\\'|''|[^'])*)'`, name))
|
||||
if matches := re.FindStringSubmatch(t.CreateStatement); matches != nil {
|
||||
p.dataDir = matches[1]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (instance *Instance) querySchemaRoutines(schema string) ([]*Routine, error) {
|
||||
db, err := instance.Connect("information_schema", "")
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,179 @@
|
|||
package tengo
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// partitionListMode enum values control edge-cases for how the list of
|
||||
// partitions is represented in SHOW CREATE TABLE.
|
||||
type partitionListMode int
|
||||
|
||||
const (
|
||||
partitionListDefault partitionListMode = iota
|
||||
partitionListExplicit // List each partition individually
|
||||
partitionListCount // Just use a count of partitions
|
||||
partitionListNone // Omit partition list and count, implying just 1 partition
|
||||
)
|
||||
|
||||
// TablePartitioning stores partitioning configuration for a partitioned table.
|
||||
// Note that despite subpartitioning fields being present and possibly
|
||||
// populated, the rest of this package does not fully support subpartitioning
|
||||
// yet.
|
||||
type TablePartitioning struct {
|
||||
Method string // one of "RANGE", "RANGE COLUMNS", "LIST", "LIST COLUMNS", "HASH", "LINEAR HASH", "KEY", or "LINEAR KEY"
|
||||
SubMethod string // one of "" (no sub-partitioning), "HASH", "LINEAR HASH", "KEY", or "LINEAR KEY"; not fully supported yet
|
||||
Expression string
|
||||
SubExpression string // empty string if no sub-partitioning; not fully supported yet
|
||||
Partitions []*Partition
|
||||
forcePartitionList partitionListMode
|
||||
algoClause string // full text of optional ALGORITHM clause for KEY or LINEAR KEY
|
||||
}
|
||||
|
||||
// Definition returns the overall partitioning definition for a table.
|
||||
func (tp *TablePartitioning) Definition(flavor Flavor) string {
|
||||
if tp == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
plMode := tp.forcePartitionList
|
||||
if plMode == partitionListDefault {
|
||||
plMode = partitionListCount
|
||||
for n, p := range tp.Partitions {
|
||||
if p.Values != "" || p.Comment != "" || p.dataDir != "" || p.Name != fmt.Sprintf("p%d", n) {
|
||||
plMode = partitionListExplicit
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
var partitionsClause string
|
||||
if plMode == partitionListExplicit {
|
||||
pdefs := make([]string, len(tp.Partitions))
|
||||
for n, p := range tp.Partitions {
|
||||
pdefs[n] = p.Definition(flavor)
|
||||
}
|
||||
partitionsClause = fmt.Sprintf("\n(%s)", strings.Join(pdefs, ",\n "))
|
||||
} else if plMode == partitionListCount {
|
||||
partitionsClause = fmt.Sprintf("\nPARTITIONS %d", len(tp.Partitions))
|
||||
}
|
||||
|
||||
opener, closer := "/*!50100", " */"
|
||||
if flavor.VendorMinVersion(VendorMariaDB, 10, 2) {
|
||||
// MariaDB stopped wrapping partitioning clauses in version-gated comments
|
||||
// in 10.2.
|
||||
opener, closer = "", ""
|
||||
} else if strings.HasSuffix(tp.Method, "COLUMNS") {
|
||||
// RANGE COLUMNS and LIST COLUMNS were introduced in 5.5
|
||||
opener = "/*!50500"
|
||||
}
|
||||
|
||||
return fmt.Sprintf("\n%s PARTITION BY %s%s%s", opener, tp.partitionBy(flavor), partitionsClause, closer)
|
||||
}
|
||||
|
||||
// partitionBy returns the partitioning method and expression, formatted to
|
||||
// match SHOW CREATE TABLE's extremely arbitrary, completely inconsistent way.
|
||||
func (tp *TablePartitioning) partitionBy(flavor Flavor) string {
|
||||
method, expr := fmt.Sprintf("%s ", tp.Method), tp.Expression
|
||||
|
||||
if tp.Method == "RANGE COLUMNS" {
|
||||
method = "RANGE COLUMNS"
|
||||
} else if tp.Method == "LIST COLUMNS" {
|
||||
method = "LIST COLUMNS"
|
||||
}
|
||||
|
||||
if (tp.Method == "RANGE COLUMNS" || strings.HasSuffix(tp.Method, "KEY")) && !flavor.VendorMinVersion(VendorMariaDB, 10, 2) {
|
||||
expr = strings.Replace(expr, "`", "", -1)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s%s(%s)", method, tp.algoClause, expr)
|
||||
}
|
||||
|
||||
// Diff returns a set of differences between this TablePartitioning and another
|
||||
// TablePartitioning. If supported==true, the returned clauses (if executed)
|
||||
// would transform tp into other.
|
||||
func (tp *TablePartitioning) Diff(other *TablePartitioning) (clauses []TableAlterClause, supported bool) {
|
||||
// Handle cases where one or both sides are nil, meaning one or both tables are
|
||||
// unpartitioned
|
||||
if tp == nil && other == nil {
|
||||
return nil, true
|
||||
} else if tp == nil {
|
||||
return []TableAlterClause{PartitionBy{Partitioning: other}}, true
|
||||
} else if other == nil {
|
||||
return []TableAlterClause{RemovePartitioning{}}, true
|
||||
}
|
||||
|
||||
// Modifications to partitioning method or expression: re-partition
|
||||
if tp.Method != other.Method || tp.SubMethod != other.SubMethod ||
|
||||
tp.Expression != other.Expression || tp.SubExpression != other.SubExpression ||
|
||||
tp.algoClause != other.algoClause {
|
||||
clause := PartitionBy{
|
||||
Partitioning: other,
|
||||
RePartition: true,
|
||||
}
|
||||
return []TableAlterClause{clause}, true
|
||||
}
|
||||
|
||||
// Modifications to partition list: ignored for RANGE, RANGE COLUMNS, LIST,
|
||||
// LIST COLUMNS via generation of a no-op placeholder clause. This is done
|
||||
// to side-step the safety mechanism at the end of Table.Diff() which treats 0
|
||||
// clauses as indicative of an unsupported diff.
|
||||
// For other partitioning methods, changing the partition list is currently
|
||||
// unsupported.
|
||||
var foundPartitionsDiff bool
|
||||
if len(tp.Partitions) != len(other.Partitions) {
|
||||
foundPartitionsDiff = true
|
||||
} else {
|
||||
for n := range tp.Partitions {
|
||||
// all Partition fields are scalars, so simple comparison is fine
|
||||
if *tp.Partitions[n] != *other.Partitions[n] {
|
||||
foundPartitionsDiff = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if foundPartitionsDiff && (strings.HasPrefix(tp.Method, "RANGE") || strings.HasPrefix(tp.Method, "LIST")) {
|
||||
return []TableAlterClause{ModifyPartitions{}}, true
|
||||
}
|
||||
return nil, !foundPartitionsDiff
|
||||
}
|
||||
|
||||
// Partition stores information on a single partition.
|
||||
type Partition struct {
|
||||
Name string
|
||||
SubName string // empty string if no sub-partitioning; not fully supported yet
|
||||
Values string // only populated for RANGE or LIST
|
||||
Comment string
|
||||
method string
|
||||
engine string
|
||||
dataDir string
|
||||
}
|
||||
|
||||
// Definition returns this partition's definition clause, for use as part of a
|
||||
// DDL statement. This is only used for some partition methods.
|
||||
func (p *Partition) Definition(flavor Flavor) string {
|
||||
name := p.Name
|
||||
if flavor.VendorMinVersion(VendorMariaDB, 10, 2) {
|
||||
name = EscapeIdentifier(name)
|
||||
}
|
||||
|
||||
var values string
|
||||
if p.method == "RANGE" && p.Values == "MAXVALUE" {
|
||||
values = "VALUES LESS THAN MAXVALUE "
|
||||
} else if strings.Contains(p.method, "RANGE") {
|
||||
values = fmt.Sprintf("VALUES LESS THAN (%s) ", p.Values)
|
||||
} else if strings.Contains(p.method, "LIST") {
|
||||
values = fmt.Sprintf("VALUES IN (%s) ", p.Values)
|
||||
}
|
||||
|
||||
var dataDir string
|
||||
if p.dataDir != "" {
|
||||
dataDir = fmt.Sprintf("DATA DIRECTORY = '%s' ", p.dataDir) // any necessary escaping is already present in p.dataDir
|
||||
}
|
||||
|
||||
var comment string
|
||||
if p.Comment != "" {
|
||||
comment = fmt.Sprintf("COMMENT = '%s' ", EscapeValueForCreateTable(p.Comment))
|
||||
}
|
||||
|
||||
return fmt.Sprintf("PARTITION %s %s%s%sENGINE = %s", name, values, dataDir, comment, p.engine)
|
||||
}
|
||||
|
|
@ -21,8 +21,9 @@ type Table struct {
|
|||
ForeignKeys []*ForeignKey
|
||||
Comment string
|
||||
NextAutoIncrement uint64
|
||||
UnsupportedDDL bool // If true, tengo cannot diff this table or auto-generate its CREATE TABLE
|
||||
CreateStatement string // complete SHOW CREATE TABLE obtained from an instance
|
||||
Partitioning *TablePartitioning // nil if table isn't partitioned
|
||||
UnsupportedDDL bool // If true, tengo cannot diff this table or auto-generate its CREATE TABLE
|
||||
CreateStatement string // complete SHOW CREATE TABLE obtained from an instance
|
||||
}
|
||||
|
||||
// AlterStatement returns the prefix to a SQL "ALTER TABLE" statement.
|
||||
|
|
@ -70,7 +71,7 @@ func (t *Table) GeneratedCreateStatement(flavor Flavor) string {
|
|||
if t.Comment != "" {
|
||||
comment = fmt.Sprintf(" COMMENT='%s'", EscapeValueForCreateTable(t.Comment))
|
||||
}
|
||||
result := fmt.Sprintf("CREATE TABLE %s (\n %s\n) ENGINE=%s%s DEFAULT CHARSET=%s%s%s%s",
|
||||
result := fmt.Sprintf("CREATE TABLE %s (\n %s\n) ENGINE=%s%s DEFAULT CHARSET=%s%s%s%s%s",
|
||||
EscapeIdentifier(t.Name),
|
||||
strings.Join(defs, ",\n "),
|
||||
t.Engine,
|
||||
|
|
@ -79,10 +80,25 @@ func (t *Table) GeneratedCreateStatement(flavor Flavor) string {
|
|||
collate,
|
||||
createOptions,
|
||||
comment,
|
||||
t.Partitioning.Definition(flavor),
|
||||
)
|
||||
return result
|
||||
}
|
||||
|
||||
// UnpartitionedCreateStatement returns the table's CREATE statement without
|
||||
// its PARTITION BY clause. Supplying an accurate flavor improves performance,
|
||||
// but is not required; FlavorUnknown still works correctly.
|
||||
func (t *Table) UnpartitionedCreateStatement(flavor Flavor) string {
|
||||
if t.Partitioning == nil {
|
||||
return t.CreateStatement
|
||||
}
|
||||
if partClause := t.Partitioning.Definition(flavor); strings.HasSuffix(t.CreateStatement, partClause) {
|
||||
return t.CreateStatement[0 : len(t.CreateStatement)-len(partClause)]
|
||||
}
|
||||
base, _ := ParseCreatePartitioning(t.CreateStatement)
|
||||
return base
|
||||
}
|
||||
|
||||
// ColumnsByName returns a mapping of column names to Column value pointers,
|
||||
// for all columns in the table.
|
||||
func (t *Table) ColumnsByName() map[string]*Column {
|
||||
|
|
@ -317,6 +333,17 @@ func (t *Table) Diff(to *Table) (clauses []TableAlterClause, supported bool) {
|
|||
clauses = append(clauses, ChangeComment{NewComment: to.Comment})
|
||||
}
|
||||
|
||||
// Compare partitioning. This must be performed last due to a MySQL requirement
|
||||
// of PARTITION BY / REMOVE PARTITIONING occurring last in a multi-clause ALTER
|
||||
// TABLE.
|
||||
// Note that some partitioning differences aren't supported yet, and others are
|
||||
// intentionally ignored.
|
||||
partClauses, partSupported := from.Partitioning.Diff(to.Partitioning)
|
||||
clauses = append(clauses, partClauses...)
|
||||
if !partSupported {
|
||||
return clauses, false
|
||||
}
|
||||
|
||||
// If the SHOW CREATE TABLE output differed between the two tables, but we
|
||||
// did not generate any clauses, this indicates some aspect of the change is
|
||||
// unsupported (even though the two tables are individually supported). This
|
||||
|
|
|
|||
|
|
@ -73,14 +73,14 @@ func SplitHostOptionalPort(hostaddr string) (string, int, error) {
|
|||
return host, port, nil
|
||||
}
|
||||
|
||||
var reParseCreate = regexp.MustCompile(`[)] ENGINE=\w+ (AUTO_INCREMENT=(\d+) )DEFAULT CHARSET=`)
|
||||
var reParseCreateAutoInc = regexp.MustCompile(`[)] ENGINE=\w+ (AUTO_INCREMENT=(\d+) )DEFAULT CHARSET=`)
|
||||
|
||||
// ParseCreateAutoInc parses a CREATE TABLE statement, formatted in the same
|
||||
// manner as SHOW CREATE TABLE, and removes the table-level next-auto-increment
|
||||
// clause if present. The modified CREATE TABLE will be returned, along with
|
||||
// the next auto-increment value if one was found.
|
||||
func ParseCreateAutoInc(createStmt string) (string, uint64) {
|
||||
matches := reParseCreate.FindStringSubmatch(createStmt)
|
||||
matches := reParseCreateAutoInc.FindStringSubmatch(createStmt)
|
||||
if matches == nil {
|
||||
return createStmt, 0
|
||||
}
|
||||
|
|
@ -89,6 +89,19 @@ func ParseCreateAutoInc(createStmt string) (string, uint64) {
|
|||
return newStmt, nextAutoInc
|
||||
}
|
||||
|
||||
var reParseCreatePartitioning = regexp.MustCompile(`(?is)(\s*(?:/\*!?\d*)?\s*partition\s+by .*)$`)
|
||||
|
||||
// ParseCreatePartitioning parses a CREATE TABLE statement, formatted in the
|
||||
// same manner as SHOW CREATE TABLE, and splits out the base CREATE clauses from
|
||||
// the partioning clause.
|
||||
func ParseCreatePartitioning(createStmt string) (base, partitionClause string) {
|
||||
matches := reParseCreatePartitioning.FindStringSubmatch(createStmt)
|
||||
if matches == nil {
|
||||
return createStmt, ""
|
||||
}
|
||||
return createStmt[0 : len(createStmt)-len(matches[1])], matches[1]
|
||||
}
|
||||
|
||||
var normalizeCreateRegexps = []struct {
|
||||
re *regexp.Regexp
|
||||
replacement string
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ github.com/pmezard/go-difflib/difflib
|
|||
github.com/sirupsen/logrus
|
||||
# github.com/skeema/mybase v1.0.8
|
||||
github.com/skeema/mybase
|
||||
# github.com/skeema/tengo v0.8.20-0.20191115154500-6ca952692106
|
||||
# github.com/skeema/tengo v0.9.0
|
||||
github.com/skeema/tengo
|
||||
# golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
|
||||
golang.org/x/crypto/ssh/terminal
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче