Skip to content

Commit c71dbf9

Browse files
timvaillancourtshlomi-noach
timvaillancourt
and
shlomi-noach
authored
Copy auto increment (#967)
* v1.1.0 * WIP: copying AUTO_INCREMENT value to ghost table Initial commit: towards setting up a test suite Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * greping for 'expect_table_structure' content * Adding simple test for 'expect_table_structure' scenario * adding tests for AUTO_INCREMENT value after row deletes. Should initially fail * clear event beforehand * parsing AUTO_INCREMENT from alter query, reading AUTO_INCREMENT from original table, applying AUTO_INCREMENT value onto ghost table if applicable and user has not specified AUTO_INCREMENT in alter statement * support GetUint64 Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * minor update to test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * adding test for user defined AUTO_INCREMENT statement Co-authored-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
1 parent fef83af commit c71dbf9

15 files changed

Lines changed: 167 additions & 3 deletions

File tree

‎go/base/context.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ type MigrationContext struct {
206206
OriginalTableColumns *sql.ColumnList
207207
OriginalTableVirtualColumns *sql.ColumnList
208208
OriginalTableUniqueKeys [](*sql.UniqueKey)
209+
OriginalTableAutoIncrement uint64
209210
GhostTableColumns *sql.ColumnList
210211
GhostTableVirtualColumns *sql.ColumnList
211212
GhostTableUniqueKeys [](*sql.UniqueKey)

‎go/logic/applier.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,25 @@ func (this *Applier) AlterGhost() error {
207207
return nil
208208
}
209209

210+
// AlterGhost applies `alter` statement on ghost table
211+
func (this *Applier) AlterGhostAutoIncrement() error {
212+
query := fmt.Sprintf(`alter /* gh-ost */ table %s.%s AUTO_INCREMENT=%d`,
213+
sql.EscapeName(this.migrationContext.DatabaseName),
214+
sql.EscapeName(this.migrationContext.GetGhostTableName()),
215+
this.migrationContext.OriginalTableAutoIncrement,
216+
)
217+
this.migrationContext.Log.Infof("Altering ghost table AUTO_INCREMENT value %s.%s",
218+
sql.EscapeName(this.migrationContext.DatabaseName),
219+
sql.EscapeName(this.migrationContext.GetGhostTableName()),
220+
)
221+
this.migrationContext.Log.Debugf("AUTO_INCREMENT ALTER statement: %s", query)
222+
if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
223+
return err
224+
}
225+
this.migrationContext.Log.Infof("Ghost table AUTO_INCREMENT altered")
226+
return nil
227+
}
228+
210229
// CreateChangelogTable creates the changelog table on the applier host
211230
func (this *Applier) CreateChangelogTable() error {
212231
if err := this.DropChangelogTable(); err != nil {

‎go/logic/inspect.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,10 @@ func (this *Inspector) InspectOriginalTable() (err error) {
111111
if err != nil {
112112
return err
113113
}
114+
this.migrationContext.OriginalTableAutoIncrement, err = this.getAutoIncrementValue(this.migrationContext.OriginalTableName)
115+
if err != nil {
116+
return err
117+
}
114118
return nil
115119
}
116120

@@ -596,6 +600,24 @@ func (this *Inspector) applyColumnTypes(databaseName, tableName string, columnsL
596600
return err
597601
}
598602

603+
// getAutoIncrementValue get's the original table's AUTO_INCREMENT value, if exists (0 value if not exists)
604+
func (this *Inspector) getAutoIncrementValue(tableName string) (autoIncrement uint64, err error) {
605+
query := `
606+
SELECT
607+
AUTO_INCREMENT
608+
FROM INFORMATION_SCHEMA.TABLES
609+
WHERE
610+
TABLES.TABLE_SCHEMA = ?
611+
AND TABLES.TABLE_NAME = ?
612+
AND AUTO_INCREMENT IS NOT NULL
613+
`
614+
err = sqlutils.QueryRowsMap(this.db, query, func(m sqlutils.RowMap) error {
615+
autoIncrement = m.GetUint64("AUTO_INCREMENT")
616+
return nil
617+
}, this.migrationContext.DatabaseName, tableName)
618+
return autoIncrement, err
619+
}
620+
599621
// getCandidateUniqueKeys investigates a table and returns the list of unique keys
600622
// candidate for chunking
601623
func (this *Inspector) getCandidateUniqueKeys(tableName string) (uniqueKeys [](*sql.UniqueKey), err error) {

‎go/logic/migrator.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1100,6 +1100,14 @@ func (this *Migrator) initiateApplier() error {
11001100
return err
11011101
}
11021102

1103+
if this.migrationContext.OriginalTableAutoIncrement > 0 && !this.parser.IsAutoIncrementDefined() {
1104+
// Original table has AUTO_INCREMENT value and the -alter statement does not indicate any override,
1105+
// so we should copy AUTO_INCREMENT value onto our ghost table.
1106+
if err := this.applier.AlterGhostAutoIncrement(); err != nil {
1107+
this.migrationContext.Log.Errorf("Unable to ALTER ghost table AUTO_INCREMENT value, see further error details. Bailing out")
1108+
return err
1109+
}
1110+
}
11031111
this.applier.WriteChangelogState(string(GhostTableMigrated))
11041112
go this.applier.InitiateHeartbeat()
11051113
return nil

‎go/sql/parser.go‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ var (
1616
renameColumnRegexp = regexp.MustCompile(`(?i)\bchange\s+(column\s+|)([\S]+)\s+([\S]+)\s+`)
1717
dropColumnRegexp = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`)
1818
renameTableRegexp = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`)
19+
autoIncrementRegexp = regexp.MustCompile(`(?i)\bauto_increment[\s]*=[\s]*([0-9]+)`)
1920
alterTableExplicitSchemaTableRegexps = []*regexp.Regexp{
2021
// ALTER TABLE `scm`.`tbl` something
2122
regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
@@ -35,9 +36,10 @@ var (
3536
)
3637

3738
type AlterTableParser struct {
38-
columnRenameMap map[string]string
39-
droppedColumns map[string]bool
40-
isRenameTable bool
39+
columnRenameMap map[string]string
40+
droppedColumns map[string]bool
41+
isRenameTable bool
42+
isAutoIncrementDefined bool
4143

4244
alterStatementOptions string
4345
alterTokens []string
@@ -122,6 +124,12 @@ func (this *AlterTableParser) parseAlterToken(alterToken string) (err error) {
122124
this.isRenameTable = true
123125
}
124126
}
127+
{
128+
// auto_increment
129+
if autoIncrementRegexp.MatchString(alterToken) {
130+
this.isAutoIncrementDefined = true
131+
}
132+
}
125133
return nil
126134
}
127135

@@ -173,6 +181,11 @@ func (this *AlterTableParser) DroppedColumnsMap() map[string]bool {
173181
func (this *AlterTableParser) IsRenameTable() bool {
174182
return this.isRenameTable
175183
}
184+
185+
func (this *AlterTableParser) IsAutoIncrementDefined() bool {
186+
return this.isAutoIncrementDefined
187+
}
188+
176189
func (this *AlterTableParser) GetExplicitSchema() string {
177190
return this.explicitSchema
178191
}

‎go/sql/parser_test.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ func TestParseAlterStatement(t *testing.T) {
2424
test.S(t).ExpectNil(err)
2525
test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
2626
test.S(t).ExpectFalse(parser.HasNonTrivialRenames())
27+
test.S(t).ExpectFalse(parser.IsAutoIncrementDefined())
2728
}
2829

2930
func TestParseAlterStatementTrivialRename(t *testing.T) {
@@ -33,17 +34,39 @@ func TestParseAlterStatementTrivialRename(t *testing.T) {
3334
test.S(t).ExpectNil(err)
3435
test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
3536
test.S(t).ExpectFalse(parser.HasNonTrivialRenames())
37+
test.S(t).ExpectFalse(parser.IsAutoIncrementDefined())
3638
test.S(t).ExpectEquals(len(parser.columnRenameMap), 1)
3739
test.S(t).ExpectEquals(parser.columnRenameMap["ts"], "ts")
3840
}
3941

42+
func TestParseAlterStatementWithAutoIncrement(t *testing.T) {
43+
44+
statements := []string{
45+
"auto_increment=7",
46+
"auto_increment = 7",
47+
"AUTO_INCREMENT = 71",
48+
"add column t int, change ts ts timestamp, auto_increment=7 engine=innodb",
49+
"add column t int, change ts ts timestamp, auto_increment =7 engine=innodb",
50+
"add column t int, change ts ts timestamp, AUTO_INCREMENT = 7 engine=innodb",
51+
"add column t int, change ts ts timestamp, engine=innodb auto_increment=73425",
52+
}
53+
for _, statement := range statements {
54+
parser := NewAlterTableParser()
55+
err := parser.ParseAlterStatement(statement)
56+
test.S(t).ExpectNil(err)
57+
test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
58+
test.S(t).ExpectTrue(parser.IsAutoIncrementDefined())
59+
}
60+
}
61+
4062
func TestParseAlterStatementTrivialRenames(t *testing.T) {
4163
statement := "add column t int, change ts ts timestamp, CHANGE f `f` float, engine=innodb"
4264
parser := NewAlterTableParser()
4365
err := parser.ParseAlterStatement(statement)
4466
test.S(t).ExpectNil(err)
4567
test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
4668
test.S(t).ExpectFalse(parser.HasNonTrivialRenames())
69+
test.S(t).ExpectFalse(parser.IsAutoIncrementDefined())
4770
test.S(t).ExpectEquals(len(parser.columnRenameMap), 2)
4871
test.S(t).ExpectEquals(parser.columnRenameMap["ts"], "ts")
4972
test.S(t).ExpectEquals(parser.columnRenameMap["f"], "f")
@@ -64,6 +87,7 @@ func TestParseAlterStatementNonTrivial(t *testing.T) {
6487
parser := NewAlterTableParser()
6588
err := parser.ParseAlterStatement(statement)
6689
test.S(t).ExpectNil(err)
90+
test.S(t).ExpectFalse(parser.IsAutoIncrementDefined())
6791
test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
6892
renames := parser.GetNonTrivialRenames()
6993
test.S(t).ExpectEquals(len(renames), 2)

‎localtests/autoinc-copy-deletes-user-defined/create.sql‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
drop event if exists gh_ost_test;
2+
3+
drop table if exists gh_ost_test;
4+
create table gh_ost_test (
5+
id int auto_increment,
6+
i int not null,
7+
primary key(id)
8+
) auto_increment=1;
9+
10+
insert into gh_ost_test values (NULL, 11);
11+
insert into gh_ost_test values (NULL, 13);
12+
insert into gh_ost_test values (NULL, 17);
13+
insert into gh_ost_test values (NULL, 23);
14+
insert into gh_ost_test values (NULL, 29);
15+
insert into gh_ost_test values (NULL, 31);
16+
insert into gh_ost_test values (NULL, 37);
17+
delete from gh_ost_test where id>=5;

‎localtests/autoinc-copy-deletes-user-defined/expect_table_structure‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
AUTO_INCREMENT=7

‎localtests/autoinc-copy-deletes-user-defined/extra_args‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
--alter='AUTO_INCREMENT=7'

‎localtests/autoinc-copy-deletes/create.sql‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
drop event if exists gh_ost_test;
2+
3+
drop table if exists gh_ost_test;
4+
create table gh_ost_test (
5+
id int auto_increment,
6+
i int not null,
7+
primary key(id)
8+
) auto_increment=1;
9+
10+
insert into gh_ost_test values (NULL, 11);
11+
insert into gh_ost_test values (NULL, 13);
12+
insert into gh_ost_test values (NULL, 17);
13+
insert into gh_ost_test values (NULL, 23);
14+
insert into gh_ost_test values (NULL, 29);
15+
insert into gh_ost_test values (NULL, 31);
16+
insert into gh_ost_test values (NULL, 37);
17+
delete from gh_ost_test where id>=5;

0 commit comments

Comments
 (0)