Skip to content

Commit 0530b09

Browse files
author
Shlomi Noach
authored
Merge branch 'master' into nm-refactor-migration-context
2 parents 538833e + 83416fb commit 0530b09

6 files changed

Lines changed: 174 additions & 63 deletions

File tree

‎.travis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# http://docs.travis-ci.com/user/languages/go/
22
language: go
33

4-
go: 1.8
4+
go: 1.9
55

66
os:
77
- linux

‎build.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,11 @@ function build {
1010
GOOS=$3
1111
GOARCH=$4
1212

13+
if [[ $(go version | egrep "go1[.][012345678]") ]]; then
14+
echo "go version is too low. Must use 1.9 or above"
15+
exit 1
16+
fi
17+
1318
echo "Building ${osname} binary"
1419
export GOOS
1520
export GOARCH
@@ -27,7 +32,6 @@ buildpath=/tmp/gh-ost
2732
target=gh-ost
2833
timestamp=$(date "+%Y%m%d%H%M%S")
2934
ldflags="-X main.AppVersion=${RELEASE_VERSION}"
30-
export GO15VENDOREXPERIMENT=1
3135

3236
mkdir -p ${buildpath}
3337
build macOS osx darwin amd64

‎go/logic/applier.go‎

Lines changed: 99 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,28 @@ const (
2424
atomicCutOverMagicHint = "ghost-cut-over-sentry"
2525
)
2626

27+
type dmlBuildResult struct {
28+
query string
29+
args []interface{}
30+
rowsDelta int64
31+
err error
32+
}
33+
34+
func newDmlBuildResult(query string, args []interface{}, rowsDelta int64, err error) *dmlBuildResult {
35+
return &dmlBuildResult{
36+
query: query,
37+
args: args,
38+
rowsDelta: rowsDelta,
39+
err: err,
40+
}
41+
}
42+
43+
func newDmlBuildResultError(err error) *dmlBuildResult {
44+
return &dmlBuildResult{
45+
err: err,
46+
}
47+
}
48+
2749
// Applier connects and writes the the applier-server, which is the server where migration
2850
// happens. This is typically the master, but could be a replica when `--test-on-replica` or
2951
// `--execute-on-replica` are given.
@@ -904,79 +926,103 @@ func (this *Applier) ShowStatusVariable(variableName string) (result int64, err
904926
return result, nil
905927
}
906928

929+
// updateModifiesUniqueKeyColumns checks whether a UPDATE DML event actually
930+
// modifies values of the migration's unique key (the iterated key). This will call
931+
// for special handling.
932+
func (this *Applier) updateModifiesUniqueKeyColumns(dmlEvent *binlog.BinlogDMLEvent) (modifiedColumn string, isModified bool) {
933+
for _, column := range this.migrationContext.UniqueKey.Columns.Columns() {
934+
tableOrdinal := this.migrationContext.OriginalTableColumns.Ordinals[column.Name]
935+
whereColumnValue := dmlEvent.WhereColumnValues.AbstractValues()[tableOrdinal]
936+
newColumnValue := dmlEvent.NewColumnValues.AbstractValues()[tableOrdinal]
937+
if newColumnValue != whereColumnValue {
938+
return column.Name, true
939+
}
940+
}
941+
return "", false
942+
}
943+
907944
// buildDMLEventQuery creates a query to operate on the ghost table, based on an intercepted binlog
908945
// event entry on the original table.
909-
func (this *Applier) buildDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) (query string, args []interface{}, rowsDelta int64, err error) {
946+
func (this *Applier) buildDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) (results [](*dmlBuildResult)) {
910947
switch dmlEvent.DML {
911948
case binlog.DeleteDML:
912949
{
913950
query, uniqueKeyArgs, err := sql.BuildDMLDeleteQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, &this.migrationContext.UniqueKey.Columns, dmlEvent.WhereColumnValues.AbstractValues())
914-
return query, uniqueKeyArgs, -1, err
951+
return append(results, newDmlBuildResult(query, uniqueKeyArgs, -1, err))
915952
}
916953
case binlog.InsertDML:
917954
{
918955
query, sharedArgs, err := sql.BuildDMLInsertQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns, dmlEvent.NewColumnValues.AbstractValues())
919-
return query, sharedArgs, 1, err
956+
return append(results, newDmlBuildResult(query, sharedArgs, 1, err))
920957
}
921958
case binlog.UpdateDML:
922959
{
960+
if _, isModified := this.updateModifiesUniqueKeyColumns(dmlEvent); isModified {
961+
dmlEvent.DML = binlog.DeleteDML
962+
results = append(results, this.buildDMLEventQuery(dmlEvent)...)
963+
dmlEvent.DML = binlog.InsertDML
964+
results = append(results, this.buildDMLEventQuery(dmlEvent)...)
965+
return results
966+
}
923967
query, sharedArgs, uniqueKeyArgs, err := sql.BuildDMLUpdateQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns, &this.migrationContext.UniqueKey.Columns, dmlEvent.NewColumnValues.AbstractValues(), dmlEvent.WhereColumnValues.AbstractValues())
968+
args := sqlutils.Args()
924969
args = append(args, sharedArgs...)
925970
args = append(args, uniqueKeyArgs...)
926-
return query, args, 0, err
971+
return append(results, newDmlBuildResult(query, args, 0, err))
927972
}
928973
}
929-
return "", args, 0, fmt.Errorf("Unknown dml event type: %+v", dmlEvent.DML)
974+
return append(results, newDmlBuildResultError(fmt.Errorf("Unknown dml event type: %+v", dmlEvent.DML)))
930975
}
931976

932977
// ApplyDMLEventQuery writes an entry to the ghost table, in response to an intercepted
933978
// original-table binlog event
934979
func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error {
935-
query, args, rowDelta, err := this.buildDMLEventQuery(dmlEvent)
936-
if err != nil {
937-
return err
938-
}
939-
// TODO The below is in preparation for transactional writes on the ghost tables.
940-
// Such writes would be, for example:
941-
// - prepended with sql_mode setup
942-
// - prepended with time zone setup
943-
// - prepended with SET SQL_LOG_BIN=0
944-
// - prepended with SET FK_CHECKS=0
945-
// etc.
946-
//
947-
// a known problem: https://github.com/golang/go/issues/9373 -- bitint unsigned values, not supported in database/sql
948-
// is solved by silently converting unsigned bigints to string values.
949-
//
950-
951-
err = func() error {
952-
tx, err := this.db.Begin()
953-
if err != nil {
954-
return err
980+
for _, buildResult := range this.buildDMLEventQuery(dmlEvent) {
981+
if buildResult.err != nil {
982+
return buildResult.err
955983
}
956-
sessionQuery := `SET
984+
// TODO The below is in preparation for transactional writes on the ghost tables.
985+
// Such writes would be, for example:
986+
// - prepended with sql_mode setup
987+
// - prepended with time zone setup
988+
// - prepended with SET SQL_LOG_BIN=0
989+
// - prepended with SET FK_CHECKS=0
990+
// etc.
991+
//
992+
// a known problem: https://github.com/golang/go/issues/9373 -- bitint unsigned values, not supported in database/sql
993+
// is solved by silently converting unsigned bigints to string values.
994+
//
995+
996+
err := func() error {
997+
tx, err := this.db.Begin()
998+
if err != nil {
999+
return err
1000+
}
1001+
sessionQuery := `SET
9571002
SESSION time_zone = '+00:00',
9581003
sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')
9591004
`
960-
if _, err := tx.Exec(sessionQuery); err != nil {
961-
return err
962-
}
963-
if _, err := tx.Exec(query, args...); err != nil {
964-
return err
1005+
if _, err := tx.Exec(sessionQuery); err != nil {
1006+
return err
1007+
}
1008+
if _, err := tx.Exec(buildResult.query, buildResult.args...); err != nil {
1009+
return err
1010+
}
1011+
if err := tx.Commit(); err != nil {
1012+
return err
1013+
}
1014+
return nil
1015+
}()
1016+
1017+
if err != nil {
1018+
err = fmt.Errorf("%s; query=%s; args=%+v", err.Error(), buildResult.query, buildResult.args)
1019+
return log.Errore(err)
9651020
}
966-
if err := tx.Commit(); err != nil {
967-
return err
1021+
// no error
1022+
atomic.AddInt64(&this.migrationContext.TotalDMLEventsApplied, 1)
1023+
if this.migrationContext.CountTableRows {
1024+
atomic.AddInt64(&this.migrationContext.RowsDeltaEstimate, buildResult.rowsDelta)
9681025
}
969-
return nil
970-
}()
971-
972-
if err != nil {
973-
err = fmt.Errorf("%s; query=%s; args=%+v", err.Error(), query, args)
974-
return log.Errore(err)
975-
}
976-
// no error
977-
atomic.AddInt64(&this.migrationContext.TotalDMLEventsApplied, 1)
978-
if this.migrationContext.CountTableRows {
979-
atomic.AddInt64(&this.migrationContext.RowsDeltaEstimate, rowDelta)
9801026
}
9811027
return nil
9821028
}
@@ -1005,15 +1051,16 @@ func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent))
10051051
return rollback(err)
10061052
}
10071053
for _, dmlEvent := range dmlEvents {
1008-
query, args, rowDelta, err := this.buildDMLEventQuery(dmlEvent)
1009-
if err != nil {
1010-
return rollback(err)
1011-
}
1012-
if _, err := tx.Exec(query, args...); err != nil {
1013-
err = fmt.Errorf("%s; query=%s; args=%+v", err.Error(), query, args)
1014-
return rollback(err)
1054+
for _, buildResult := range this.buildDMLEventQuery(dmlEvent) {
1055+
if buildResult.err != nil {
1056+
return rollback(buildResult.err)
1057+
}
1058+
if _, err := tx.Exec(buildResult.query, buildResult.args...); err != nil {
1059+
err = fmt.Errorf("%s; query=%s; args=%+v", err.Error(), buildResult.query, buildResult.args)
1060+
return rollback(err)
1061+
}
1062+
totalDelta += buildResult.rowsDelta
10151063
}
1016-
totalDelta += rowDelta
10171064
}
10181065
if err := tx.Commit(); err != nil {
10191066
return err

‎localtests/fail-update-pk-column/create.sql‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
drop table if exists gh_ost_test;
2+
create table gh_ost_test (
3+
id int auto_increment,
4+
i int not null,
5+
primary key(id)
6+
) auto_increment=1;
7+
8+
insert into gh_ost_test values (null, 101);
9+
insert into gh_ost_test values (null, 102);
10+
insert into gh_ost_test values (null, 103);
11+
insert into gh_ost_test values (null, 104);
12+
insert into gh_ost_test values (null, 105);
13+
insert into gh_ost_test values (null, 106);
14+
insert into gh_ost_test values (null, 107);
15+
insert into gh_ost_test values (null, 108);
16+
insert into gh_ost_test values (null, 109);
17+
insert into gh_ost_test values (null, 110);
18+
insert into gh_ost_test values (null, 111);
19+
insert into gh_ost_test values (null, 112);
20+
insert into gh_ost_test values (null, 113);
21+
insert into gh_ost_test values (null, 114);
22+
insert into gh_ost_test values (null, 115);
23+
insert into gh_ost_test values (null, 116);
24+
insert into gh_ost_test values (null, 117);
25+
insert into gh_ost_test values (null, 118);
26+
insert into gh_ost_test values (null, 119);
27+
insert into gh_ost_test values (null, 120);
28+
insert into gh_ost_test values (null, 121);
29+
insert into gh_ost_test values (null, 122);
30+
insert into gh_ost_test values (null, 123);
31+
insert into gh_ost_test values (null, 124);
32+
insert into gh_ost_test values (null, 125);
33+
insert into gh_ost_test values (null, 126);
34+
insert into gh_ost_test values (null, 127);
35+
insert into gh_ost_test values (null, 128);
36+
insert into gh_ost_test values (null, 129);
37+
38+
drop event if exists gh_ost_test;
39+
delimiter ;;
40+
create event gh_ost_test
41+
on schedule every 1 second
42+
starts current_timestamp + interval 3 second
43+
ends current_timestamp + interval 60 second
44+
on completion not preserve
45+
enable
46+
do
47+
begin
48+
update gh_ost_test set id=-2 where id=21;
49+
update gh_ost_test set id=55 where id=22;
50+
update gh_ost_test set id=23 where id=23;
51+
update gh_ost_test set i=5024 where id=24;
52+
end ;;

‎localtests/test.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,12 @@ test_single() {
170170

171171
build_binary() {
172172
echo "Building"
173+
rm -f $ghost_binary
173174
go build -o $ghost_binary go/cmd/gh-ost/main.go
175+
if [ $? -ne 0 ] ; then
176+
echo "Build failure"
177+
exit 1
178+
fi
174179
}
175180

176181
test_all() {

‎script/ensure-go-installed‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,20 @@
11
#!/bin/bash
22

3-
GO_VERSION=go1.7
3+
PREFERRED_GO_VERSION=go1.9.2
4+
SUPPORTED_GO_VERSIONS='go1.[89]'
45

5-
GO_PKG_DARWIN=${GO_VERSION}.darwin-amd64.pkg
6-
GO_PKG_DARWIN_SHA=e7089843bc7148ffcc147759985b213604d22bb9fd19bd930b515aa981bf1b22
6+
GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg
7+
GO_PKG_DARWIN_SHA=73fd5840d55f5566d8db6c0ffdd187577e8ebe650c783f68bd27cbf95bde6743
78

8-
GO_PKG_LINUX=${GO_VERSION}.linux-amd64.tar.gz
9-
GO_PKG_LINUX_SHA=702ad90f705365227e902b42d91dd1a40e48ca7f67a2f4b2fd052aaa4295cd95
9+
GO_PKG_LINUX=${PREFERRED_GO_VERSION}.linux-amd64.tar.gz
10+
GO_PKG_LINUX_SHA=de874549d9a8d8d8062be05808509c09a88a248e77ec14eb77453530829ac02b
1011

1112
export ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
1213
cd $ROOTDIR
1314

1415
# If Go isn't installed globally, setup environment variables for local install.
15-
if [ -z "$(which go)" ] || [ -z "$(go version | grep $GO_VERSION)" ]; then
16-
GODIR="$ROOTDIR/.vendor/go17"
16+
if [ -z "$(which go)" ] || [ -z "$(go version | grep "$SUPPORTED_GO_VERSIONS")" ]; then
17+
GODIR="$ROOTDIR/.vendor/go19"
1718

1819
if [ $(uname -s) = "Darwin" ]; then
1920
export GOROOT="$GODIR/usr/local/go"
@@ -25,7 +26,7 @@ if [ -z "$(which go)" ] || [ -z "$(go version | grep $GO_VERSION)" ]; then
2526
fi
2627

2728
# Check if local install exists, and install otherwise.
28-
if [ -z "$(which go)" ] || [ -z "$(go version | grep $GO_VERSION)" ]; then
29+
if [ -z "$(which go)" ] || [ -z "$(go version | grep "$SUPPORTED_GO_VERSIONS")" ]; then
2930
[ -d "$GODIR" ] && rm -rf $GODIR
3031
mkdir -p "$GODIR"
3132
cd "$GODIR";
@@ -42,7 +43,9 @@ if [ -z "$(which go)" ] || [ -z "$(go version | grep $GO_VERSION)" ]; then
4243
fi
4344

4445
# Prove we did something right
45-
echo "$GO_VERSION installed in $GODIR: Go Binary: $(which go)"
46+
echo "$(go version) installed in $GODIR: Go Binary: $(which go)"
47+
else
48+
echo "$(go version) found in $GODIR: Go Binary: $(which go)"
4649
fi
4750

4851
cd $ROOTDIR

0 commit comments

Comments
 (0)