Merge branch 'master' into incorrect-datetime-1970

Author: Shlomi Noach

.github/PULL_REQUEST_TEMPLATE.md

@@ -11,7 +11,7 @@ Related issue: https://github.com/github/gh-ost/issues/0123456789
 
 ### Description
 
-This PR [briefly explain what is does]
+This PR [briefly explain what it does]
 
 > In case this PR introduced Go code changes:
 

.travis.yml

@@ -8,13 +8,22 @@ os:
 
 env:
 - MYSQL_USER=root
+- CURRENT_CI_ENV=travis
+
+addons:
+  apt:
+    packages:
+    - git
+    - numactl
+    - libaio1
 
 before_install:
   - mysql -e 'CREATE DATABASE IF NOT EXISTS test;'
 
 install: true
 
-script: script/cibuild
+script:
+  - script/cibuild
 
 notifications:
   email: false

README.md

@@ -94,7 +94,7 @@ Please see [Coding gh-ost](doc/coding-ghost.md) for a guide to getting started d
 
 [Download latest release here](https://github.com/github/gh-ost/releases/latest)
 
-`gh-ost` is a Go project; it is built with Go `1.8` (though `1.7` should work as well). To build on your own, use either:
+`gh-ost` is a Go project; it is built with Go `1.9` and above. To build on your own, use either:
 - [script/build](https://github.com/github/gh-ost/blob/master/script/build) - this is the same build script used by CI hence the authoritative; artifact is `./bin/gh-ost` binary.
 - [build.sh](https://github.com/github/gh-ost/blob/master/build.sh) for building `tar.gz` artifacts in `/tmp/gh-ost`
 
@@ -107,3 +107,5 @@ Generally speaking, `master` branch is stable, but only [releases](https://githu
 - [@ggunson](https://github.com/ggunson)
 - [@tomkrouper](https://github.com/tomkrouper)
 - [@shlomi-noach](https://github.com/shlomi-noach)
+- [@jessbreckenridge](https://github.com/jessbreckenridge)
+- [@gtowey](https://github.com/gtowey)

RELEASE_VERSION

@@ -1 +1 @@
-1.0.42
+1.0.45

build.sh

@@ -10,7 +10,7 @@ function build {
     GOOS=$3
     GOARCH=$4
 
-    if [[ $(go version | egrep "go1[.][012345678]") ]]; then
+    if ! go version | egrep -q 'go(1[.]9|1[.]1[0-9])' ; then
       echo "go version is too low. Must use 1.9 or above"
       exit 1
     fi

doc/rds.md

@@ -1,4 +1,4 @@
-`gh-ost` has been updated to work with Amazon RDS however due to GitHub not relying using AWS for databases, this documentation is community driven so if you find a bug please [open an issue][new_issue]!
+`gh-ost` has been updated to work with Amazon RDS however due to GitHub not using AWS for databases, this documentation is community driven so if you find a bug please [open an issue][new_issue]!
 
 # Amazon RDS
 

doc/requirements-and-limitations.md

@@ -52,3 +52,5 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th
 - If you have en `enum` field as part of your migration key (typically the `PRIMARY KEY`), migration performance will be degraded and potentially bad. [Read more](https://github.com/github/gh-ost/pull/277#issuecomment-254811520)
 
 - Migrating a `FEDERATED` table is unsupported and is irrelevant to the problem `gh-ost` tackles.
+
+- `ALTER TABLE ... RENAME TO some_other_name` is not supported (and you shouldn't use `gh-ost` for such a trivial operation).

go/base/context.go

@@ -28,23 +28,23 @@ type RowsEstimateMethod string
 
 const (
 	TableStatusRowsEstimate RowsEstimateMethod = "TableStatusRowsEstimate"
-	ExplainRowsEstimate                        = "ExplainRowsEstimate"
-	CountRowsEstimate                          = "CountRowsEstimate"
+	ExplainRowsEstimate     RowsEstimateMethod = "ExplainRowsEstimate"
+	CountRowsEstimate       RowsEstimateMethod = "CountRowsEstimate"
 )
 
 type CutOver int
 
 const (
-	CutOverAtomic  CutOver = iota
-	CutOverTwoStep         = iota
+	CutOverAtomic CutOver = iota
+	CutOverTwoStep
 )
 
 type ThrottleReasonHint string
 
 const (
 	NoThrottleReasonHint                 ThrottleReasonHint = "NoThrottleReasonHint"
-	UserCommandThrottleReasonHint                           = "UserCommandThrottleReasonHint"
-	LeavingHibernationThrottleReasonHint                    = "LeavingHibernationThrottleReasonHint"
+	UserCommandThrottleReasonHint        ThrottleReasonHint = "UserCommandThrottleReasonHint"
+	LeavingHibernationThrottleReasonHint ThrottleReasonHint = "LeavingHibernationThrottleReasonHint"
 )
 
 const (
@@ -91,6 +91,7 @@ type MigrationContext struct {
 	SkipRenamedColumns       bool
 	IsTungsten               bool
 	DiscardForeignKeys       bool
+	AliyunRDS                bool
 
 	config            ContextConfig
 	configMutex       *sync.Mutex
@@ -118,6 +119,8 @@ type MigrationContext struct {
 	CriticalLoadHibernateSeconds        int64
 	PostponeCutOverFlagFile             string
 	CutOverLockTimeoutSeconds           int64
+	CutOverExponentialBackoff           bool
+	ExponentialBackoffMaxInterval       int64
 	ForceNamedCutOverCommand            bool
 	PanicFlagFile                       string
 	HooksPath                           string
@@ -341,6 +344,14 @@ func (this *MigrationContext) SetCutOverLockTimeoutSeconds(timeoutSeconds int64)
 	return nil
 }
 
+func (this *MigrationContext) SetExponentialBackoffMaxInterval(intervalSeconds int64) error {
+	if intervalSeconds < 2 {
+		return fmt.Errorf("Minimal maximum interval is 2sec. Timeout remains at %d", this.ExponentialBackoffMaxInterval)
+	}
+	this.ExponentialBackoffMaxInterval = intervalSeconds
+	return nil
+}
+
 func (this *MigrationContext) SetDefaultNumRetries(retries int64) {
 	this.throttleMutex.Lock()
 	defer this.throttleMutex.Unlock()

go/base/utils.go

@@ -64,17 +64,26 @@ func StringContainsAll(s string, substrings ...string) bool {
 	return nonEmptyStringsFound
 }
 
-func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig) (string, error) {
-	query := `select @@global.port, @@global.version`
+func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, migrationContext *MigrationContext) (string, error) {
+	versionQuery := `select @@global.version`
 	var port, extraPort int
 	var version string
-	if err := db.QueryRow(query).Scan(&port, &version); err != nil {
+	if err := db.QueryRow(versionQuery).Scan(&version); err != nil {
 		return "", err
 	}
 	extraPortQuery := `select @@global.extra_port`
 	if err := db.QueryRow(extraPortQuery).Scan(&extraPort); err != nil {
 		// swallow this error. not all servers support extra_port
 	}
+	// AliyunRDS set users port to "NULL", replace it by gh-ost param
+	if migrationContext.AliyunRDS {
+		port = connectionConfig.Key.Port
+	} else {
+		portQuery := `select @@global.port`
+		if err := db.QueryRow(portQuery).Scan(&port); err != nil {
+			return "", err
+		}
+	}
 
 	if connectionConfig.Key.Port == port || (extraPort > 0 && connectionConfig.Key.Port == extraPort) {
 		log.Infof("connection validated on %+v", connectionConfig.Key)

go/binlog/binlog_dml_event.go

@@ -7,17 +7,18 @@ package binlog
 
 import (
 	"fmt"
-	"github.com/github/gh-ost/go/sql"
 	"strings"
+
+	"github.com/github/gh-ost/go/sql"
 )
 
 type EventDML string
 
 const (
 	NotDML    EventDML = "NoDML"
-	InsertDML          = "Insert"
-	UpdateDML          = "Update"
-	DeleteDML          = "Delete"
+	InsertDML EventDML = "Insert"
+	UpdateDML EventDML = "Update"
+	DeleteDML EventDML = "Delete"
 )
 
 func ToEventDML(description string) EventDML {