Skip to content

Commit 5e37110

Browse files
timvaillancourttimvaillancourt
committed
merge tar-xz-dbdeployer-upgrade
1 parent a25f63a commit 5e37110

7 files changed

Lines changed: 62 additions & 15 deletions

File tree

‎build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ function build {
4040
builddir=$(setuptree)
4141
cp $buildpath/$target $builddir/gh-ost/usr/bin
4242
cd $buildpath
43-
fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach <shlomi-noach+gh-ost-deb@github.com>' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm .
43+
fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach <shlomi-noach+gh-ost-deb@github.com>' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm --rpm-rpmbuild-define "_build_id_links none" .
4444
fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach <shlomi-noach+gh-ost-deb@github.com>' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files .
4545
fi
4646
}

‎doc/hooks.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ The following variables are available on all hooks:
6666
- `GH_OST_ESTIMATED_ROWS` - estimated total rows in table
6767
- `GH_OST_COPIED_ROWS` - number of rows copied by `gh-ost`
6868
- `GH_OST_INSPECTED_LAG` - lag in seconds (floating point) of inspected server
69+
- `GH_OST_HEARTBEAT_LAG` - lag in seconds (floating point) of heartbeat
6970
- `GH_OST_PROGRESS` - progress pct ([0..100], floating point) of migration
7071
- `GH_OST_MIGRATED_HOST`
7172
- `GH_OST_INSPECTED_HOST`

‎go/base/context.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,8 @@ type MigrationContext struct {
178178
RenameTablesEndTime time.Time
179179
pointOfInterestTime time.Time
180180
pointOfInterestTimeMutex *sync.Mutex
181+
lastHeartbeatOnChangelogTime time.Time
182+
lastHeartbeatOnChangelogMutex *sync.Mutex
181183
CurrentLag int64
182184
currentProgress uint64
183185
ThrottleHTTPStatusCode int64
@@ -271,6 +273,7 @@ func NewMigrationContext() *MigrationContext {
271273
throttleControlReplicaKeys: mysql.NewInstanceKeyMap(),
272274
configMutex: &sync.Mutex{},
273275
pointOfInterestTimeMutex: &sync.Mutex{},
276+
lastHeartbeatOnChangelogMutex: &sync.Mutex{},
274277
ColumnRenameMap: make(map[string]string),
275278
PanicAbort: make(chan error),
276279
Log: NewDefaultLogger(),
@@ -454,6 +457,10 @@ func (this *MigrationContext) MarkRowCopyEndTime() {
454457
this.RowCopyEndTime = time.Now()
455458
}
456459

460+
func (this *MigrationContext) TimeSinceLastHeartbeatOnChangelog() time.Duration {
461+
return time.Since(this.GetLastHeartbeatOnChangelogTime())
462+
}
463+
457464
func (this *MigrationContext) GetCurrentLagDuration() time.Duration {
458465
return time.Duration(atomic.LoadInt64(&this.CurrentLag))
459466
}
@@ -493,6 +500,20 @@ func (this *MigrationContext) TimeSincePointOfInterest() time.Duration {
493500
return time.Since(this.pointOfInterestTime)
494501
}
495502

503+
func (this *MigrationContext) SetLastHeartbeatOnChangelogTime(t time.Time) {
504+
this.lastHeartbeatOnChangelogMutex.Lock()
505+
defer this.lastHeartbeatOnChangelogMutex.Unlock()
506+
507+
this.lastHeartbeatOnChangelogTime = t
508+
}
509+
510+
func (this *MigrationContext) GetLastHeartbeatOnChangelogTime() time.Time {
511+
this.lastHeartbeatOnChangelogMutex.Lock()
512+
defer this.lastHeartbeatOnChangelogMutex.Unlock()
513+
514+
return this.lastHeartbeatOnChangelogTime
515+
}
516+
496517
func (this *MigrationContext) SetHeartbeatIntervalMilliseconds(heartbeatIntervalMilliseconds int64) {
497518
if heartbeatIntervalMilliseconds < 100 {
498519
heartbeatIntervalMilliseconds = 100

‎go/logic/hooks.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [
6464
env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.GetInspectorHostname()))
6565
env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", this.migrationContext.Hostname))
6666
env = append(env, fmt.Sprintf("GH_OST_INSPECTED_LAG=%f", this.migrationContext.GetCurrentLagDuration().Seconds()))
67+
env = append(env, fmt.Sprintf("GH_OST_HEARTBEAT_LAG=%f", this.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds()))
6768
env = append(env, fmt.Sprintf("GH_OST_PROGRESS=%f", this.migrationContext.GetProgressPct()))
6869
env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage))
6970
env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_OWNER=%s", this.migrationContext.HooksHintOwner))

‎go/logic/inspect.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -528,7 +528,7 @@ func (this *Inspector) CountTableRows() error {
528528

529529
this.migrationContext.Log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while")
530530

531-
query := fmt.Sprintf(`select /* gh-ost */ count(*) as rows from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
531+
query := fmt.Sprintf(`select /* gh-ost */ count(*) as count_rows from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
532532
var rowsEstimate int64
533533
if err := this.db.QueryRow(query).Scan(&rowsEstimate); err != nil {
534534
return err

‎go/logic/migrator.go‎

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -207,12 +207,20 @@ func (this *Migrator) canStopStreaming() bool {
207207
return atomic.LoadInt64(&this.migrationContext.CutOverCompleteFlag) != 0
208208
}
209209

210-
// onChangelogStateEvent is called when a binlog event operation on the changelog table is intercepted.
211-
func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (err error) {
210+
// onChangelogEvent is called when a binlog event operation on the changelog table is intercepted.
211+
func (this *Migrator) onChangelogEvent(dmlEvent *binlog.BinlogDMLEvent) (err error) {
212212
// Hey, I created the changelog table, I know the type of columns it has!
213-
if hint := dmlEvent.NewColumnValues.StringColumn(2); hint != "state" {
213+
switch hint := dmlEvent.NewColumnValues.StringColumn(2); hint {
214+
case "state":
215+
return this.onChangelogStateEvent(dmlEvent)
216+
case "heartbeat":
217+
return this.onChangelogHeartbeatEvent(dmlEvent)
218+
default:
214219
return nil
215220
}
221+
}
222+
223+
func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (err error) {
216224
changelogStateString := dmlEvent.NewColumnValues.StringColumn(3)
217225
changelogState := ReadChangelogState(changelogStateString)
218226
this.migrationContext.Log.Infof("Intercepted changelog state %s", changelogState)
@@ -245,6 +253,18 @@ func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (er
245253
return nil
246254
}
247255

256+
func (this *Migrator) onChangelogHeartbeatEvent(dmlEvent *binlog.BinlogDMLEvent) (err error) {
257+
changelogHeartbeatString := dmlEvent.NewColumnValues.StringColumn(3)
258+
259+
heartbeatTime, err := time.Parse(time.RFC3339Nano, changelogHeartbeatString)
260+
if err != nil {
261+
return this.migrationContext.Log.Errore(err)
262+
} else {
263+
this.migrationContext.SetLastHeartbeatOnChangelogTime(heartbeatTime)
264+
return nil
265+
}
266+
}
267+
248268
// listenOnPanicAbort aborts on abort request
249269
func (this *Migrator) listenOnPanicAbort() {
250270
err := <-this.migrationContext.PanicAbort
@@ -476,6 +496,13 @@ func (this *Migrator) cutOver() (err error) {
476496
this.migrationContext.Log.Debugf("checking for cut-over postpone")
477497
this.sleepWhileTrue(
478498
func() (bool, error) {
499+
heartbeatLag := this.migrationContext.TimeSinceLastHeartbeatOnChangelog()
500+
maxLagMillisecondsThrottle := time.Duration(atomic.LoadInt64(&this.migrationContext.MaxLagMillisecondsThrottleThreshold)) * time.Millisecond
501+
cutOverLockTimeout := time.Duration(this.migrationContext.CutOverLockTimeoutSeconds) * time.Second
502+
if heartbeatLag > maxLagMillisecondsThrottle || heartbeatLag > cutOverLockTimeout {
503+
this.migrationContext.Log.Debugf("current HeartbeatLag (%.2fs) is too high, it needs to be less than both --max-lag-millis (%.2fs) and --cut-over-lock-timeout-seconds (%.2fs) to continue", heartbeatLag.Seconds(), maxLagMillisecondsThrottle.Seconds(), cutOverLockTimeout.Seconds())
504+
return true, nil
505+
}
479506
if this.migrationContext.PostponeCutOverFlagFile == "" {
480507
return false, nil
481508
}
@@ -962,13 +989,14 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) {
962989

963990
currentBinlogCoordinates := *this.eventsStreamer.GetCurrentBinlogCoordinates()
964991

965-
status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %.2fs, State: %s; ETA: %s",
992+
status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %.2fs, HeartbeatLag: %.2fs, State: %s; ETA: %s",
966993
totalRowsCopied, rowsEstimate, progressPct,
967994
atomic.LoadInt64(&this.migrationContext.TotalDMLEventsApplied),
968995
len(this.applyEventsQueue), cap(this.applyEventsQueue),
969996
base.PrettifyDurationOutput(elapsedTime), base.PrettifyDurationOutput(this.migrationContext.ElapsedRowCopyTime()),
970997
currentBinlogCoordinates,
971998
this.migrationContext.GetCurrentLagDuration().Seconds(),
999+
this.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds(),
9721000
state,
9731001
eta,
9741002
)
@@ -995,7 +1023,7 @@ func (this *Migrator) initiateStreaming() error {
9951023
this.migrationContext.DatabaseName,
9961024
this.migrationContext.GetChangelogTableName(),
9971025
func(dmlEvent *binlog.BinlogDMLEvent) error {
998-
return this.onChangelogStateEvent(dmlEvent)
1026+
return this.onChangelogEvent(dmlEvent)
9991027
},
10001028
)
10011029

‎script/cibuild-gh-ost-replica-tests‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ whoami
77
fetch_ci_env() {
88
# Clone gh-ost-ci-env
99
# Only clone if not already running locally at latest commit
10-
remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git tar-xz-binaries | cut -f1)
10+
remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git HEAD | cut -f1)
1111
local_commit="unknown"
1212
[ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1)
1313

@@ -17,11 +17,6 @@ fetch_ci_env() {
1717
if [ "$remote_commit" != "$local_commit" ] ; then
1818
rm -rf ./gh-ost-ci-env
1919
git clone https://github.com/github/gh-ost-ci-env.git
20-
(
21-
cd gh-ost-ci-env
22-
git fetch origin tar-xz-binaries
23-
git checkout tar-xz-binaries
24-
)
2520
fi
2621
}
2722

@@ -46,12 +41,13 @@ test_mysql_version() {
4641
mkdir -p sandboxes
4742
rm -rf sandboxes/*
4843

49-
if echo "$mysql_version" | egrep "5[.]5[.]" ; then
44+
local mysql_version_num=${mysql_version#*-}
45+
if echo "$mysql_version_num" | egrep "5[.]5[.]" ; then
5046
gtid=""
5147
else
5248
gtid="--gtid"
5349
fi
54-
gh-ost-ci-env/bin/linux/dbdeployer deploy replication "$mysql_version" --nodes 2 --sandbox-binary ${PWD}/sandbox/binary --sandbox-home ${PWD}/sandboxes ${gtid} --my-cnf-options log_slave_updates --my-cnf-options log_bin --my-cnf-options binlog_format=ROW --sandbox-directory rsandbox
50+
gh-ost-ci-env/bin/linux/dbdeployer deploy replication "$mysql_version_num" --nodes 2 --sandbox-binary ${PWD}/sandbox/binary --sandbox-home ${PWD}/sandboxes ${gtid} --my-cnf-options log_slave_updates --my-cnf-options log_bin --my-cnf-options binlog_format=ROW --sandbox-directory rsandbox
5551

5652
sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-master
5753
echo 'sandboxes/rsandbox/m "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-master

0 commit comments

Comments
 (0)