Skip to content

Commit c9f2da8

Browse files
author
Shlomi Noach
authored
Merge branch 'master' into json-57-test
2 parents 7c38a50 + 7fa16df commit c9f2da8

4 files changed

Lines changed: 90 additions & 28 deletions

File tree

‎go/logic/applier.go‎

Lines changed: 32 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -398,35 +398,41 @@ func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange boo
398398
if this.migrationContext.MigrationIterationRangeMinValues == nil {
399399
this.migrationContext.MigrationIterationRangeMinValues = this.migrationContext.MigrationRangeMinValues
400400
}
401-
query, explodedArgs, err := sql.BuildUniqueKeyRangeEndPreparedQuery(
402-
this.migrationContext.DatabaseName,
403-
this.migrationContext.OriginalTableName,
404-
&this.migrationContext.UniqueKey.Columns,
405-
this.migrationContext.MigrationIterationRangeMinValues.AbstractValues(),
406-
this.migrationContext.MigrationRangeMaxValues.AbstractValues(),
407-
atomic.LoadInt64(&this.migrationContext.ChunkSize),
408-
this.migrationContext.GetIteration() == 0,
409-
fmt.Sprintf("iteration:%d", this.migrationContext.GetIteration()),
410-
)
411-
if err != nil {
412-
return hasFurtherRange, err
413-
}
414-
rows, err := this.db.Query(query, explodedArgs...)
415-
if err != nil {
416-
return hasFurtherRange, err
417-
}
418-
iterationRangeMaxValues := sql.NewColumnValues(this.migrationContext.UniqueKey.Len())
419-
for rows.Next() {
420-
if err = rows.Scan(iterationRangeMaxValues.ValuesPointers...); err != nil {
401+
for i := 0; i < 2; i++ {
402+
buildFunc := sql.BuildUniqueKeyRangeEndPreparedQueryViaOffset
403+
if i == 1 {
404+
buildFunc = sql.BuildUniqueKeyRangeEndPreparedQueryViaTemptable
405+
}
406+
query, explodedArgs, err := buildFunc(
407+
this.migrationContext.DatabaseName,
408+
this.migrationContext.OriginalTableName,
409+
&this.migrationContext.UniqueKey.Columns,
410+
this.migrationContext.MigrationIterationRangeMinValues.AbstractValues(),
411+
this.migrationContext.MigrationRangeMaxValues.AbstractValues(),
412+
atomic.LoadInt64(&this.migrationContext.ChunkSize),
413+
this.migrationContext.GetIteration() == 0,
414+
fmt.Sprintf("iteration:%d", this.migrationContext.GetIteration()),
415+
)
416+
if err != nil {
421417
return hasFurtherRange, err
422418
}
423-
hasFurtherRange = true
424-
}
425-
if !hasFurtherRange {
426-
log.Debugf("Iteration complete: no further range to iterate")
427-
return hasFurtherRange, nil
419+
rows, err := this.db.Query(query, explodedArgs...)
420+
if err != nil {
421+
return hasFurtherRange, err
422+
}
423+
iterationRangeMaxValues := sql.NewColumnValues(this.migrationContext.UniqueKey.Len())
424+
for rows.Next() {
425+
if err = rows.Scan(iterationRangeMaxValues.ValuesPointers...); err != nil {
426+
return hasFurtherRange, err
427+
}
428+
hasFurtherRange = true
429+
}
430+
if hasFurtherRange {
431+
this.migrationContext.MigrationIterationRangeMaxValues = iterationRangeMaxValues
432+
return hasFurtherRange, nil
433+
}
428434
}
429-
this.migrationContext.MigrationIterationRangeMaxValues = iterationRangeMaxValues
435+
log.Debugf("Iteration complete: no further range to iterate")
430436
return hasFurtherRange, nil
431437
}
432438

‎go/sql/builder.go‎

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,62 @@ func BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableNa
235235
return BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, mappedSharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, includeRangeStartValues, transactionalTable)
236236
}
237237

238-
func BuildUniqueKeyRangeEndPreparedQuery(databaseName, tableName string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) {
238+
func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) {
239+
if uniqueKeyColumns.Len() == 0 {
240+
return "", explodedArgs, fmt.Errorf("Got 0 columns in BuildUniqueKeyRangeEndPreparedQuery")
241+
}
242+
databaseName = EscapeName(databaseName)
243+
tableName = EscapeName(tableName)
244+
245+
var startRangeComparisonSign ValueComparisonSign = GreaterThanComparisonSign
246+
if includeRangeStartValues {
247+
startRangeComparisonSign = GreaterThanOrEqualsComparisonSign
248+
}
249+
rangeStartComparison, rangeExplodedArgs, err := BuildRangePreparedComparison(uniqueKeyColumns, rangeStartArgs, startRangeComparisonSign)
250+
if err != nil {
251+
return "", explodedArgs, err
252+
}
253+
explodedArgs = append(explodedArgs, rangeExplodedArgs...)
254+
rangeEndComparison, rangeExplodedArgs, err := BuildRangePreparedComparison(uniqueKeyColumns, rangeEndArgs, LessThanOrEqualsComparisonSign)
255+
if err != nil {
256+
return "", explodedArgs, err
257+
}
258+
explodedArgs = append(explodedArgs, rangeExplodedArgs...)
259+
260+
uniqueKeyColumnNames := duplicateNames(uniqueKeyColumns.Names())
261+
uniqueKeyColumnAscending := make([]string, len(uniqueKeyColumnNames), len(uniqueKeyColumnNames))
262+
uniqueKeyColumnDescending := make([]string, len(uniqueKeyColumnNames), len(uniqueKeyColumnNames))
263+
for i, column := range uniqueKeyColumns.Columns() {
264+
uniqueKeyColumnNames[i] = EscapeName(uniqueKeyColumnNames[i])
265+
if column.Type == EnumColumnType {
266+
uniqueKeyColumnAscending[i] = fmt.Sprintf("concat(%s) asc", uniqueKeyColumnNames[i])
267+
uniqueKeyColumnDescending[i] = fmt.Sprintf("concat(%s) desc", uniqueKeyColumnNames[i])
268+
} else {
269+
uniqueKeyColumnAscending[i] = fmt.Sprintf("%s asc", uniqueKeyColumnNames[i])
270+
uniqueKeyColumnDescending[i] = fmt.Sprintf("%s desc", uniqueKeyColumnNames[i])
271+
}
272+
}
273+
result = fmt.Sprintf(`
274+
select /* gh-ost %s.%s %s */
275+
%s
276+
from
277+
%s.%s
278+
where %s and %s
279+
order by
280+
%s
281+
limit 1
282+
offset %d
283+
`, databaseName, tableName, hint,
284+
strings.Join(uniqueKeyColumnNames, ", "),
285+
databaseName, tableName,
286+
rangeStartComparison, rangeEndComparison,
287+
strings.Join(uniqueKeyColumnAscending, ", "),
288+
(chunkSize - 1),
289+
)
290+
return result, explodedArgs, nil
291+
}
292+
293+
func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) {
239294
if uniqueKeyColumns.Len() == 0 {
240295
return "", explodedArgs, fmt.Errorf("Got 0 columns in BuildUniqueKeyRangeEndPreparedQuery")
241296
}

‎go/sql/builder_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,7 @@ func TestBuildUniqueKeyRangeEndPreparedQuery(t *testing.T) {
283283
rangeStartArgs := []interface{}{3, 17}
284284
rangeEndArgs := []interface{}{103, 117}
285285

286-
query, explodedArgs, err := BuildUniqueKeyRangeEndPreparedQuery(databaseName, originalTableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test")
286+
query, explodedArgs, err := BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, originalTableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test")
287287
test.S(t).ExpectNil(err)
288288
expected := `
289289
select /* gh-ost mydb.tbl test */ name, position

‎localtests/test.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ test_single() {
9191
--postpone-cut-over-flag-file=/tmp/gh-ost.test.postpone.flag \
9292
--test-on-replica \
9393
--default-retries=1 \
94+
--chunk-size=10 \
9495
--verbose \
9596
--debug \
9697
--stack \

0 commit comments

Comments
 (0)