Merge pull request #546 from github/gh-ost-ci-env

Shlomi Noach · web-flow · commit f83facc10963 · 2018-02-27T14:06:08.000+02:00
Adding localtests via gh-ost-ci-env
diff --git a/.travis.yml b/.travis.yml
@@ -8,13 +8,22 @@ os:
 
 env:
 - MYSQL_USER=root
+- CURRENT_CI_ENV=travis
+
+addons:
+  apt:
+    packages:
+    - git
+    - numactl
+    - libaio1
 
 before_install:
   - mysql -e 'CREATE DATABASE IF NOT EXISTS test;'
 
 install: true
 
-script: script/cibuild
+script:
+  - script/cibuild
 
 notifications:
   email: false
diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go
@@ -44,7 +44,6 @@ func acceptSignals(migrationContext *base.MigrationContext) {
 // main is the application's entry point. It will either spawn a CLI or HTTP interfaces.
 func main() {
 	migrationContext := base.NewMigrationContext()
-
 	flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)")
 	flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master")
 	flag.IntVar(&migrationContext.InspectorConnectionConfig.Key.Port, "port", 3306, "MySQL port (preferably a replica, not the master)")
diff --git a/localtests/datetime-submillis-zeroleading/ignore_versions b/localtests/datetime-submillis-zeroleading/ignore_versions
@@ -0,0 +1 @@
+(5.5)
diff --git a/localtests/datetime-submillis/ignore_versions b/localtests/datetime-submillis/ignore_versions
@@ -0,0 +1 @@
+(5.5)
diff --git a/localtests/datetime-to-timestamp-pk-fail/ignore_versions b/localtests/datetime-to-timestamp-pk-fail/ignore_versions
@@ -0,0 +1 @@
+(5.5)
diff --git a/localtests/datetime/ignore_versions b/localtests/datetime/ignore_versions
@@ -0,0 +1 @@
+(5.5)
diff --git a/localtests/json57/ignore_versions b/localtests/json57/ignore_versions
@@ -0,0 +1 @@
+(5.5|5.6)
diff --git a/localtests/json57dml/ignore_versions b/localtests/json57dml/ignore_versions
@@ -0,0 +1 @@
+(5.5|5.6)
diff --git a/localtests/swap-pk-uk/ignore_versions b/localtests/swap-pk-uk/ignore_versions
@@ -0,0 +1 @@
+(5.5)
diff --git a/localtests/swap-uk-uk/ignore_versions b/localtests/swap-uk-uk/ignore_versions
@@ -0,0 +1 @@
+(5.5)
diff --git a/localtests/test.sh b/localtests/test.sh
@@ -9,23 +9,38 @@
 
 tests_path=$(dirname $0)
 test_logfile=/tmp/gh-ost-test.log
-ghost_binary=/tmp/gh-ost-test
+default_ghost_binary=/tmp/gh-ost-test
+ghost_binary=""
 exec_command_file=/tmp/gh-ost-test.bash
-orig_content_output_file=/gh-ost-test.orig.content.csv
-ghost_content_output_file=/gh-ost-test.ghost.content.csv
-test_pattern="${1:-.}"
+orig_content_output_file=/tmp/gh-ost-test.orig.content.csv
+ghost_content_output_file=/tmp/gh-ost-test.ghost.content.csv
 
 master_host=
 master_port=
 replica_host=
 replica_port=
 
+OPTIND=1
+while getopts "b:" OPTION
+do
+  case $OPTION in
+    b)
+      ghost_binary="$OPTARG"
+    ;;
+  esac
+done
+shift $((OPTIND-1))
+
+test_pattern="${1:-.}"
+
 verify_master_and_replica() {
   if [ "$(gh-ost-test-mysql-master -e "select 1" -ss)" != "1" ] ; then
     echo "Cannot verify gh-ost-test-mysql-master"
     exit 1
   fi
   read master_host master_port <<< $(gh-ost-test-mysql-master -e "select @@hostname, @@port" -ss)
+  [ "$master_host" == "$(hostname)" ] && master_host="127.0.0.1"
+  echo "# master verified at $master_host:$master_port"
   if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then
     echo "Cannot verify gh-ost-test-mysql-replica"
     exit 1
@@ -35,6 +50,8 @@ verify_master_and_replica() {
     exit 1
   fi
   read replica_host replica_port <<< $(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss)
+  [ "$replica_host" == "$(hostname)" ] && replica_host="127.0.0.1"
+  echo "# replica verified at $replica_host:$replica_port"
 }
 
 exec_cmd() {
@@ -66,6 +83,15 @@ test_single() {
   local test_name
   test_name="$1"
 
+  if [ -f $tests_path/$test_name/ignore_versions ] ; then
+    ignore_versions=$(cat $tests_path/$test_name/ignore_versions)
+    mysql_version=$(gh-ost-test-mysql-master -s -s -e "select @@version")
+    if echo "$mysql_version" | egrep -q "^${ignore_versions}" ; then
+      echo -n "Skipping: $test_name"
+      return 0
+    fi
+  fi
+
   echo -n "Testing: $test_name"
 
   echo_dot
@@ -98,6 +124,7 @@ test_single() {
     --password=gh-ost \
     --host=$replica_host \
     --port=$replica_port \
+    --assume-master-host=${master_host}:${master_port}
     --database=test \
     --table=gh_ost_test \
     --alter='engine=innodb' \
@@ -148,7 +175,8 @@ test_single() {
 
   if [ $execution_result -ne 0 ] ; then
     echo
-    echo "ERROR $test_name execution failure. cat $test_logfile"
+    echo "ERROR $test_name execution failure. cat $test_logfile:"
+    cat $test_logfile
     return 1
   fi
 
@@ -170,7 +198,12 @@ test_single() {
 
 build_binary() {
   echo "Building"
-  rm -f $ghost_binary
+  rm -f $default_ghost_binary
+  [ "$ghost_binary" == "" ] && ghost_binary="$default_ghost_binary"
+  if [ -f "$ghost_binary" ] ; then
+    echo "Using binary: $ghost_binary"
+    return 0
+  fi
   go build -o $ghost_binary go/cmd/gh-ost/main.go
   if [ $? -ne 0 ] ; then
     echo "Build failure"
diff --git a/localtests/timestamp-to-datetime/ignore_versions b/localtests/timestamp-to-datetime/ignore_versions
@@ -0,0 +1 @@
+(5.5)
diff --git a/localtests/timestamp/ignore_versions b/localtests/timestamp/ignore_versions
@@ -0,0 +1 @@
+(5.5)
diff --git a/localtests/tz-datetime-ts/ignore_versions b/localtests/tz-datetime-ts/ignore_versions
@@ -0,0 +1 @@
+(5.5)
diff --git a/localtests/tz/ignore_versions b/localtests/tz/ignore_versions
@@ -0,0 +1 @@
+(5.5)
diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+set -e
+
+whoami
+
+# Clone gh-ost-ci-env
+# Only clone if not already running locally at latest commit
+remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git HEAD | cut -f1)
+local_commit="unknown"
+[ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1)
+
+echo "remote commit is: $remote_commit"
+echo "local commit is:  $local_commit"
+
+if [ "$remote_commit" != "$local_commit" ] ; then
+  rm -rf ./gh-ost-ci-env
+  git clone https://github.com/github/gh-ost-ci-env.git
+fi
+
+test_mysql_version() {
+  local mysql_version
+  mysql_version="$1"
+
+  echo "##### Testing $mysql_version"
+
+  echo "### Setting up sandbox for $mysql_version"
+
+  find sandboxes -name "stop_all" | bash
+
+  mkdir -p sandbox/binary
+  rm -rf sandbox/binary/*
+  gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.gz --unpack-version="$mysql_version" --sandbox-binary ${PWD}/sandbox/binary
+
+  mkdir -p sandboxes
+  rm -rf sandboxes/*
+
+  if echo "$mysql_version" | egrep "5[.]5[.]" ; then
+    gtid=""
+  else
+    gtid="--gtid"
+  fi
+  gh-ost-ci-env/bin/linux/dbdeployer deploy replication "$mysql_version" --nodes 2 --sandbox-binary ${PWD}/sandbox/binary --sandbox-home ${PWD}/sandboxes ${gtid} --my-cnf-options log_slave_updates --my-cnf-options log_bin --my-cnf-options binlog_format=ROW --sandbox-directory rsandbox
+
+  sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-master
+  echo 'sandboxes/rsandbox/m "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-master
+
+  sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-replica
+  echo 'sandboxes/rsandbox/s1 "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-replica
+
+  export PATH="${PWD}/gh-ost-ci-env/bin/:${PATH}"
+
+  gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%' identified by 'gh-ost'"
+
+  echo "### Running gh-ost tests for $mysql_version"
+  ./localtests/test.sh -b bin/gh-ost
+
+  find sandboxes -name "stop_all" | bash
+}
+
+echo "Building..."
+. script/build
+# Test all versions:
+find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do
+  test_mysql_version "$mysql_version"
+done
