Merge pull request #136 from tinyspeck/bramos_force_upstream_f93c96c

[upstream] Get us back to upstream master(f93c96c)

.dockerignore

@@ -4,4 +4,4 @@ _test
 java/*/target
 java/*/bin
 php/vendor
-
+releases

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,7 +1,6 @@
 ---
 name: Bug Report
 about: You're experiencing an issue with Vitess that is different than the documented behavior.
-
 ---
 
 When filing a bug, please include the following headings if

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,7 +1,6 @@
 ---
 name: Feature Request
 about: If you have something you think Vitess could improve or add support for.
-
 ---
 
 Please search the existing issues for relevant feature requests, and use the [reaction feature](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to add upvotes to pre-existing requests.

.github/ISSUE_TEMPLATE/question.md

@@ -1,7 +1,6 @@
 ---
 name: Question
 about: If you have a question, please check out our other community resources instead of opening an issue.
-
 ---
 
 Issues on GitHub are intended to be related to bugs or feature requests, so we recommend using our other community resources instead of asking here.

Makefile

@@ -1,11 +1,11 @@
 # Copyright 2017 Google Inc.
-# 
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -29,6 +29,10 @@ ifdef VT_GO_PARALLEL_VALUE
 export VT_GO_PARALLEL := -p $(VT_GO_PARALLEL_VALUE)
 endif
 
+ifdef VT_EXTRA_BUILD_FLAGS
+export EXTRA_BUILD_FLAGS := $(VT_EXTRA_BUILD_FLAGS)
+endif
+
 # Link against the MySQL library in $VT_MYSQL_ROOT if it's specified.
 ifdef VT_MYSQL_ROOT
 # Clutter the env var only if it's a non-standard path.
@@ -46,7 +50,7 @@ build:
 ifndef NOBANNER
 	echo $$(date): Building source tree
 endif
-	go install $(VT_GO_PARALLEL) -ldflags "$(shell tools/build_version_flags.sh)" ./go/...
+	go install $(EXTRA_BUILD_FLAGS) $(VT_GO_PARALLEL) -ldflags "$(shell tools/build_version_flags.sh)" ./go/...
 
 parser:
 	make -C go/vt/sqlparser
@@ -186,7 +190,7 @@ docker_base_mysql56:
 
 docker_base_mysql80:
 	chmod -R o=g *
-	docker build -f docker/base/Dockerfile.mysql56 -t vitess/base:mysql80 .
+	docker build -f docker/base/Dockerfile.mysql80 -t vitess/base:mysql80 .
 
 docker_base_mariadb:
 	chmod -R o=g *
@@ -206,7 +210,7 @@ docker_base_percona57:
 
 docker_base_percona80:
 	chmod -R o=g *
-	docker build -f docker/base/Dockerfile.percona57 -t vitess/base:percona80 .
+	docker build -f docker/base/Dockerfile.percona80 -t vitess/base:percona80 .
 
 # Run "make docker_lite PROMPT_NOTICE=false" to avoid that the script
 # prompts you to press ENTER and confirm that the vitess/base image is not
@@ -276,3 +280,12 @@ release: docker_base
 	echo "A git tag was created, you can push it with:"
 	echo "git push origin v$(VERSION)"
 	echo "Also, don't forget the upload releases/v$(VERSION).tar.gz file to GitHub releases"
+
+packages: docker_base
+	@if [ -z "$VERSION" ]; then \
+	  echo "Set the env var VERSION with the release version"; exit 1;\
+	fi
+	mkdir -p releases
+	docker build -f docker/packaging/Dockerfile -t vitess/packaging .
+	docker run --rm -v ${PWD}/releases:/vt/releases --env VERSION=$(VERSION) vitess/packaging --package /vt/releases -t deb --deb-no-default-config-files
+	docker run --rm -v ${PWD}/releases:/vt/releases --env VERSION=$(VERSION) vitess/packaging --package /vt/releases -t rpm

azure-pipelines.yml

@@ -0,0 +1,21 @@
+pr:
+- master
+
+pool:
+  vmImage: 'ubuntu-latest'
+
+variables:
+  flags: "-docker -use_docker_cache -timeout=8m -print-log"
+  shards: [0, 1, 2, 3, 4]
+  flavors: ["mysql56", "mysql57", "mysql80", "mariadb", "mariadb103", "percona57", "percona80"]
+jobs:
+- job: tests
+  strategy:
+    matrix:
+      ${{ each flavor in variables.flavors }}:
+        ${{ each shard in variables.shards }}:
+         ${{ format('{0}{1}', flavor, shard) }}:
+           flavor: ${{ flavor }}
+           shard: ${{ shard }}
+  steps:
+  - script: go run test.go -shard $(shard) -flavor $(flavor) $(flags)

bootstrap.sh

@@ -195,13 +195,22 @@ function install_etcd() {
   local version="$1"
   local dist="$2"
 
+  case $(uname) in
+    Linux)  local platform=linux; local ext=tar.gz;;
+    Darwin) local platform=darwin; local ext=zip;;
+  esac
+
   download_url=https://github.com/coreos/etcd/releases/download
-  tar_file="etcd-${version}-linux-amd64.tar.gz"
+  file="etcd-${version}-${platform}-amd64.${ext}"
 
-  wget "$download_url/$version/$tar_file"
-  tar xzf "$tar_file"
-  rm "$tar_file"
-  ln -snf "$dist/etcd-${version}-linux-amd64/etcd" "$VTROOT/bin/etcd"
+  wget "$download_url/$version/$file"
+  if [ "$ext" = "tar.gz" ]; then
+    tar xzf "$file"
+  else
+    unzip "$file"
+  fi
+  rm "$file"
+  ln -snf "$dist/etcd-${version}-${platform}-amd64/etcd" "$VTROOT/bin/etcd"
 }
 install_dep "etcd" "v3.3.10" "$VTROOT/dist/etcd" install_etcd
 
@@ -211,9 +220,14 @@ function install_consul() {
   local version="$1"
   local dist="$2"
 
+  case $(uname) in
+    Linux)  local platform=linux;;
+    Darwin) local platform=darwin;;
+  esac
+
   download_url=https://releases.hashicorp.com/consul
-  wget "${download_url}/${version}/consul_${version}_linux_amd64.zip"
-  unzip "consul_${version}_linux_amd64.zip"
+  wget "${download_url}/${version}/consul_${version}_${platform}_amd64.zip"
+  unzip "consul_${version}_${platform}_amd64.zip"
   ln -snf "$dist/consul" "$VTROOT/bin/consul"
 }
 install_dep "Consul" "1.4.0" "$VTROOT/dist/consul" install_consul
@@ -290,7 +304,6 @@ gotools=" \
        golang.org/x/tools/cmd/cover \
        golang.org/x/tools/cmd/goimports \
        golang.org/x/tools/cmd/goyacc \
-       honnef.co/go/tools/cmd/unused \
 "
 echo "Installing dev tools with 'go get'..."
 # shellcheck disable=SC2086
@@ -325,13 +338,13 @@ if [ "$BUILD_TESTS" == 1 ] ; then
     echo "MYSQL_FLAVOR environment variable not set. Using default: $MYSQL_FLAVOR"
   fi
   case "$MYSQL_FLAVOR" in
-    "MySQL56")
+    "MySQL56" | "MySQL80")
       myversion="$("$VT_MYSQL_ROOT/bin/mysql" --version)"
       [[ "$myversion" =~ Distrib\ 5\.[67] || "$myversion" =~ Ver\ 8\. ]] || fail "Couldn't find MySQL 5.6+ in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
       echo "Found MySQL 5.6+ installation in $VT_MYSQL_ROOT."
       ;;
 
-    "MariaDB" | "MariaDB103" )
+    "MariaDB" | "MariaDB103")
       myversion="$("$VT_MYSQL_ROOT/bin/mysql" --version)"
       [[ "$myversion" =~ MariaDB ]] || fail "Couldn't find MariaDB in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
       echo "Found MariaDB installation in $VT_MYSQL_ROOT."

config/init_db.sql

@@ -1,9 +1,16 @@
 # This file is executed immediately after mysql_install_db,
 # to initialize a fresh data directory.
 
-##########################################
+###############################################################################
+# WARNING: This sql is *NOT* safe for production use,
+#          as it contains default well-known users and passwords.
+#          Care should be taken to change these users and passwords
+#          for production.
+###############################################################################
+
+###############################################################################
 # Equivalent of mysql_secure_installation
-##########################################
+###############################################################################
 
 # Changes during the init db should not make it to the binlog.
 # They could potentially create errant transactions on replicas.
@@ -17,9 +24,9 @@ DELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost';
 # Remove test database.
 DROP DATABASE IF EXISTS test;
 
-##########################################
+###############################################################################
 # Vitess defaults
-##########################################
+###############################################################################
 
 # Vitess-internal database.
 CREATE DATABASE IF NOT EXISTS _vt;
@@ -28,12 +35,14 @@ CREATE DATABASE IF NOT EXISTS _vt;
 CREATE TABLE IF NOT EXISTS _vt.local_metadata (
   name VARCHAR(255) NOT NULL,
   value VARCHAR(255) NOT NULL,
-  PRIMARY KEY (name)
+  db_name VARBINARY(255) NOT NULL,
+  PRIMARY KEY (db_name, name)
   ) ENGINE=InnoDB;
 CREATE TABLE IF NOT EXISTS _vt.shard_metadata (
   name VARCHAR(255) NOT NULL,
   value MEDIUMBLOB NOT NULL,
-  PRIMARY KEY (name)
+  db_name VARBINARY(255) NOT NULL,
+  PRIMARY KEY (db_name, name)
   ) ENGINE=InnoDB;
 
 # Admin user with all privileges.

config/mycnf/default-fast.cnf

@@ -1,41 +1,27 @@
 # basic config parameters for all db instances in the grid
 
 sql_mode = STRICT_TRANS_TABLES
-back_log = 50
 binlog_format = statement
 character_set_server = utf8
 collation_server = utf8_general_ci
 connect_timeout = 30
 datadir = {{.DataDir}}
-default-storage-engine = innodb
 expire_logs_days = 3
-innodb_autoextend_increment = 1
 innodb_buffer_pool_size = 64M
-innodb_data_file_path = ibdata1:10M:autoextend
 innodb_data_home_dir = {{.InnodbDataHomeDir}}
-innodb_file_per_table
 innodb_flush_log_at_trx_commit = 2
-innodb_flush_method = O_DIRECT
 innodb_lock_wait_timeout = 20
 innodb_log_buffer_size = 1M
-innodb_log_file_size = 1M
-innodb_log_files_in_group = 2
+innodb_log_file_size = 4M
 innodb_log_group_home_dir = {{.InnodbLogGroupHomeDir}}
-innodb_max_dirty_pages_pct = 75
-innodb_thread_concurrency = 2
 key_buffer_size = 2M
 log-error = {{.ErrorLogPath}}
 long_query_time = 2
-max_allowed_packet = 16M
-max_connections = 200
-net_write_timeout = 60
 pid-file = {{.PidFile}}
 port = {{.MysqlPort}}
 # all db instances should start in read-only mode - once the db is started and
 # fully functional, we'll push it into read-write mode
 read-only
-read_buffer_size = 1M
-read_rnd_buffer_size = 1M
 server-id = {{.ServerID}}
 skip-name-resolve
 # we now need networking for replication. this is a tombstone to simpler times.
@@ -49,9 +35,4 @@ slave_load_tmpdir = {{.SlaveLoadTmpDir}}
 slow-query-log
 slow-query-log-file = {{.SlowLogPath}}
 socket = {{.SocketFile}}
-sort_buffer_size = 2M
-table_open_cache = 2048
-thread_cache_size = 200
 tmpdir = {{.TmpDir}}
-tmp_table_size = 32M
-transaction-isolation = REPEATABLE-READ

config/mycnf/default.cnf

@@ -1,41 +1,26 @@
 # basic config parameters for all db instances in the grid
 
 sql_mode = STRICT_TRANS_TABLES
-back_log = 50
 binlog_format = statement
 character_set_server = utf8
 collation_server = utf8_general_ci
 connect_timeout = 30
 datadir = {{.DataDir}}
-default-storage-engine = innodb
 expire_logs_days = 3
-innodb_autoextend_increment = 64
 innodb_buffer_pool_size = 32M
-innodb_data_file_path = ibdata1:10M:autoextend
 innodb_data_home_dir = {{.InnodbDataHomeDir}}
-innodb_file_per_table
 innodb_flush_log_at_trx_commit = 2
-innodb_flush_method = O_DIRECT
 innodb_lock_wait_timeout = 20
-innodb_log_buffer_size = 8M
-innodb_log_file_size = 64M
-innodb_log_files_in_group = 2
 innodb_log_group_home_dir = {{.InnodbLogGroupHomeDir}}
-innodb_max_dirty_pages_pct = 75
-innodb_thread_concurrency = 20
-key_buffer_size = 32M
 log-error = {{.ErrorLogPath}}
 long_query_time = 2
 max_allowed_packet = 64M
 max_connections = 500
-net_write_timeout = 60
 pid-file = {{.PidFile}}
 port = {{.MysqlPort}}
 # all db instances should start in read-only mode - once the db is started and
 # fully functional, we'll push it into read-write mode
 read-only
-read_buffer_size = 8M
-read_rnd_buffer_size = 8M
 server-id = {{.ServerID}}
 skip-name-resolve
 # all db instances should skip the slave startup - that way we can do any
@@ -47,9 +32,4 @@ slave_load_tmpdir = {{.SlaveLoadTmpDir}}
 slow-query-log
 slow-query-log-file = {{.SlowLogPath}}
 socket = {{.SocketFile}}
-sort_buffer_size = 2M
-table_open_cache = 2048
-thread_cache_size = 200
 tmpdir = {{.TmpDir}}
-tmp_table_size = 32M
-transaction-isolation = REPEATABLE-READ

config/mycnf/master_mysql80.cnf

@@ -0,0 +1,38 @@
+# Options for enabling GTID
+# https://dev.mysql.com/doc/refman/5.6/en/replication-gtids-howto.html
+gtid_mode = ON
+log_bin
+log_slave_updates
+enforce_gtid_consistency
+
+# Crash-safe replication settings.
+master_info_repository = TABLE
+relay_log_info_repository = TABLE
+relay_log_purge = 1
+relay_log_recovery = 1
+
+# Native AIO tends to run into aio-max-nr limit during test startup.
+innodb_use_native_aio = 0
+
+# Semi-sync replication is required for automated unplanned failover
+# (when the master goes away). Here we just load the plugin so it's
+# available if desired, but it's disabled at startup.
+#
+# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync
+# at the proper time when replication is set up, or when masters are
+# promoted or demoted.
+plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so
+
+# When semi-sync is enabled, don't allow fallback to async
+# if you get no ack, or have no slaves. This is necessary to
+# prevent alternate futures when doing a failover in response to
+# a master that becomes unresponsive.
+rpl_semi_sync_master_timeout = 1000000000000000000
+rpl_semi_sync_master_wait_no_slave = 1
+
+# disable mysqlx
+mysqlx = 0
+
+# 8.0 changes the default auth-plugin to caching_sha2_password
+default_authentication_plugin = mysql_native_password
+secure_file_priv = NULL

doc/BackupAndRestore.md

@@ -168,8 +168,16 @@ to restore a backup to that tablet.
 
 As noted in the [Prerequisites](#prerequisites) section, the flag is
 generally enabled all of the time for all of the tablets in a shard.
-If Vitess cannot find a backup in the Backup Storage system, it just
-starts the vttablet as a new tablet.
+By default, if Vitess cannot find a backup in the Backup Storage system,
+the tablet will start up empty. This behavior allows you to bootstrap a new
+shard before any backups exist.
+
+If the `-wait_for_backup_interval` flag is set to a value greater than zero,
+the tablet will instead keep checking for a backup to appear at that interval.
+This can be used to ensure tablets launched concurrently while an initial backup
+is being seeded for the shard (e.g. uploaded from cold storage or created by
+another tablet) will wait until the proper time and then pull the new backup
+when it's ready.
 
 ``` sh
 vttablet ... -backup_storage_implementation=file \