Release v1.5.2 (#274)

* Bugfix issue240 (#241) * Preliminary fix for issue #240 * Disable header check for TestConf as it must not affect overall usage the tool * Added check for [TestConf] header as well * Finalizing the issue fix * Adding --lock-ddl-per-table and --lock-ddl support (#244) * Preliminary fix for issue #243 * Calling for first test * Added some logging message * Redirecting the output to /dev/null * Adding x permission * Running the new sql commands * Trying to run this ddl_test.sh bash script * Using full string * making some changes * fixing redirect issue * unpexteced toke done * Changed workflow * Added u+x * calling it without bash command * fixing bash caller * fixing the path name * calling with /usr/bin/bash * Removed sleep statement * getting output from sqls * trying to fix issue again * trying to make cleanup actions * changing the try/except block position * chenged workflow again * Disabling sysbench run * Disabled this feature * Bugfix issue246 (#247) * Preliminary fix for issue #246 * Added options to config generator for --test_mode * Fixing directory naming * implicit conversion * Calling shutil.copytree here * Adding '_archive' string * copying backup_dir * Finalizing the feature implementation * Bugfix issue245 (#249) * Preliminary fix for issue #245 * Executing loop in concurrent.futures * Changing static method to use Popen * Reducing worker count * Increasing worker max count to 5000 * Changed back the value to 1000 * Reducing max worker count to 100 * Calling from ordinary loop * Implemented, finalizing the feature * Bugfix issue250 (#251) * Preliminary fix for issue #250 * Added rename basedirs bats file * Preliminary fix for issue #252 (#253) * Preliminary fix for issue #254 (#255) * Bugfix issue260 (#261) * Checking if archive is directory or not * Giving the full path * Bugfix issue264 (#265) * Preliminary fix for issue #264 * Decrypting some tables to be altered to unencrypted general tablespace file * Disabled converting tables to myisam * Bugfix issue262 (#266) * Preliminary fix for issue #262 * Unexpected argument error * Preliminary fix for issue #263 (#267) * Bugfix issue268 (#270) * Added 2 temporary table creation for issue #268 * Fixing typo * Added some bash files and removed previous calls * Calling the staticmethod * Fixing the table name * Fixing the sql * Added file permissions * Added create index run for creating temporary tables * Fixing the database name * again fix for database name * Disabled sleep query runs also added some kill process calls * Bugfix issue271 (#273) * Preliminary fix for issue #271: Added several optional bash files to be executed. Called it from take_backup.py * Removing drop statement * Added kill statements * VERSION BUMP * Bugfix issue275 (#276) * Parsing config properly. Added keyring_vault cmd option to pass. For issue #275 * Fixing the .sql extension files to be in package_data * changed a bit flow * Enabling back the general tablespace encryption * UnboundLocalError: local variable 'pid' referenced before assignment * Redirecting the output
ShahriyarR · Apr 24, 2018 · 8ae7927 · 8ae7927
1 parent 06aecbb
commit 8ae7927
Show file tree

Hide file tree

Showing 20 changed files with 501 additions and 83 deletions.
diff --git a/autoxtrabackup.py b/autoxtrabackup.py
@@ -45,7 +45,7 @@ def print_version(ctx, param, value):
     click.echo("Email: [email protected]")
     click.echo(
         "Based on Percona XtraBackup: https://github.com/percona/percona-xtrabackup/")
-    click.echo('MySQL-AutoXtraBackup Version: 1.5.1')
+    click.echo('MySQL-AutoXtraBackup Version: 1.5.2')
     ctx.exit()
 
 
@@ -54,7 +54,7 @@ def check_file_content(file):
     with open(file, 'r') as config_file:
         file_content = config_file.read()
 
-    config_headers = ["MySQL", "Backup", "Encrypt", "Compress", "Commands", "TestConf"]
+    config_headers = ["MySQL", "Backup", "Encrypt", "Compress", "Commands"]
     config_keys = [
         "mysql",
         "mycnf",
@@ -146,9 +146,28 @@ def validate_file(file):
                                  'ERROR',
                                  'CRITICAL']),
               help="Set log level")
+@click.option('--log_file_max_bytes',
+              default=1073741824,
+              show_default=True,
+              nargs=1,
+              type=int,
+              help="Set log file max size in bytes")
+@click.option('--log_file_backup_count',
+              default=7,
+              show_default=True,
+              nargs=1,
+              type=int,
+              help="Set log file backup count")
+@click.option('--keyring_vault',
+              default=0,
+              show_default=True,
+              nargs=1,
+              type=int,
+              help="Enable this when you pass keyring_vault options in default mysqld options in config"
+                   "[Only for using with --test_mode]")
 @click.option('--test_mode',
               is_flag=True,
-              help="Enable test mode.Must be used with --defaults_file and only for TESTs for XtraBackup")
+              help="Enable test mode. Must be used with --defaults_file and only for TESTs for XtraBackup")
 @click.option('--help',
               is_flag=True,
               callback=print_help,
@@ -157,7 +176,9 @@ def validate_file(file):
               help="Print help message and exit.")
 @click.pass_context
 def all_procedure(ctx, prepare, backup, partial, tag, show_tags,
-                  verbose, log_file, log, defaults_file, dry_run, test_mode):
+                  verbose, log_file, log, defaults_file,
+                  dry_run, test_mode, log_file_max_bytes,
+                  log_file_backup_count, keyring_vault):
     logger.setLevel(log)
     formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(message)s',
                                   datefmt='%Y-%m-%d %H:%M:%S')
@@ -169,7 +190,8 @@ def all_procedure(ctx, prepare, backup, partial, tag, show_tags,
 
     if log_file:
         try:
-            file_handler = RotatingFileHandler(log_file, mode='a', maxBytes=104857600, backupCount=7)
+            file_handler = RotatingFileHandler(log_file, mode='a',
+                                               maxBytes=log_file_max_bytes, backupCount=log_file_backup_count)
             file_handler.setFormatter(formatter)
             logger.addHandler(file_handler)
         except PermissionError as err:
@@ -200,7 +222,10 @@ def all_procedure(ctx, prepare, backup, partial, tag, show_tags,
                 test_obj = RunnerTestMode(config=defaults_file)
                 for basedir in test_obj.basedirs:
                     if ('5.7' in basedir) and ('2_4_ps_5_7' in defaults_file):
-                        test_obj.wipe_backup_prepare_copyback(basedir=basedir)
+                        if keyring_vault == 1:
+                            test_obj.wipe_backup_prepare_copyback(basedir=basedir, keyring_vault=1)
+                        else:
+                            test_obj.wipe_backup_prepare_copyback(basedir=basedir)
                     elif ('5.6' in basedir) and ('2_4_ps_5_6' in defaults_file):
                         test_obj.wipe_backup_prepare_copyback(basedir=basedir)
                     elif ('5.6' in basedir) and ('2_3_ps_5_6' in defaults_file):

diff --git a/general_conf/bck.conf b/general_conf/bck.conf
@@ -30,6 +30,8 @@ xtra_prepare=--apply-log-only
 xtra_options=--no-version-check
 #Optional: set archive and rotation
 #archive_dir=/home/shahriyar.rzaev/XB_TEST/backup_archives
+#prepare_archive=1
+#move_archive=0
 #full_backup_interval=1 day
 #max_archive_size=100GiB
 #max_archive_duration=4 Days
@@ -87,14 +89,14 @@ chown_command=chown -R mysql:mysql
 
 
 # Do not touch; this is for --test_mode, which is testing for XtraBackup itself.
-[TestConf]
-ps_branches=5.5 5.6 5.7
-pxb_branches=2.3 2.4
-gitcmd=--recursive --depth=1 https://github.com/percona/percona-server.git
-pxb_gitcmd=https://github.com/percona/percona-xtrabackup.git
-testpath=/home/shahriyar.rzaev/XB_TEST/server_dir
-incremental_count=3
+#[TestConf]
+#ps_branches=5.5 5.6 5.7
+#pxb_branches=2.3 2.4
+#gitcmd=--recursive --depth=1 https://github.com/percona/percona-server.git
+#pxb_gitcmd=https://github.com/percona/percona-xtrabackup.git
+#testpath=/home/shahriyar.rzaev/XB_TEST/server_dir
+#incremental_count=3
 #make_slaves=1
-xb_configs=xb_2_4_ps_5_6.conf xb_2_4_ps_5_7.conf xb_2_3_ps_5_6.conf xb_2_3_ps_5_5.conf xb_2_4_ps_5_5.conf
-default_mysql_options=--log-bin=mysql-bin,--log-slave-updates,--server-id={},--gtid-mode=ON,--enforce-gtid-consistency,--binlog-format=row
-mysql_options=--innodb_buffer_pool_size=1G 2G 3G,--innodb_log_file_size=1G 2G 3G,--innodb_page_size=4K 8K 16K 32K 64K
+#xb_configs=xb_2_4_ps_5_6.conf xb_2_4_ps_5_7.conf xb_2_3_ps_5_6.conf xb_2_3_ps_5_5.conf xb_2_4_ps_5_5.conf
+#default_mysql_options=--log-bin=mysql-bin,--log-slave-updates,--server-id={},--gtid-mode=ON,--enforce-gtid-consistency,--binlog-format=row
+#mysql_options=--innodb_buffer_pool_size=1G 2G 3G,--innodb_log_file_size=1G 2G 3G,--innodb_page_size=4K 8K 16K 32K 64K
diff --git a/general_conf/generalops.py b/general_conf/generalops.py
@@ -29,8 +29,7 @@ def __init__(self, config='/etc/bck.conf'):
             self.datadir = DB['datadir']
             # self.tmpdir = DB['tmpdir']
             # self.tmp = DB['tmp']
-
-
+
             BCK = con['Backup']
             if 'pid_dir' in BCK:
                 self.pid_dir = BCK['pid_dir']
@@ -60,6 +59,10 @@ def __init__(self, config='/etc/bck.conf'):
                 self.full_backup_interval = 86400
             if 'archive_dir' in BCK:
                 self.archive_dir = BCK['archive_dir']
+            if 'prepare_archive' in BCK:
+                self.prepare_archive = BCK['prepare_archive']
+            if 'move_archive' in BCK:
+                self.move_archive = BCK['move_archive']
             if 'max_archive_size' in BCK:
                 self.max_archive_size = humanfriendly.parse_size(
                     BCK['max_archive_size'])
@@ -68,8 +71,7 @@ def __init__(self, config='/etc/bck.conf'):
                     BCK['max_archive_duration'])
             if 'partial_list' in BCK:
                 self.partial_list = BCK['partial_list']
-
-
+
             if 'Remote' in con:
                 RM = con['Remote']
                 if 'remote_conn' in RM:
@@ -117,25 +119,33 @@ def __init__(self, config='/etc/bck.conf'):
             if 'xbs_decrypt' in XBS:
                 self.xbs_decrypt = XBS['xbs_decrypt']
 
-
             CM = con['Commands']
             self.start_mysql = CM['start_mysql_command']
             self.stop_mysql = CM['stop_mysql_command']
             self.chown_command = CM['chown_command']
 
-            TEST = con['TestConf']
-            self.ps_branches = TEST['ps_branches']
-            self.pxb_branches = TEST['pxb_branches']
-            self.gitcmd = TEST['gitcmd']
-            self.pxb_gitcmd = TEST['pxb_gitcmd']
-            self.testpath = TEST['testpath']
-            self.incremental_count = TEST['incremental_count']
-            self.xb_configs = TEST['xb_configs']
-            self.default_mysql_options = TEST['default_mysql_options']
-            if 'mysql_options' in TEST:
-                self.mysql_options = TEST['mysql_options']
-            if 'make_slaves' in TEST:
-                self.make_slaves = TEST['make_slaves']
+            if 'TestConf' in con:
+                TEST = con['TestConf']
+                if 'ps_branches' in TEST:
+                    self.ps_branches = TEST['ps_branches']
+                if 'pxb_branches' in TEST:
+                    self.pxb_branches = TEST['pxb_branches']
+                if 'gitcmd' in TEST:
+                    self.gitcmd = TEST['gitcmd']
+                if 'pxb_gitcmd' in TEST:
+                    self.pxb_gitcmd = TEST['pxb_gitcmd']
+                if 'testpath' in TEST:
+                    self.testpath = TEST['testpath']
+                if 'incremental_count' in TEST:
+                    self.incremental_count = TEST['incremental_count']
+                if 'xb_configs' in TEST:
+                    self.xb_configs = TEST['xb_configs']
+                if 'default_mysql_options' in TEST:
+                    self.default_mysql_options = TEST['default_mysql_options']
+                if 'mysql_options' in TEST:
+                    self.mysql_options = TEST['mysql_options']
+                if 'make_slaves' in TEST:
+                    self.make_slaves = TEST['make_slaves']
 
         else:
             logger.critical("Missing config file : /etc/bck.conf")
diff --git a/master_backup_script/backuper.py b/master_backup_script/backuper.py
@@ -215,25 +215,47 @@ def create_backup_archives(self):
         # Creating .tar.gz archive files of taken backups
         for i in os.listdir(self.full_dir):
             if len(os.listdir(self.full_dir)) == 1 or i != max(os.listdir(self.full_dir)):
-                run_tar = "tar -zcf %s %s %s" % (self.archive_dir + '/' + i + '.tar.gz', self.full_dir, self.inc_dir)
                 logger.debug("Preparing backups prior archiving them...")
-                prepare_obj = Prepare(config=self.conf, dry_run=self.dry, tag=self.tag)
-                prepare_obj.prepare_inc_full_backups()
-                logger.debug("Started to archive previous backups")
-                status, output = subprocess.getstatusoutput(run_tar)
-                if status == 0:
-                    logger.debug("OK: Old full backup and incremental backups archived!")
-                    return True
+
+                if hasattr(self, 'prepare_archive'):
+                    logger.debug("Started to prepare backups, prior archiving!")
+                    prepare_obj = Prepare(config=self.conf, dry_run=self.dry, tag=self.tag)
+                    prepare_obj.prepare_inc_full_backups()
+
+                if hasattr(self, 'move_archive') and (int(self.move_archive) == 1):
+                    dir_name = self.archive_dir + '/' + i + '_archive'
+                    try:
+                        shutil.copytree(self.backupdir, dir_name)
+                    except Exception as err:
+                        logger.error("FAILED: Archiving ")
+                        logger.error(err)
+                        raise
+                    else:
+                        return True
                 else:
-                    logger.error("FAILED: Archiving ")
-                    logger.error(output)
-                    raise RuntimeError("FAILED: Archiving -> {}".format(output))
+                    run_tar = "tar -zcf %s %s %s" % (
+                    self.archive_dir + '/' + i + '.tar.gz', self.full_dir, self.inc_dir)
+                    logger.debug("Started to archive previous backups")
+
+                    status, output = subprocess.getstatusoutput(run_tar)
+                    if status == 0:
+                        logger.debug("OK: Old full backup and incremental backups archived!")
+                        return True
+                    else:
+                        logger.error("FAILED: Archiving ")
+                        logger.error(output)
+                        raise RuntimeError("FAILED: Archiving -> {}".format(output))
 
     def clean_old_archives(self):
         logger.debug("Starting cleaning of old archives")
         for archive in self.sorted_ls(self.archive_dir):
-            archive_date = datetime.strptime(
-                archive, "%Y-%m-%d_%H-%M-%S.tar.gz")
+            if '_archive' in archive:
+                archive_date = datetime.strptime(
+                    archive, "%Y-%m-%d_%H-%M-%S_archive")
+            else:
+                archive_date = datetime.strptime(
+                    archive, "%Y-%m-%d_%H-%M-%S.tar.gz")
+
             now = datetime.now()
 
             # Finding if last full backup older than the interval or more from
@@ -246,15 +268,21 @@ def clean_old_archives(self):
                     "/" +
                     archive +
                     " due to max archive age")
-                os.remove(self.archive_dir + "/" + archive)
+                if os.path.isdir(self.archive_dir + "/" + archive):
+                    shutil.rmtree(self.archive_dir + "/" + archive)
+                else:
+                    os.remove(self.archive_dir + "/" + archive)
             elif self.get_directory_size(self.archive_dir) > self.max_archive_size:
                 logger.debug(
                     "Removing archive: " +
                     self.archive_dir +
                     "/" +
                     archive +
                     " due to max archive size")
-                os.remove(self.archive_dir + "/" + archive)
+                if os.path.isdir(self.archive_dir + "/" + archive):
+                    shutil.rmtree(self.archive_dir + "/" + archive)
+                else:
+                    os.remove(self.archive_dir + "/" + archive)
 
     def clean_full_backup_dir(self):
         # Deleting old full backup after taking new full backup.

diff --git a/prepare_env_test_mode/call_create_index_temp.sh b/prepare_env_test_mode/call_create_index_temp.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+PATH=$1
+BASEDIR=$2
+MYSQL_SOCK=$3
+
+while true; do /usr/bin/bash ${PATH}/create_index_temp.sh ${BASEDIR} ${MYSQL_SOCK}; done
diff --git a/prepare_env_test_mode/call_ddl_test.sh b/prepare_env_test_mode/call_ddl_test.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+PATH=$1
+BASEDIR=$2
+MYSQL_SOCK=$3
+
+while true; do /usr/bin/bash ${PATH}/ddl_test.sh ${BASEDIR} ${MYSQL_SOCK}; done
diff --git a/prepare_env_test_mode/call_innodb_online_alter_encryption_alters.sh b/prepare_env_test_mode/call_innodb_online_alter_encryption_alters.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# This bash file is for creating necessary environment for running tests
+# See https://jira.percona.com/browse/PS-3819
+
+BASEDIR=$1
+MYSQL_SOCK=$2
+FILE_DIR=$3
+MYSQL_USER=root
+
+while true; do ${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} < ${FILE_DIR}/innodb_online_alter_encryption_alters.sql >> /dev/null 2>&1; done
diff --git a/prepare_env_test_mode/call_innodb_online_alter_encryption_sql.sh b/prepare_env_test_mode/call_innodb_online_alter_encryption_sql.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# This bash file is for creating necessary environment for running tests
+# See https://jira.percona.com/browse/PS-3819
+
+BASEDIR=$1
+MYSQL_SOCK=$2
+FILE_DIR=$3
+MYSQL_USER=root
+
+${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} < ${FILE_DIR}/innodb_online_alter_encryption.sql
diff --git a/prepare_env_test_mode/call_temp_table_test.sh b/prepare_env_test_mode/call_temp_table_test.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+PATH=$1
+BASEDIR=$2
+MYSQL_SOCK=$3
+
+while true; do /usr/bin/bash ${PATH}/temp_table_test.sh ${BASEDIR} ${MYSQL_SOCK}; done
diff --git a/prepare_env_test_mode/config_generator.py b/prepare_env_test_mode/config_generator.py
@@ -82,7 +82,9 @@ def generate_config_files(test_path, conf_file, basedir, datadir, sock_file, bac
                                                          "--kill-long-queries-timeout=1 "   
                                                          "--kill-wait-query-type=all "
                                                          "--kill-long-query-type=all "
-                                                         "--no-backup-locks "   
+                                                         "--no-backup-locks "
+                                                         #"--lock-ddl-per-table "
+                                                         #"--lock-ddl "   
                                                          "--keyring-file-data={}/mysql-keyring/keyring ".format(basedir))
                 else:
                     config.set(section2, "xtra_options", "--slave-info --no-version-check --core-file "
@@ -95,10 +97,12 @@ def generate_config_files(test_path, conf_file, basedir, datadir, sock_file, bac
                                                          "--kill-long-queries-timeout=1 "   
                                                          "--kill-wait-query-type=all "
                                                          "--kill-long-query-type=all "
-                                                         "--no-backup-locks "   
+                                                         "--no-backup-locks "
                                                          "--parallel=10 --throttle=40 --check-privileges ")
                 config.set(section2, "#Optional: set archive and rotation")
                 config.set(section2, "#archive_dir", "/home/shahriyar.rzaev/XB_TEST/backup_archives")
+                config.set(section2, "#prepare_archive", "1")
+                config.set(section2, "#move_archive", "0")
                 config.set(section2, "#full_backup_interval", "1 day")
                 config.set(section2, "#max_archive_size", "100GiB")
                 config.set(section2, "#max_archive_duration", "4 Days")
@@ -183,7 +187,11 @@ def generate_config_files(test_path, conf_file, basedir, datadir, sock_file, bac
                                          "--early-plugin-load=keyring_file.so,"
                                          "--keyring_file_data={}/mysql-keyring/keyring,"
                                          "--log-bin=mysql-bin,--log-slave-updates,--server-id={},"
-                                         "--gtid-mode=ON,--enforce-gtid-consistency,--binlog-format=row")
+                                         "--gtid-mode=ON,--enforce-gtid-consistency,--binlog-format=row,"
+                                         "--encrypt_binlog=ON,--master_verify_checksum=ON,--binlog_checksum=CRC32,"
+                                         "--innodb_encrypt_tables=ON,"
+                                         "--innodb_encrypt_online_alter_logs=ON,"
+                                         "--innodb_temp_tablespace_encrypt=ON")
                 elif '5_6' in conf_file:
                     config.set(section8, "default_mysql_options",
                                          "--log-bin=mysql-bin,--log-slave-updates,--server-id={},"

diff --git a/prepare_env_test_mode/create_index_temp.sh b/prepare_env_test_mode/create_index_temp.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+BASEDIR=$1
+MYSQL_SOCK=$2
+MYSQL_USER=root
+#CONN_STR=${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK}
+
+
+${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "CREATE INDEX t10_b ON sysbench_test_db.t10 (b)"
+${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "DROP INDEX t10_b ON sysbench_test_db.t10"
+
+${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "CREATE INDEX t10_b ON sysbench_test_db.t10 (b) ALGORITHM=COPY"
+${BASEDIR}/bin/mysql --user=${MYSQL_USER} --socket=${MYSQL_SOCK} -e "DROP INDEX t10_b ON sysbench_test_db.t10 ALGORITHM=COPY"