mirror of
https://github.com/percona/percona-toolkit.git
synced 2025-09-02 02:34:19 +00:00
PT-1717 - resume pt-online-schema-change if it's interrupted
- Added options --history, --history-table, and --binary-index - Added code that creates history table to store pt-osc progress
This commit is contained in:
Sveta Smirnova
@@ -8578,6 +8578,7 @@ sub main {
|
||||
# Get configuration information.
|
||||
# ########################################################################
|
||||
my $q = new Quoter();
|
||||
my $tp = TableParser->new(Quoter => $q);
|
||||
my $o = new OptionParser();
|
||||
$o->get_specs();
|
||||
$o->get_opts();
|
||||
@@ -8795,6 +8796,32 @@ sub main {
|
||||
my $cxn = $make_cxn->(dsn => $dsn);
|
||||
my $aux_cxn = $make_cxn->(dsn => $dsn, prev_dsn => $dsn);
|
||||
|
||||
if ( $o->get('history') ) {
|
||||
create_history_table(
|
||||
dbh => $cxn->dbh(),
|
||||
hist_table => $o->get('history-table'),
|
||||
OptionParser => $o,
|
||||
TableParser => $tp,
|
||||
Quoter => $q,
|
||||
);
|
||||
# parsing args here
|
||||
use JSON;
|
||||
my $json = JSON->new->allow_nonref;
|
||||
my %opts = $o->opts();
|
||||
_d("DB: $db, tbl: $tbl");
|
||||
my $opt_hash = {};
|
||||
while ( my ($opt, $value) = each (%opts) ) {
|
||||
if ( $value->{got} ) {
|
||||
$opt_hash->{$opt} = $value->{value} ;
|
||||
}
|
||||
}
|
||||
my $json_str = $json->encode($opt_hash);
|
||||
print "$json_str\n";
|
||||
|
||||
#_d(Dumper($o->opts()));
|
||||
|
||||
}
|
||||
|
||||
my $cluster = Percona::XtraDB::Cluster->new;
|
||||
if ( $cluster->is_cluster_node($cxn) ) {
|
||||
# Because of https://bugs.launchpad.net/codership-mysql/+bug/1040108
|
||||
@@ -9208,7 +9235,6 @@ sub main {
|
||||
# ########################################################################
|
||||
# Setup and check the original table.
|
||||
# ########################################################################
|
||||
my $tp = TableParser->new(Quoter => $q);
|
||||
|
||||
# Common table data struct (that modules like NibbleIterator expect).
|
||||
my $orig_tbl = {
|
||||
@@ -10487,6 +10513,110 @@ sub main {
|
||||
# Subroutines.
|
||||
# ############################################################################
|
||||
|
||||
sub create_history_table {
|
||||
my ( %args ) = @_;
|
||||
my @required_args = qw(dbh hist_table OptionParser TableParser Quoter);
|
||||
|
||||
foreach my $arg ( @required_args ) {
|
||||
die "I need a $arg argument" unless $args{$arg};
|
||||
}
|
||||
my ($dbh, $hist_table, $o, $tp, $q) = @args{@required_args};
|
||||
|
||||
PTDEBUG && _d('Creating --history table', $hist_table);
|
||||
|
||||
# ########################################################################
|
||||
# Create the --history database.
|
||||
# ########################################################################
|
||||
|
||||
# Create history database if it not exists.
|
||||
my ($db, $tbl) = $q->split_unquote($hist_table);
|
||||
my $show_db_sql = "SHOW DATABASES LIKE '$db'";
|
||||
PTDEBUG && _d($show_db_sql);
|
||||
my @db_exists = $dbh->selectrow_array($show_db_sql);
|
||||
|
||||
if ( !@db_exists ) {
|
||||
my $create_db_sql
|
||||
= "CREATE DATABASE IF NOT EXISTS "
|
||||
. $q->quote($db)
|
||||
. " /* pt-table-checksum */";
|
||||
PTDEBUG && _d($create_db_sql);
|
||||
eval {
|
||||
$dbh->do($create_db_sql);
|
||||
};
|
||||
|
||||
# CREATE DATABASE IF NOT EXISTS failed but the db could already
|
||||
# exist and the error could be due, for example, to the user not
|
||||
# having privs to create it, but they still have privs to use it.
|
||||
if ( $EVAL_ERROR) {
|
||||
warn $EVAL_ERROR;
|
||||
die "--history database $db does not exist and it cannot be "
|
||||
. "created automatically. You need to create the database.\n";
|
||||
}
|
||||
}
|
||||
|
||||
# ########################################################################
|
||||
# Create the --history table.
|
||||
# ########################################################################
|
||||
|
||||
# Prepare DDL
|
||||
PTDEBUG && _d('Creating --history table', $hist_table);
|
||||
my $sql = $o->read_para_after(__FILE__, qr/MAGIC_create_pt_osc_history/);
|
||||
$sql =~ s/CREATE TABLE pt_osc_history/CREATE TABLE `$tbl`/;
|
||||
if ( $o->get('binary-index') ) {
|
||||
$sql =~ s/`?lower_boundary`?\s+TEXT/`lower_boundary` BLOB/is;
|
||||
$sql =~ s/`?upper_boundary`?\s+TEXT/`upper_boundary` BLOB/is;
|
||||
}
|
||||
PTDEBUG && _d($dbh, $sql);
|
||||
|
||||
# Check if the history table exists; if not, create it, maybe.
|
||||
my $tbl_exists = $tp->check_table(
|
||||
dbh => $dbh,
|
||||
db => $db,
|
||||
tbl => $tbl,
|
||||
);
|
||||
PTDEBUG && _d('--history table exists:', $tbl_exists ? 'yes' : 'no');
|
||||
|
||||
if ( $tbl_exists ) {
|
||||
# check if the table has expected structure
|
||||
my $tbl_sql = $tp->get_create_table(
|
||||
$dbh,
|
||||
$db,
|
||||
$tbl,
|
||||
);
|
||||
# we will compare SQL code
|
||||
my $sql_minimized = $sql;
|
||||
my $tbl_sql_minimized = $tbl_sql;
|
||||
$sql_minimized =~ s/;$//;
|
||||
$sql_minimized =~ s/\s+/ /g;
|
||||
$sql_minimized =~ s/`//g;
|
||||
$sql_minimized =~ s/^\s+//g;
|
||||
$sql_minimized =~ s/engine\=[\w\d_= ]+$//i;
|
||||
$tbl_sql_minimized =~ s/\s+/ /g;
|
||||
$tbl_sql_minimized =~ s/`//g;
|
||||
$tbl_sql_minimized =~ s/^\s+//g;
|
||||
$tbl_sql_minimized =~ s/engine\=[\w\d_= ]+$//i;
|
||||
|
||||
if ( lc($sql_minimized) ne lc($tbl_sql_minimized) ) {
|
||||
die "--history table $tbl exists but does not have expected structure.\n"
|
||||
. "Explected structure:\n$sql \n"
|
||||
. "User-defined table:\n$tbl_sql\n"
|
||||
. "Exiting to avoid damage of the user-defined table.";
|
||||
}
|
||||
}
|
||||
else {
|
||||
$sql =~ s/CREATE TABLE `$tbl`/CREATE TABLE IF NOT EXISTS `$db`.`$tbl`/;
|
||||
PTDEBUG && _d("Executing SQL for the --history table:\n $sql");
|
||||
eval {
|
||||
$dbh->do($sql);
|
||||
};
|
||||
if ( $EVAL_ERROR ) {
|
||||
die "Create --history table $tbl failed: $EVAL_ERROR";
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
sub validate_tries {
|
||||
my ($o) = @_;
|
||||
my @ops = qw(
|
||||
@@ -11912,6 +12042,9 @@ sub exec_nibble {
|
||||
PTDEBUG && _d($sth->{nibble}->{Statement},
|
||||
'lower boundary:', @{$boundary->{lower}},
|
||||
'upper boundary:', @{$boundary->{upper}});
|
||||
# _d($sth->{nibble}->{Statement},
|
||||
# 'lower boundary:', @{$boundary->{lower}},
|
||||
# 'upper boundary:', @{$boundary->{upper}});
|
||||
$sth->{nibble}->execute(
|
||||
# WHERE
|
||||
@{$boundary->{lower}}, # upper boundary values
|
||||
@@ -12414,6 +12547,15 @@ the server very busy, this can cause an outage.
|
||||
|
||||
Prompt for a password when connecting to MySQL.
|
||||
|
||||
=item --binary-index
|
||||
|
||||
This option modifies the behavior of L<"--history"> such that the
|
||||
history table's upper and lower boundary columns are created with the BLOB
|
||||
data type.
|
||||
This is useful in cases where you changing large tables with keys that
|
||||
include a binary data type or that have non-standard character sets.
|
||||
See L<"--history"> and L<"--resume">.
|
||||
|
||||
=item --channel
|
||||
|
||||
type: string
|
||||
@@ -12615,6 +12757,17 @@ type: Array
|
||||
Read this comma-separated list of config files; if specified, this must be the
|
||||
first option on the command line.
|
||||
|
||||
=item --history-table
|
||||
|
||||
type: string; default: percona.pt_osc_history
|
||||
|
||||
Create the L<"--history"> database and table if they do not exist.
|
||||
The structure of the history table is the same as the suggested table
|
||||
mentioned in L<"--history">.
|
||||
|
||||
By default, L<"--history"> creates the database and the table, specified by this option,
|
||||
automatically if they do not exist.
|
||||
|
||||
=item --critical-load
|
||||
|
||||
type: Array; default: Threads_running=50
|
||||
@@ -12764,6 +12917,28 @@ This option also allows to use option --where without options --no-drop-new-tabl
|
||||
|
||||
Show help and exit.
|
||||
|
||||
=item --history
|
||||
|
||||
default: 0
|
||||
|
||||
Write job progress to a table. Unfinished jobs may be restarted by the option L<"--resume">.
|
||||
The history table must have this structure (MAGIC_create_pt_osc_history):
|
||||
|
||||
CREATE TABLE pt_osc_history (
|
||||
job_id INT NOT NULL AUTO_INCREMENT,
|
||||
db CHAR(64) NOT NULL,
|
||||
tbl CHAR(64) NOT NULL,
|
||||
altr TEXT NOT NULL,
|
||||
args TEXT NOT NULL,
|
||||
lower_boundary TEXT,
|
||||
upper_boundary TEXT,
|
||||
done ENUM('yes','no') NOT NULL DEFAULT 'no',
|
||||
ts TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (job_id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
Note: lower_boundary and upper_boundary data type can be BLOB. See L<"--binary-index">.
|
||||
|
||||
=item --host
|
||||
|
||||
short form: -h; type: string
|
||||
|
||||
@@ -12019,7 +12019,7 @@ sub check_repl_table {
|
||||
# Create the --replicate database.
|
||||
# ########################################################################
|
||||
|
||||
# If the repl db doesn't exit, auto-create it, maybe.
|
||||
# If the repl db doesn't exist, auto-create it, maybe.
|
||||
my ($db, $tbl) = $q->split_unquote($repl_table);
|
||||
my $show_db_sql = "SHOW DATABASES LIKE '$db'";
|
||||
PTDEBUG && _d($show_db_sql);
|
||||
|
||||
@@ -310,15 +310,11 @@ sub find_possible_keys {
|
||||
# * dbh dbh: active dbh
|
||||
# * db scalar: database name to check
|
||||
# * tbl scalar: table name to check
|
||||
# Optional args:
|
||||
# * all_privs bool: check for all privs (select,insert,update,delete)
|
||||
# Returns: bool
|
||||
# Can die: no
|
||||
# check_table() checks the given table for certain criteria and returns
|
||||
# true if all criteria are found, else it returns false. The existence
|
||||
# of the table is always checked; if no optional args are given, then this
|
||||
# is the only check. Any error causes a false return value (e.g. if the
|
||||
# table is crashed).
|
||||
# check_table() checks the given table for the existence and returns
|
||||
# true if the table is found, else it returns false.
|
||||
# Any error causes a false return value (e.g. if the table is crashed).
|
||||
sub check_table {
|
||||
my ( $self, %args ) = @_;
|
||||
my @required_args = qw(dbh db tbl);
|
||||
|
||||
229
t/pt-online-schema-change/pt-1717.t
Normal file
229
t/pt-online-schema-change/pt-1717.t
Normal file
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env perl
|
||||
|
||||
BEGIN {
|
||||
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
|
||||
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
|
||||
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
|
||||
};
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use English qw(-no_match_vars);
|
||||
use Test::More;
|
||||
|
||||
use PerconaTest;
|
||||
use Sandbox;
|
||||
require "$trunk/bin/pt-online-schema-change";
|
||||
require VersionParser;
|
||||
|
||||
use Data::Dumper;
|
||||
|
||||
my $dp = new DSNParser(opts=>$dsn_opts);
|
||||
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
|
||||
my $master_dbh = $sb->get_dbh_for('master');
|
||||
my $slave_dbh = $sb->get_dbh_for('slave1');
|
||||
|
||||
if ( !$master_dbh ) {
|
||||
plan skip_all => 'Cannot connect to sandbox master';
|
||||
}
|
||||
elsif ( !$slave_dbh ) {
|
||||
plan skip_all => 'Cannot connect to sandbox slave';
|
||||
}
|
||||
|
||||
my @args = qw(--set-vars innodb_lock_wait_timeout=3);
|
||||
my $output = "";
|
||||
my $dsn = "h=127.1,P=12345,u=msandbox,p=msandbox";
|
||||
my $exit = 0;
|
||||
my $sample = "t/pt-online-schema-change/samples";
|
||||
|
||||
$sb->load_file('master', "$sample/basic_no_fks_innodb.sql");
|
||||
|
||||
# First test option --history
|
||||
# * - Test done for the development step
|
||||
# ** - Test done for two development steps
|
||||
# 1.** If table percona.pt_osc not created when option not specified
|
||||
# 2. If table percona.pt_osc created when option present
|
||||
# 2.1.** Default name
|
||||
# 2.2.** Custom name
|
||||
# 2.3.* Second run should not fail or modify this table (except inserting a row for new job)
|
||||
# 2.4.* Case for binary index
|
||||
# 2.5.** Second run for the binary index
|
||||
# 2.6.** Case for invalid existing table
|
||||
# 2.7.** Case for invalid existing table and binary index
|
||||
# 3. Inserting db, tbl, alter, args
|
||||
# 4. Updating lower and upper boundaries
|
||||
# 4.1. In situation when pt-osc finishes correctly
|
||||
# 4.1.1. `done` set to 'yes'
|
||||
# 4.2. In failures
|
||||
# 4.2.1. `done` set to 'no'
|
||||
|
||||
($output, $exit) = full_output(
|
||||
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
|
||||
'--alter', 'engine=innodb', '--execute') }
|
||||
);
|
||||
|
||||
is(
|
||||
$exit,
|
||||
0,
|
||||
'basic test finished OK'
|
||||
) or diag($output);
|
||||
|
||||
$output = `/tmp/12345/use -N -e "select count(*) from information_schema.tables where TABLE_SCHEMA='percona' and table_name='pt_osc_history'"`;
|
||||
is(
|
||||
$output + 0,
|
||||
0,
|
||||
'--history table not created when option --history not provided'
|
||||
);
|
||||
|
||||
($output, $exit) = full_output(
|
||||
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
|
||||
'--alter', 'engine=innodb', '--execute', '--history') }
|
||||
);
|
||||
|
||||
is(
|
||||
$exit,
|
||||
0,
|
||||
'basic test with option --history finished OK'
|
||||
) or diag($output);
|
||||
|
||||
$output = `/tmp/12345/use -N -e "select count(*) from information_schema.tables where TABLE_SCHEMA='percona' and table_name='pt_osc_history'"`;
|
||||
is(
|
||||
$output + 0,
|
||||
1,
|
||||
'--history table created when option --history was provided'
|
||||
);
|
||||
|
||||
($output, $exit) = full_output(
|
||||
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
|
||||
'--alter', 'engine=innodb', '--execute', '--history') }
|
||||
);
|
||||
|
||||
is(
|
||||
$exit,
|
||||
0,
|
||||
'basic test with option --history finished OK when table already exists'
|
||||
) or diag($output);
|
||||
|
||||
$output = `/tmp/12345/use -N -e "select count(*) from information_schema.tables where TABLE_SCHEMA='percona' and table_name='pt_osc_history'"`;
|
||||
is(
|
||||
$output + 0,
|
||||
1,
|
||||
'--history table was created when option --history was provided only once'
|
||||
);
|
||||
|
||||
diag(`/tmp/12345/use -N -e "drop table percona.pt_osc_history"`);
|
||||
|
||||
($output, $exit) = full_output(
|
||||
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
|
||||
'--alter', 'engine=innodb', '--execute', '--history', '--binary-index') }
|
||||
);
|
||||
|
||||
is(
|
||||
$exit,
|
||||
0,
|
||||
'basic test with option --binary-index finished OK'
|
||||
) or diag($output);
|
||||
|
||||
$output = `/tmp/12345/use -N -e "select count(*) from information_schema.tables where TABLE_SCHEMA='percona' and table_name='pt_osc_history'"`;
|
||||
is(
|
||||
$output + 0,
|
||||
1,
|
||||
'--history table was created when option --history and --binary-index were provided'
|
||||
);
|
||||
|
||||
$output = `/tmp/12345/use -e "show create table percona.pt_osc_history"`;
|
||||
like(
|
||||
$output,
|
||||
qr/`lower_boundary` blob,\\n\s+`upper_boundary` blob/i,
|
||||
'--history table created with BLOB data type for boundary columns with --binary-index'
|
||||
);
|
||||
|
||||
($output, $exit) = full_output(
|
||||
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
|
||||
'--alter', 'engine=innodb', '--execute', '--history', '--binary-index') }
|
||||
);
|
||||
|
||||
is(
|
||||
$exit,
|
||||
0,
|
||||
'second with option --binary-index finished OK'
|
||||
) or diag($output);
|
||||
|
||||
$output = `/tmp/12345/use -N -e "select count(*) from information_schema.tables where TABLE_SCHEMA='percona' and table_name='pt_osc_history'"`;
|
||||
is(
|
||||
$output + 0,
|
||||
1,
|
||||
'--history table was created only once with --binary-index'
|
||||
);
|
||||
|
||||
($output, $exit) = full_output(
|
||||
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
|
||||
'--alter', 'engine=innodb', '--execute', '--history') }
|
||||
);
|
||||
|
||||
isnt(
|
||||
$exit,
|
||||
0,
|
||||
'pt_osc with --history failed if table with the same name and different structure exists'
|
||||
) or diag($output);
|
||||
|
||||
diag(`/tmp/12345/use -e "alter table percona.pt_osc_history add column foo int"`);
|
||||
|
||||
($output, $exit) = full_output(
|
||||
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
|
||||
'--alter', 'engine=innodb', '--execute', '--history', '--binary-index') }
|
||||
);
|
||||
|
||||
isnt(
|
||||
$exit,
|
||||
0,
|
||||
'pt_osc with --history and --binary-index failed if table with the same name and different structure exists'
|
||||
) or diag($output);
|
||||
|
||||
# Custom table
|
||||
($output, $exit) = full_output(
|
||||
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
|
||||
'--alter', 'engine=innodb', '--execute', '--history',
|
||||
'--history-table=pt_1717.pt_1717_history') }
|
||||
);
|
||||
|
||||
is(
|
||||
$exit,
|
||||
0,
|
||||
'basic test with option --history-table finished OK'
|
||||
) or diag($output);
|
||||
|
||||
$output = `/tmp/12345/use -N -e "select count(*) from information_schema.tables where TABLE_SCHEMA='pt_1717' and table_name='pt_1717_history'"`;
|
||||
is(
|
||||
$output + 0,
|
||||
1,
|
||||
'Custom --history table created'
|
||||
);
|
||||
|
||||
($output, $exit) = full_output(
|
||||
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
|
||||
'--alter', 'engine=innodb', '--execute', '--history',
|
||||
'--history-table=pt_1717.pt_1717_history') }
|
||||
);
|
||||
|
||||
is(
|
||||
$exit,
|
||||
0,
|
||||
'basic test with option --history-table finished OK when table already exists'
|
||||
) or diag($output);
|
||||
|
||||
$output = `/tmp/12345/use -N -e "select count(*) from information_schema.tables where TABLE_SCHEMA='percona' and table_name='pt_osc_history'"`;
|
||||
is(
|
||||
$output + 0,
|
||||
1,
|
||||
'Custom --history table was created only once'
|
||||
);
|
||||
|
||||
# #############################################################################
|
||||
# Done.
|
||||
# #############################################################################
|
||||
|
||||
$sb->wipe_clean($master_dbh);
|
||||
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
|
||||
#
|
||||
done_testing;
|
||||
Reference in New Issue
Block a user