percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-17 17:27:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '3.0' into impimp-PT-1824

Browse Source
This commit is contained in:
Carlos Salguero
2020-04-16 13:26:28 -03:00
parent 76cffde87e 4ba96ad7f1
commit 1038816275
6 changed files with 491 additions and 440 deletions
Download Patch File Download Diff File Expand all files Collapse all files

258
bin/pt-online-schema-change
View File

@@ -8376,7 +8376,6 @@ my $dont_interrupt_now = 0;
my @drop_trigger_sqls;
my @triggers_not_dropped;
my $pxc_version = '0';
my $can_drop_triggers = 1;
my $triggers_info = [];
@@ -9551,7 +9550,6 @@ sub main {
# called which will drop whichever triggers were created.
my $drop_triggers = $o->get('drop-triggers');
push @cleanup_tasks, sub {
PTDEBUG && _d('Clean up triggers');
# --plugin hook
if ( $plugin && $plugin->can('before_drop_triggers') ) {
@@ -9562,12 +9560,12 @@ sub main {
);
}
if ( !$oktorun || !_can_drop_triggers()) {
if ( !$oktorun ) {
print "Not dropping triggers because the tool was interrupted. "
. "To drop the triggers, execute:\n"
. join("\n", @drop_trigger_sqls) . "\n";
}
elsif ( !$drop_triggers || !_can_drop_triggers()) {
elsif ( !$drop_triggers ) {
print "Not dropping triggers because --no-drop-triggers was "
. "specified. To drop the triggers, execute:\n"
. join("\n", @drop_trigger_sqls) . "\n";
@@ -10000,76 +9998,6 @@ sub main {
$plugin->after_copy_rows();
}
# #####################################################################
# Step 5a: Update foreign key constraints if there are child tables.
# #####################################################################
if ( $child_tables ) {
# --plugin hook
if ( $plugin && $plugin->can('before_update_foreign_keys') ) {
$plugin->before_update_foreign_keys();
}
eval {
if ( $alter_fk_method eq 'none' ) {
# This shouldn't happen, but in case it does we should know.
warn "The tool detected child tables but "
. "--alter-foreign-keys-method=none";
}
elsif ( $alter_fk_method eq 'rebuild_constraints' ) {
rebuild_constraints(
orig_tbl => $new_tbl,
old_tbl => $orig_tbl,
# orig_tbl => $orig_tbl,
# old_tbl => $old_tbl,
child_tables => $child_tables,
OptionParser => $o,
Quoter => $q,
Cxn => $cxn,
TableParser => $tp,
stats => \%stats,
Retry => $retry,
tries => $tries,
);
}
elsif ( $alter_fk_method eq 'drop_swap' ) {
drop_swap(
orig_tbl => $new_tbl,
new_tbl => $orig_tbl,
# orig_tbl => $orig_tbl,
# new_tbl => $new_tbl,
Cxn => $cxn,
OptionParser => $o,
stats => \%stats,
Retry => $retry,
tries => $tries,
analyze_table => $analyze_table,
);
}
elsif ( !$alter_fk_method
&& $o->has('alter-foreign-keys-method')
&& ($o->get('alter-foreign-keys-method') || '') eq 'auto' ) {
# If --alter-foreign-keys-method is 'auto' and we are on a dry run,
# $alter_fk_method is left as an empty string.
print "Not updating foreign key constraints because this is a dry run.\n";
}
else {
# This should "never" happen because we check this var earlier.
_die("Invalid --alter-foreign-keys-method: $alter_fk_method", INVALID_ALTER_FK_METHOD);
}
};
if ( $EVAL_ERROR ) {
# TODO: improve error message and handling.
$can_drop_triggers=undef;
$oktorun=undef;
_die("Error updating foreign key constraints: $EVAL_ERROR", ERROR_UPDATING_FKS);
}
# --plugin hook
if ( $plugin && $plugin->can('after_update_foreign_keys') ) {
$plugin->after_update_foreign_keys();
}
}
# #####################################################################
# XXX
# Step 5: Rename tables: orig -> old, new -> orig
@@ -10164,66 +10092,66 @@ sub main {
# #####################################################################
# Step 6: Update foreign key constraints if there are child tables.
# #####################################################################
# if ( $child_tables ) {
# # --plugin hook
# if ( $plugin && $plugin->can('before_update_foreign_keys') ) {
# $plugin->before_update_foreign_keys();
# }
#
# eval {
# if ( $alter_fk_method eq 'none' ) {
# # This shouldn't happen, but in case it does we should know.
# warn "The tool detected child tables but "
# . "--alter-foreign-keys-method=none";
# }
# elsif ( $alter_fk_method eq 'rebuild_constraints' ) {
# rebuild_constraints(
# orig_tbl => $orig_tbl,
# old_tbl => $old_tbl,
# child_tables => $child_tables,
# OptionParser => $o,
# Quoter => $q,
# Cxn => $cxn,
# TableParser => $tp,
# stats => \%stats,
# Retry => $retry,
# tries => $tries,
# );
# }
# elsif ( $alter_fk_method eq 'drop_swap' ) {
# drop_swap(
# orig_tbl => $orig_tbl,
# new_tbl => $new_tbl,
# Cxn => $cxn,
# OptionParser => $o,
# stats => \%stats,
# Retry => $retry,
# tries => $tries,
# analyze_table => $analyze_table,
# );
# }
# elsif ( !$alter_fk_method
# && $o->has('alter-foreign-keys-method')
# && ($o->get('alter-foreign-keys-method') || '') eq 'auto' ) {
# # If --alter-foreign-keys-method is 'auto' and we are on a dry run,
# # $alter_fk_method is left as an empty string.
# print "Not updating foreign key constraints because this is a dry run.\n";
# }
# else {
# # This should "never" happen because we check this var earlier.
# _die("Invalid --alter-foreign-keys-method: $alter_fk_method", INVALID_ALTER_FK_METHOD);
# }
# };
# if ( $EVAL_ERROR ) {
# # TODO: improve error message and handling.
# _die("Error updating foreign key constraints: $EVAL_ERROR", ERROR_UPDATING_FKS);
# }
#
# # --plugin hook
# if ( $plugin && $plugin->can('after_update_foreign_keys') ) {
# $plugin->after_update_foreign_keys();
# }
# }
if ( $child_tables ) {
# --plugin hook
if ( $plugin && $plugin->can('before_update_foreign_keys') ) {
$plugin->before_update_foreign_keys();
}
eval {
if ( $alter_fk_method eq 'none' ) {
# This shouldn't happen, but in case it does we should know.
warn "The tool detected child tables but "
. "--alter-foreign-keys-method=none";
}
elsif ( $alter_fk_method eq 'rebuild_constraints' ) {
rebuild_constraints(
orig_tbl => $orig_tbl,
old_tbl => $old_tbl,
child_tables => $child_tables,
OptionParser => $o,
Quoter => $q,
Cxn => $cxn,
TableParser => $tp,
stats => \%stats,
Retry => $retry,
tries => $tries,
);
}
elsif ( $alter_fk_method eq 'drop_swap' ) {
drop_swap(
orig_tbl => $orig_tbl,
new_tbl => $new_tbl,
Cxn => $cxn,
OptionParser => $o,
stats => \%stats,
Retry => $retry,
tries => $tries,
analyze_table => $analyze_table,
);
}
elsif ( !$alter_fk_method
&& $o->has('alter-foreign-keys-method')
&& ($o->get('alter-foreign-keys-method') || '') eq 'auto' ) {
# If --alter-foreign-keys-method is 'auto' and we are on a dry run,
# $alter_fk_method is left as an empty string.
print "Not updating foreign key constraints because this is a dry run.\n";
}
else {
# This should "never" happen because we check this var earlier.
_die("Invalid --alter-foreign-keys-method: $alter_fk_method", INVALID_ALTER_FK_METHOD);
}
};
if ( $EVAL_ERROR ) {
# TODO: improve error message and handling.
_die("Error updating foreign key constraints: $EVAL_ERROR", ERROR_UPDATING_FKS);
}
# --plugin hook
if ( $plugin && $plugin->can('after_update_foreign_keys') ) {
$plugin->after_update_foreign_keys();
}
}
# ########################################################################
# Step 7: Drop the old table.
@@ -10297,10 +10225,6 @@ sub main {
# Subroutines.
# ############################################################################
sub _can_drop_triggers {
return $can_drop_triggers;
}
sub validate_tries {
my ($o) = @_;
my @ops = qw(
@@ -11082,8 +11006,6 @@ sub rebuild_constraints {
print ts("Rebuilding foreign key constraints...\n");
}
my $foreign_key_checks;
CHILD_TABLE:
foreach my $child_tbl ( @$child_tables ) {
my $table_def = $tp->get_create_table(
@@ -11144,22 +11066,15 @@ sub rebuild_constraints {
$constraint =~ s/CONSTRAINT `$fk`/CONSTRAINT `$new_fk`/;
my $sql = "DROP FOREIGN KEY `$fk`, ADD $constraint";
my $sql = "DROP FOREIGN KEY `$fk`, "
. "ADD $constraint";
push @rebuilt_constraints, $sql;
}
my $server_version = VersionParser->new($cxn->dbh());
if ($server_version >= '5.6' && $o->get('disable-fk-checks')) {
my $row = $cxn->dbh()->selectrow_arrayref('SELECT @@foreign_key_checks');
$foreign_key_checks = @$row[0];
ts("Disabling FK checks");
$cxn->dbh()->do("SET FOREIGN_KEY_CHECKS=0");
}
my $sql = "ALTER TABLE $child_tbl->{name} "
. join(', ', @rebuilt_constraints);
print $sql, "\n" if $o->get('print');
if ($o->get('execute')) {
eval {
if ( $o->get('execute') ) {
osc_retry(
Cxn => $cxn,
Retry => $retry,
@@ -11171,20 +11086,6 @@ sub rebuild_constraints {
$stats->{rebuilt_constraint}++;
},
);
};
if ($foreign_key_checks) {
ts("Re-enabling FK checks");
$cxn->dbh()->do("SET FOREIGN_KEY_CHECKS=$foreign_key_checks");
}
if ($EVAL_ERROR) {
$can_drop_triggers=undef;
$oktorun=undef;
_d("Foreing keys rebuild failed. To rebuild constraints please manually run:");
foreach my $cmd (@rebuilt_constraints) {
print "$cmd\n";
}
_die("Foreing keys were not rebuilt");
}
}
}
@@ -11311,11 +11212,10 @@ sub create_triggers {
"$new_tbl->{name}.$new_qcol <=> OLD.$old_qcol"
} @{$tbl_struct->{keys}->{$del_index}->{cols}} );
my $ignore_clause = $o->get("delete-ignore") ? "IGNORE" : "";
my $delete_trigger
= "CREATE TRIGGER `${prefix}_del` AFTER DELETE ON $orig_tbl->{name} "
. "FOR EACH ROW "
. "DELETE $ignore_clause FROM $new_tbl->{name} "
. "DELETE IGNORE FROM $new_tbl->{name} "
. "WHERE $del_index_cols";
# ---------------------------------------------------------------------------------------
@@ -11341,7 +11241,7 @@ sub create_triggers {
= "CREATE TRIGGER `${prefix}_upd` AFTER UPDATE ON $orig_tbl->{name} "
. "FOR EACH ROW "
. "BEGIN "
. "DELETE $ignore_clause FROM $new_tbl->{name} WHERE !($upd_index_cols) AND $del_index_cols;"
. "DELETE IGNORE FROM $new_tbl->{name} WHERE !($upd_index_cols) AND $del_index_cols;"
. "REPLACE INTO $new_tbl->{name} ($qcols) VALUES ($new_vals);"
. "END ";
@@ -11640,11 +11540,6 @@ sub osc_retry {
) {
# These errors/warnings can be retried, so don't print
# a warning yet; do that in final_fail.
# If we found a lock wait timeout and $tries == 0, we are in the rebuilt_constraints part
# so we should keep retrying
if ($error =~ m/Lock wait timeout exceeded/ && !$tries->{tries}) {
return 1;
}
$stats->{ error_event($error) }++;
return 1; # try again
}
@@ -11967,10 +11862,6 @@ foreign keys and indexes as the original table (unless you specify differently
in your ALTER statement), but the names of the objects may be changed slightly
to avoid object name collisions in MySQL and InnoDB.
You should take into consideration that data consistency is not maintained by
default if you have FKs, and you must pass this additional new argument to
avoid data loss. Please read L<"--delete-ignore">
For safety, the tool does not modify the table unless you specify the
L<"--execute"> option, which is not enabled by default. The tool supports a
variety of other measures to prevent unwanted load or other problems, including
@@ -12488,21 +12379,6 @@ short form: -F; type: string
Only read mysql options from the given file. You must give an absolute
pathname.
=item --[no]delete-ignore
default: yes
Do not use C<IGNORE> on C<DELETE> triggers.
This is an special case that affects forign keys handling. Since C<DELETE> only return errors when
there is a problem deleting rows affected by foreign keys, it might be cases when we want to
receive such errors during the program execution to ensure FKs handling is correct.
=item --disable-fk-checks
Disable foreign keys check during rebuild constraints. This option improves the process speed
but it is risky since FKs checks will be disabled for a short period of time, allowing invalid
values in tables. Please use it carefully.
=item --[no]drop-new-table
default: yes

1
t/pt-online-schema-change/ansi_quotes.t
View File

@@ -51,6 +51,7 @@ ok(
);
SKIP: {
skip "Skipping in MySQL 8.0.4-rc since there is an error in the server itself. See https://bugs.mysql.com/bug.php?id=89441", 9 if ($sandbox_version ge '8.0');
($output, $exit_status) = full_output(
sub { pt_online_schema_change::main(@args,
"$master_dsn,D=issue26211,t=process_model_inst",

4
t/pt-online-schema-change/rename_fk_constraints.t
View File

@@ -23,6 +23,10 @@ my $master_dbh = $sb->get_dbh_for('master');
my $vp = VersionParser->new($master_dbh);
if ($vp->cmp('8.0.14') > -1 && $vp->flavor() !~ m/maria/i) {
plan skip_all => 'Cannot run this test under the current MySQL version';
}
if ( !$master_dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}

45
t/pt-online-schema-change/rename_self_ref_fk_constraints.t
View File

@@ -32,7 +32,6 @@ my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox';
my @args = (qw(--set-vars innodb_lock_wait_timeout=3 --alter-foreign-keys-method rebuild_constraints));
my $output;
my $exit_status;
my $sample = "t/pt-online-schema-change/samples/";
# ############################################################################
# https://bugs.launchpad.net/percona-toolkit/+bug/1632522
@@ -40,7 +39,7 @@ my $sample = "t/pt-online-schema-change/samples/";
# ############################################################################
diag("Before loading sql");
$sb->load_file('master', "$sample/bug-1632522.sql");
$sb->load_file('master', "t/pt-online-schema-change/samples/bug-1632522.sql");
diag("after loading sql");
# run once: we expect the constraint name to be appended with one underscore
@@ -52,14 +51,22 @@ diag("after loading sql");
qw(--execute)) },
);
my $constraints = $master_dbh->selectall_arrayref("SELECT TABLE_NAME, CONSTRAINT_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE table_schema='bug1632522' and (TABLE_NAME='test_table' OR TABLE_NAME='person') and CONSTRAINT_NAME LIKE '%fk_%' ORDER BY TABLE_NAME, CONSTRAINT_NAME");
my $query = <<"END";
SELECT TABLE_NAME, CONSTRAINT_NAME
FROM information_schema.KEY_COLUMN_USAGE
WHERE table_schema='bug1632522'
AND (TABLE_NAME='test_table' OR TABLE_NAME='person')
AND CONSTRAINT_NAME LIKE '%fk_%'
ORDER BY TABLE_NAME, CONSTRAINT_NAME
END
my $constraints = $master_dbh->selectall_arrayref($query);
is_deeply(
$constraints,
[
['person', '_fk_testId'],
['test_table', 'fk_person'],
['test_table', 'fk_refId'],
['test_table', '_fk_person'],
['test_table', '__fk_refId'],
],
"First run adds or removes underscore from constraint names, accordingly"
);
@@ -73,15 +80,23 @@ is_deeply(
qw(--execute)) },
);
$constraints = $master_dbh->selectall_arrayref("SELECT TABLE_NAME, CONSTRAINT_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE table_schema='bug1632522' and (TABLE_NAME='test_table' OR TABLE_NAME='person') and CONSTRAINT_NAME LIKE '%fk_%' ORDER BY TABLE_NAME, CONSTRAINT_NAME");
$query = <<"END";
SELECT TABLE_NAME, CONSTRAINT_NAME
FROM information_schema.KEY_COLUMN_USAGE
WHERE table_schema='bug1632522'
AND (TABLE_NAME='test_table' OR TABLE_NAME='person')
AND CONSTRAINT_NAME LIKE '%fk_%'
ORDER BY TABLE_NAME, CONSTRAINT_NAME
END
$constraints = $master_dbh->selectall_arrayref($query);
is_deeply(
$constraints,
[
['person', '_fk_testId'],
['test_table', 'fk_person'],
['test_table', 'fk_refId'],
['person', '__fk_testId'],
['test_table', '_fk_refId'],
['test_table', '__fk_person'],
],
"Second run self-referencing will be one due to rebuild_constraints"
);
@@ -94,13 +109,21 @@ is_deeply(
qw(--execute)) },
);
$constraints = $master_dbh->selectall_arrayref("SELECT TABLE_NAME, CONSTRAINT_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE table_schema='bug1632522' and (TABLE_NAME='test_table' OR TABLE_NAME='person') and CONSTRAINT_NAME LIKE '%fk_%' ORDER BY TABLE_NAME, CONSTRAINT_NAME");
$query = <<"END";
SELECT TABLE_NAME, CONSTRAINT_NAME
FROM information_schema.KEY_COLUMN_USAGE
WHERE table_schema='bug1632522'
and (TABLE_NAME='test_table' OR TABLE_NAME='person')
and CONSTRAINT_NAME LIKE '%fk_%'
ORDER BY TABLE_NAME, CONSTRAINT_NAME
END
$constraints = $master_dbh->selectall_arrayref($query);
is_deeply(
$constraints,
[
['person', '_fk_testId'],
['person', 'fk_testId'],
['test_table', 'fk_person'],
['test_table', 'fk_refId'],
],

138
t/pt-online-schema-change/rename_self_ref_fk_constraints_pt_1802.t Normal file
View File

@@ -0,0 +1,138 @@
#!/usr/bin/env perl
BEGIN {
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
};
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Test::More;
use Data::Dumper;
use PerconaTest;
use Sandbox;
require "$trunk/bin/pt-online-schema-change";
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $master_dbh = $sb->get_dbh_for('master');
if ( !$master_dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
# so we need to specify --set-vars innodb_lock_wait_timeout-3 else the
# tool will die.
my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox';
my @args = (qw(--set-vars innodb_lock_wait_timeout=3 --alter-foreign-keys-method rebuild_constraints));
my $output;
my $exit_status;
# ############################################################################
# https://bugs.launchpad.net/percona-toolkit/+bug/1632522
# pt-online-schema-change fails with duplicate key in table for self-referencing FK
# ############################################################################
diag("Before loading sql");
$sb->load_file('master', "t/pt-online-schema-change/samples/bug-1632522.sql");
diag("after loading sql");
# run once: we expect the constraint name to be appended with one underscore
# but the self-referencing constraint will have 2 underscore
($output, $exit_status) = full_output(
sub { pt_online_schema_change::main(@args,
"$master_dsn,D=bug1632522,t=test_table",
"--alter", "ENGINE=InnoDB",
qw(--execute)) },
);
my $query = <<"END";
SELECT TABLE_NAME, CONSTRAINT_NAME
FROM information_schema.KEY_COLUMN_USAGE
WHERE table_schema='bug1632522'
AND (TABLE_NAME='test_table' OR TABLE_NAME='person')
AND CONSTRAINT_NAME LIKE '%fk_%'
ORDER BY TABLE_NAME, CONSTRAINT_NAME
END
my $constraints = $master_dbh->selectall_arrayref($query);
is_deeply(
$constraints,
[
['person', '_fk_testId'],
['test_table', 'fk_person'],
['test_table', 'fk_refId'],
],
"First run adds or removes underscore from constraint names, accordingly"
);
# run second time: we expect constraint names to be prefixed with one underscore
# if they havre't one, and to remove 2 if they have 2
($output, $exit_status) = full_output(
sub { pt_online_schema_change::main(@args,
"$master_dsn,D=bug1632522,t=test_table",
"--alter", "ENGINE=InnoDB",
qw(--execute)) },
);
$query = <<"END";
SELECT TABLE_NAME, CONSTRAINT_NAME
FROM information_schema.KEY_COLUMN_USAGE
WHERE table_schema='bug1632522'
AND (TABLE_NAME='test_table' OR TABLE_NAME='person')
AND CONSTRAINT_NAME LIKE '%fk_%'
ORDER BY TABLE_NAME, CONSTRAINT_NAME
END
$constraints = $master_dbh->selectall_arrayref($query);
is_deeply(
$constraints,
[
['person', '_fk_testId'],
['test_table', 'fk_person'],
['test_table', 'fk_refId'],
],
"Second run self-referencing will be one due to rebuild_constraints"
);
# run third time: we expect constraints to be the same as we started (toggled back)
($output, $exit_status) = full_output(
sub { pt_online_schema_change::main(@args,
"$master_dsn,D=bug1632522,t=test_table",
"--alter", "ENGINE=InnoDB",
qw(--execute)) },
);
$query = <<"END";
SELECT TABLE_NAME, CONSTRAINT_NAME
FROM information_schema.KEY_COLUMN_USAGE
WHERE table_schema='bug1632522'
and (TABLE_NAME='test_table' OR TABLE_NAME='person')
and CONSTRAINT_NAME LIKE '%fk_%'
ORDER BY TABLE_NAME, CONSTRAINT_NAME
END
$constraints = $master_dbh->selectall_arrayref($query);
is_deeply(
$constraints,
[
['person', '_fk_testId'],
['test_table', 'fk_person'],
['test_table', 'fk_refId'],
],
"Third run toggles constraint names back to how they were"
);
# #############################################################################
# Done.
# #############################################################################
$sb->wipe_clean($master_dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
done_testing;

9
t/pt-online-schema-change/samples/pt-1802.sql Normal file
View File

@@ -0,0 +1,9 @@
CREATE TABLE `joinit` (
`i` int(11) NOT NULL AUTO_INCREMENT,
`s` varchar(64) DEFAULT NULL,
`t` time NOT NULL,
`g` int(11) NOT NULL,
`j` int(11) NOT NULL DEFAULT 1,
PRIMARY KEY (`i`))
ENGINE=InnoDB;
ALTER TABLE joinit ADD FOREIGN KEY i_fk (j) REFERENCES joinit (i) ON UPDATE cascade ON DELETE restrict;