percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-11 05:29:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #762 from percona/PT-1751_adds_--where_param_to_pt-online-schema-change

Browse Source 
PT-1751 - adds --where param to pt-online-schema-change
This commit is contained in:
Sveta Smirnova
2024-02-05 13:56:16 +03:00
committed by GitHub
parent cf255e01e4 cbc00ad761
commit be880d82d7
2 changed files with 484 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
bin/pt-online-schema-change
View File

@@ -8693,6 +8693,13 @@ sub main {
}
}
if ( $o->get('where') && !$o->get('force')
&& ( $o->get('drop-new-table') || $o->get('swap-tables') ) )
{
_die("Using option --where together with --drop-new-table and --swap-tables may lead to data loss, therefore this operation is only allowed if option --force also specified. Aborting.", UNSUPPORTED_OPERATION);
}
# See the "pod-based-option-value-validation" spec for how this may
# be automagically validated.
if ( $alter_fk_method
@@ -9254,6 +9261,7 @@ sub main {
Cxn => $cxn,
Quoter => $q,
only_same_schema_fks => $o->get('only-same-schema-fks'),
where => $o->get('where'),
);
if ( !$child_tables ) {
if ( $alter_fk_method ) {
@@ -9281,6 +9289,12 @@ sub main {
$child_table->{row_est} || '?';
}
if ( $o->get('where') && !$o->get('force') ) {
print "Child tables found and option --where specified. Rebuilding foreign key constraints may lead to errors.\n";
print "Use --force to override this check\n";
return 1;
}
# TODO: Fix self referencing foreign keys handling.
# See: https://jira.percona.com/browse/PT-1802
# https://jira.percona.com/browse/PT-1853
@@ -9900,6 +9914,7 @@ sub main {
my ($n_rows) = NibbleIterator::get_row_estimate(
Cxn => $slave,
tbl => $tbl,
where => $o->get('where'),
);
PTDEBUG && _d('Table on',$slave->name(),'has', $n_rows, 'rows');
if ( $limit && $n_rows && $n_rows > ($tbl->{chunk_size} * $limit) ) {
@@ -10230,6 +10245,7 @@ sub main {
max_rows => $max_rows,
Cxn => $cxn,
OptionParser => $o,
where => $o->get('where'),
);
if ( $alter_fk_method eq 'drop_swap' ) {
@@ -11183,6 +11199,7 @@ sub find_child_tables {
my ($n_rows) = NibbleIterator::get_row_estimate(
Cxn => $cxn,
tbl => $tbl,
where => $args{where},
);
$tbl->{row_est} = $n_rows;
@@ -11218,6 +11235,7 @@ sub determine_alter_fk_method {
my ($n_rows) = NibbleIterator::get_row_estimate(
Cxn => $cxn,
tbl => $child_tbl,
where => $args{where},
);
if ( $n_rows > $max_rows ) {
print "too many rows: $n_rows; must use drop_swap\n";
@@ -12738,7 +12756,9 @@ duplicate rows and this data will be lost.
=item --force
This options bypasses confirmation in case of using alter-foreign-keys-method = none , which might break foreign key constraints.
This option bypasses confirmation in case of using alter-foreign-keys-method = none, which might break foreign key constraints.
This option also allows to use option --where without options --no-drop-new-table and --no-swap-tables.
=item --help
@@ -13193,6 +13213,23 @@ tool.
For more information, visit L<https://www.percona.com/doc/percona-toolkit/LATEST/version-check.html>.
=item --where
type: string
Copy only rows matching this WHERE clause. You can use this option to limit
the copying to only part of the table. This is particularly useful if you want
to restart pt-online-schema-change job after it was failed.
This option is much like the -w option to mysqldump. Do not specify the WHERE
keyword. You might need to quote the value. Here is an example:
pt-online-schema-change --where "id > 12345678"
IMPORTANT. If used without options --no-drop-new-table and --no-swap-tables
may lead to data loss, therefore this operation only allowed if option --force
also specified.
=item --[no]fail-on-stopped-replication
default: yes

446
t/pt-online-schema-change/pt-1751.t Normal file
View File

@@ -0,0 +1,446 @@
#!/usr/bin/env perl
BEGIN {
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
};
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Test::More;
use PerconaTest;
use Sandbox;
require "$trunk/bin/pt-online-schema-change";
require VersionParser;
use Data::Dumper;
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $master_dbh = $sb->get_dbh_for('master');
my $slave_dbh = $sb->get_dbh_for('slave1');
if ( !$master_dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
elsif ( !$slave_dbh ) {
plan skip_all => 'Cannot connect to sandbox slave';
}
my @args = qw(--set-vars innodb_lock_wait_timeout=3);
my $output = "";
my $dsn = "h=127.1,P=12345,u=msandbox,p=msandbox";
my $exit = 0;
my $sample = "t/pt-online-schema-change/samples";
$sb->load_file('master', "$sample/basic_no_fks_innodb.sql");
# #############################################################################
# --where does not run without --no-drop-new-table and --no-swap-tables
# unless option --force also specified
# #############################################################################
($output, $exit) = full_output(
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
'--alter', 'drop column id', '--where', 'id > 10', '--execute') }
);
like(
$output,
qr/Using option --where together with --drop-new-table and --swap-tables may lead to data loss, therefore this operation is only allowed if option --force also specified. Aborting./i,
'Did not run with --where and without --no-drop-new-table, --no-swap-tables and --force'
) or diag($output);
is(
$exit,
17,
'Exit code 17 (UNSUPPORTED_OPERATION) with --where and without --no-drop-new-table, --no-swap-tables and --force'
);
($output, $exit) = full_output(
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
'--alter', 'drop column id', '--where', 'id > 10', '--execute',
'--no-drop-new-table') }
);
like(
$output,
qr/Using option --where together with --drop-new-table and --swap-tables may lead to data loss, therefore this operation is only allowed if option --force also specified. Aborting./i,
'Did not run with --where and without --no-swap-tables and --force'
) or diag($output);
is(
$exit,
17,
'Exit code 17 (UNSUPPORTED_OPERATION) with --where and without --no-swap-tables and --force'
);
($output, $exit) = full_output(
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
'--alter', 'drop column id', '--where', 'id > 10', '--execute',
'--no-swap-tables') }
);
like(
$output,
qr/Using option --where together with --drop-new-table and --swap-tables may lead to data loss, therefore this operation is only allowed if option --force also specified. Aborting./i,
'Did not run with --where and without --no-drop-new-table and --force'
) or diag($output);
is(
$exit,
17,
'Exit code 17 (UNSUPPORTED_OPERATION) with --where and without --no-drop-new-table and --force'
);
# #############################################################################
# Multiple situations when option --where works
# #############################################################################
$output = output(
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
'--alter', 'add column e int', '--where', 'id > 10', '--execute',
'--no-swap-tables', '--no-drop-new-table',
'--new-table-name', 't_pt_1751') }
);
like(
$output,
qr/Successfully altered/i,
'Option --where runs with --no-drop-new-table and --no-swap-tables'
) or diag($output);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.t_pt_1751"`;
is(
$output + 0,
10,
'Only 10 rows copied'
);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.t_pt_1751 where id <= 10"`;
is(
$output + 0,
0,
'Rows, satisfying --where condition are not copied'
) or diag($output);
$sb->load_file('master', "$sample/basic_no_fks_innodb.sql");
$output = output(
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
'--alter', 'add column e int', '--where', 'id > 10', '--execute',
'--force') }
);
like(
$output,
qr/Successfully altered/i,
'Option --where runs with --force'
) or diag($output);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.t"`;
is(
$output + 0,
10,
'Only 10 rows copied'
);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.t where id <= 10"`;
is(
$output + 0,
0,
'Rows, satisfying --where condition are not copied'
) or diag($output);
# Same test with chunk size = 1
$sb->load_file('master', "$sample/basic_no_fks_innodb.sql");
$output = output(
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
'--alter', 'add column e int', '--where', 'id > 10', '--execute',
'--no-swap-tables', '--no-drop-new-table', '--chunk-size', '1',
'--new-table-name', 't_pt_1751') }
);
like(
$output,
qr/Successfully altered/i,
'Option --where runs with --no-drop-new-table and --no-swap-tables'
) or diag($output);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.t_pt_1751"`;
is(
$output + 0,
10,
'Only 10 rows copied'
);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.t_pt_1751 where id <= 10"`;
is(
$output + 0,
0,
'Rows, satisfying --where condition are not copied'
) or diag($output);
$sb->load_file('master', "$sample/basic_no_fks_innodb.sql");
$output = output(
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
'--alter', 'add column e int', '--where', 'id > 10', '--execute',
'--force', '--chunk-size', '1',) }
);
like(
$output,
qr/Successfully altered/i,
'Option --where runs with --force'
) or diag($output);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.t"`;
is(
$output + 0,
10,
'Only 10 rows copied'
);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.t where id <= 10"`;
is(
$output + 0,
0,
'Rows, satisfying --where condition are not copied'
) or diag($output);
# #############################################################################
# Option --where and foreign keys
# #############################################################################
$sb->load_file('master', "$sample/basic_with_fks.sql");
$output = output(
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=city",
'--alter', 'drop column last_update', '--where', 'city_id >= 3', '--execute',
'--alter-foreign-keys-method', 'rebuild_constraints', '--force') }
);
like(
$output,
qr/Successfully altered/i,
'Option --where runs with --force and --alter-foreign-keys-method=rebuild_constraints'
) or diag($output);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.city"`;
is(
$output + 0,
3,
'Only 3 rows copied'
);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.city where city_id < 3"`;
is(
$output + 0,
0,
'Rows, satisfying --where condition are not copied'
) or diag($output);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.country"`;
is(
$output + 0,
5,
'Table country not corrupted'
) or diag($output);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.address"`;
is(
$output + 0,
5,
'Table address not modified'
) or diag($output);
$sb->load_file('master', "$sample/basic_with_fks.sql");
$output = output(
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=city",
'--alter', 'drop column last_update', '--where', 'city_id >= 3', '--execute',
'--alter-foreign-keys-method', 'auto', '--force') }
);
like(
$output,
qr/Successfully altered/i,
'Option --where runs with --force and --alter-foreign-keys-method=auto'
) or diag($output);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.city"`;
is(
$output + 0,
3,
'Only 3 rows copied'
);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.city where city_id < 3"`;
is(
$output + 0,
0,
'Rows, satisfying --where condition are not copied'
) or diag($output);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.country"`;
is(
$output + 0,
5,
'Table country not corrupted'
) or diag($output);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.address"`;
is(
$output + 0,
5,
'Table address not modified'
) or diag($output);
$sb->load_file('master', "$sample/basic_with_fks.sql");
$output = output(
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=city",
'--alter', 'drop column last_update', '--where', 'city_id >= 3', '--execute',
'--alter-foreign-keys-method', 'drop_swap', '--force') }
);
like(
$output,
qr/Successfully altered/i,
'Option --where runs with --force and --alter-foreign-keys-method=drop_swap'
) or diag($output);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.city"`;
is(
$output + 0,
3,
'Only 3 rows copied'
);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.city where city_id < 3"`;
is(
$output + 0,
0,
'Rows, satisfying --where condition are not copied'
) or diag($output);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.country"`;
is(
$output + 0,
5,
'Table country not corrupted'
) or diag($output);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.address"`;
is(
$output + 0,
5,
'Table address not modified'
) or diag($output);
$sb->load_file('master', "$sample/basic_with_fks.sql");
$output = output(
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=city",
'--alter', 'drop column last_update', '--where', 'city_id >= 3', '--execute',
'--alter-foreign-keys-method', 'none', '--force') }
);
like(
$output,
qr/Successfully altered/i,
'Option --where runs with --force and --alter-foreign-keys-method=none'
) or diag($output);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.city"`;
is(
$output + 0,
3,
'Only 3 rows copied'
);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.city where city_id < 3"`;
is(
$output + 0,
0,
'Rows, satisfying --where condition are not copied'
) or diag($output);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.country"`;
is(
$output + 0,
5,
'Table country not corrupted'
) or diag($output);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.address"`;
is(
$output + 0,
5,
'Table address not modified'
) or diag($output);
$sb->load_file('master', "$sample/basic_with_fks.sql");
($output, $exit) = full_output(
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=city",
'--alter', 'drop column last_update', '--where', 'city_id >= 3', '--execute',
'--alter-foreign-keys-method', 'rebuild_constraints',
'--no-drop-new-table', '--no-swap-tables') }
);
like(
$output,
qr/Child tables found and option --where specified. Rebuilding foreign key constraints may lead to errors./i,
'Option --where does not run without --force and --alter-foreign-keys-method=rebuild_constraints when child tables are found'
) or diag($output);
is(
$exit,
1,
'Exit code 1 with --where and child tables'
);
$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.address"`;
is(
$output + 0,
5,
'Table address not modified'
) or diag($output);
$sb->load_file('master', "$sample/basic_with_fks.sql");
($output, $exit) = full_output(
sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=city",
'--alter', 'drop column last_update', '--where', 'city_id >= 3', '--execute',
'--alter-foreign-keys-method', 'auto',
'--no-drop-new-table', '--no-swap-tables') }
);
like(
$output,
qr/Child tables found and option --where specified. Rebuilding foreign key constraints may lead to errors./i,
'Option --where does not run without --force and --alter-foreign-keys-method=auto when child tables are found'
) or diag($output);
is(
$exit,
1,
'Exit code 1 with --where and child tables'
);
# #############################################################################
# Done.
# #############################################################################
$sb->wipe_clean($master_dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
#
done_testing;