diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index 6ffa5762..c883c75e 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -8693,6 +8693,13 @@ sub main { } } + if ( $o->get('where') && !$o->get('force') + && ( $o->get('drop-new-table') || $o->get('swap-tables') ) ) + { + _die("Using option --where together with --drop-new-table and --swap-tables may lead to data loss, therefore this operation is only allowed if option --force also specified. Aborting.", UNSUPPORTED_OPERATION); + } + + # See the "pod-based-option-value-validation" spec for how this may # be automagically validated. if ( $alter_fk_method @@ -9254,6 +9261,7 @@ sub main { Cxn => $cxn, Quoter => $q, only_same_schema_fks => $o->get('only-same-schema-fks'), + where => $o->get('where'), ); if ( !$child_tables ) { if ( $alter_fk_method ) { @@ -9281,6 +9289,12 @@ sub main { $child_table->{row_est} || '?'; } + if ( $o->get('where') && !$o->get('force') ) { + print "Child tables found and option --where specified. Rebuilding foreign key constraints may lead to errors.\n"; + print "Use --force to override this check\n"; + return 1; + } + # TODO: Fix self referencing foreign keys handling. # See: https://jira.percona.com/browse/PT-1802 # https://jira.percona.com/browse/PT-1853 @@ -9900,6 +9914,7 @@ sub main { my ($n_rows) = NibbleIterator::get_row_estimate( Cxn => $slave, tbl => $tbl, + where => $o->get('where'), ); PTDEBUG && _d('Table on',$slave->name(),'has', $n_rows, 'rows'); if ( $limit && $n_rows && $n_rows > ($tbl->{chunk_size} * $limit) ) { @@ -10230,6 +10245,7 @@ sub main { max_rows => $max_rows, Cxn => $cxn, OptionParser => $o, + where => $o->get('where'), ); if ( $alter_fk_method eq 'drop_swap' ) { @@ -11183,6 +11199,7 @@ sub find_child_tables { my ($n_rows) = NibbleIterator::get_row_estimate( Cxn => $cxn, tbl => $tbl, + where => $args{where}, ); $tbl->{row_est} = $n_rows; @@ -11218,6 +11235,7 @@ sub determine_alter_fk_method { my ($n_rows) = NibbleIterator::get_row_estimate( Cxn => $cxn, tbl => $child_tbl, + where => $args{where}, ); if ( $n_rows > $max_rows ) { print "too many rows: $n_rows; must use drop_swap\n"; @@ -12738,7 +12756,9 @@ duplicate rows and this data will be lost. =item --force -This options bypasses confirmation in case of using alter-foreign-keys-method = none , which might break foreign key constraints. +This option bypasses confirmation in case of using alter-foreign-keys-method = none, which might break foreign key constraints. + +This option also allows to use option --where without options --no-drop-new-table and --no-swap-tables. =item --help @@ -13193,6 +13213,23 @@ tool. For more information, visit L. +=item --where + +type: string + +Copy only rows matching this WHERE clause. You can use this option to limit +the copying to only part of the table. This is particularly useful if you want +to restart pt-online-schema-change job after it was failed. + +This option is much like the -w option to mysqldump. Do not specify the WHERE +keyword. You might need to quote the value. Here is an example: + + pt-online-schema-change --where "id > 12345678" + +IMPORTANT. If used without options --no-drop-new-table and --no-swap-tables +may lead to data loss, therefore this operation only allowed if option --force +also specified. + =item --[no]fail-on-stopped-replication default: yes diff --git a/t/pt-online-schema-change/pt-1751.t b/t/pt-online-schema-change/pt-1751.t new file mode 100644 index 00000000..4d9cb92b --- /dev/null +++ b/t/pt-online-schema-change/pt-1751.t @@ -0,0 +1,446 @@ +#!/usr/bin/env perl + +BEGIN { + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; +}; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use Test::More; + +use PerconaTest; +use Sandbox; +require "$trunk/bin/pt-online-schema-change"; +require VersionParser; + +use Data::Dumper; + +my $dp = new DSNParser(opts=>$dsn_opts); +my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); +my $master_dbh = $sb->get_dbh_for('master'); +my $slave_dbh = $sb->get_dbh_for('slave1'); + +if ( !$master_dbh ) { + plan skip_all => 'Cannot connect to sandbox master'; +} +elsif ( !$slave_dbh ) { + plan skip_all => 'Cannot connect to sandbox slave'; +} + +my @args = qw(--set-vars innodb_lock_wait_timeout=3); +my $output = ""; +my $dsn = "h=127.1,P=12345,u=msandbox,p=msandbox"; +my $exit = 0; +my $sample = "t/pt-online-schema-change/samples"; + +$sb->load_file('master', "$sample/basic_no_fks_innodb.sql"); + +# ############################################################################# +# --where does not run without --no-drop-new-table and --no-swap-tables +# unless option --force also specified +# ############################################################################# + +($output, $exit) = full_output( + sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t", + '--alter', 'drop column id', '--where', 'id > 10', '--execute') } +); + +like( + $output, + qr/Using option --where together with --drop-new-table and --swap-tables may lead to data loss, therefore this operation is only allowed if option --force also specified. Aborting./i, + 'Did not run with --where and without --no-drop-new-table, --no-swap-tables and --force' +) or diag($output); + +is( + $exit, + 17, + 'Exit code 17 (UNSUPPORTED_OPERATION) with --where and without --no-drop-new-table, --no-swap-tables and --force' +); + +($output, $exit) = full_output( + sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t", + '--alter', 'drop column id', '--where', 'id > 10', '--execute', + '--no-drop-new-table') } +); + +like( + $output, + qr/Using option --where together with --drop-new-table and --swap-tables may lead to data loss, therefore this operation is only allowed if option --force also specified. Aborting./i, + 'Did not run with --where and without --no-swap-tables and --force' +) or diag($output); + +is( + $exit, + 17, + 'Exit code 17 (UNSUPPORTED_OPERATION) with --where and without --no-swap-tables and --force' +); + +($output, $exit) = full_output( + sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t", + '--alter', 'drop column id', '--where', 'id > 10', '--execute', + '--no-swap-tables') } +); + +like( + $output, + qr/Using option --where together with --drop-new-table and --swap-tables may lead to data loss, therefore this operation is only allowed if option --force also specified. Aborting./i, + 'Did not run with --where and without --no-drop-new-table and --force' +) or diag($output); + +is( + $exit, + 17, + 'Exit code 17 (UNSUPPORTED_OPERATION) with --where and without --no-drop-new-table and --force' +); + +# ############################################################################# +# Multiple situations when option --where works +# ############################################################################# + +$output = output( + sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t", + '--alter', 'add column e int', '--where', 'id > 10', '--execute', + '--no-swap-tables', '--no-drop-new-table', + '--new-table-name', 't_pt_1751') } +); + +like( + $output, + qr/Successfully altered/i, + 'Option --where runs with --no-drop-new-table and --no-swap-tables' +) or diag($output); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.t_pt_1751"`; +is( + $output + 0, + 10, + 'Only 10 rows copied' +); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.t_pt_1751 where id <= 10"`; +is( + $output + 0, + 0, + 'Rows, satisfying --where condition are not copied' +) or diag($output); + +$sb->load_file('master', "$sample/basic_no_fks_innodb.sql"); + +$output = output( + sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t", + '--alter', 'add column e int', '--where', 'id > 10', '--execute', + '--force') } +); + +like( + $output, + qr/Successfully altered/i, + 'Option --where runs with --force' +) or diag($output); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.t"`; +is( + $output + 0, + 10, + 'Only 10 rows copied' +); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.t where id <= 10"`; +is( + $output + 0, + 0, + 'Rows, satisfying --where condition are not copied' +) or diag($output); + +# Same test with chunk size = 1 + +$sb->load_file('master', "$sample/basic_no_fks_innodb.sql"); + +$output = output( + sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t", + '--alter', 'add column e int', '--where', 'id > 10', '--execute', + '--no-swap-tables', '--no-drop-new-table', '--chunk-size', '1', + '--new-table-name', 't_pt_1751') } +); + +like( + $output, + qr/Successfully altered/i, + 'Option --where runs with --no-drop-new-table and --no-swap-tables' +) or diag($output); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.t_pt_1751"`; +is( + $output + 0, + 10, + 'Only 10 rows copied' +); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.t_pt_1751 where id <= 10"`; +is( + $output + 0, + 0, + 'Rows, satisfying --where condition are not copied' +) or diag($output); + +$sb->load_file('master', "$sample/basic_no_fks_innodb.sql"); + +$output = output( + sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t", + '--alter', 'add column e int', '--where', 'id > 10', '--execute', + '--force', '--chunk-size', '1',) } +); + +like( + $output, + qr/Successfully altered/i, + 'Option --where runs with --force' +) or diag($output); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.t"`; +is( + $output + 0, + 10, + 'Only 10 rows copied' +); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.t where id <= 10"`; +is( + $output + 0, + 0, + 'Rows, satisfying --where condition are not copied' +) or diag($output); + + +# ############################################################################# +# Option --where and foreign keys +# ############################################################################# + +$sb->load_file('master', "$sample/basic_with_fks.sql"); + +$output = output( + sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=city", + '--alter', 'drop column last_update', '--where', 'city_id >= 3', '--execute', + '--alter-foreign-keys-method', 'rebuild_constraints', '--force') } +); + +like( + $output, + qr/Successfully altered/i, + 'Option --where runs with --force and --alter-foreign-keys-method=rebuild_constraints' +) or diag($output); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.city"`; +is( + $output + 0, + 3, + 'Only 3 rows copied' +); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.city where city_id < 3"`; +is( + $output + 0, + 0, + 'Rows, satisfying --where condition are not copied' +) or diag($output); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.country"`; +is( + $output + 0, + 5, + 'Table country not corrupted' +) or diag($output); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.address"`; +is( + $output + 0, + 5, + 'Table address not modified' +) or diag($output); + +$sb->load_file('master', "$sample/basic_with_fks.sql"); + +$output = output( + sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=city", + '--alter', 'drop column last_update', '--where', 'city_id >= 3', '--execute', + '--alter-foreign-keys-method', 'auto', '--force') } +); + +like( + $output, + qr/Successfully altered/i, + 'Option --where runs with --force and --alter-foreign-keys-method=auto' +) or diag($output); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.city"`; +is( + $output + 0, + 3, + 'Only 3 rows copied' +); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.city where city_id < 3"`; +is( + $output + 0, + 0, + 'Rows, satisfying --where condition are not copied' +) or diag($output); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.country"`; +is( + $output + 0, + 5, + 'Table country not corrupted' +) or diag($output); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.address"`; +is( + $output + 0, + 5, + 'Table address not modified' +) or diag($output); + +$sb->load_file('master', "$sample/basic_with_fks.sql"); + +$output = output( + sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=city", + '--alter', 'drop column last_update', '--where', 'city_id >= 3', '--execute', + '--alter-foreign-keys-method', 'drop_swap', '--force') } +); + +like( + $output, + qr/Successfully altered/i, + 'Option --where runs with --force and --alter-foreign-keys-method=drop_swap' +) or diag($output); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.city"`; +is( + $output + 0, + 3, + 'Only 3 rows copied' +); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.city where city_id < 3"`; +is( + $output + 0, + 0, + 'Rows, satisfying --where condition are not copied' +) or diag($output); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.country"`; +is( + $output + 0, + 5, + 'Table country not corrupted' +) or diag($output); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.address"`; +is( + $output + 0, + 5, + 'Table address not modified' +) or diag($output); + +$sb->load_file('master', "$sample/basic_with_fks.sql"); + +$output = output( + sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=city", + '--alter', 'drop column last_update', '--where', 'city_id >= 3', '--execute', + '--alter-foreign-keys-method', 'none', '--force') } +); + +like( + $output, + qr/Successfully altered/i, + 'Option --where runs with --force and --alter-foreign-keys-method=none' +) or diag($output); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.city"`; +is( + $output + 0, + 3, + 'Only 3 rows copied' +); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.city where city_id < 3"`; +is( + $output + 0, + 0, + 'Rows, satisfying --where condition are not copied' +) or diag($output); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.country"`; +is( + $output + 0, + 5, + 'Table country not corrupted' +) or diag($output); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.address"`; +is( + $output + 0, + 5, + 'Table address not modified' +) or diag($output); + +$sb->load_file('master', "$sample/basic_with_fks.sql"); + +($output, $exit) = full_output( + sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=city", + '--alter', 'drop column last_update', '--where', 'city_id >= 3', '--execute', + '--alter-foreign-keys-method', 'rebuild_constraints', + '--no-drop-new-table', '--no-swap-tables') } +); + +like( + $output, + qr/Child tables found and option --where specified. Rebuilding foreign key constraints may lead to errors./i, + 'Option --where does not run without --force and --alter-foreign-keys-method=rebuild_constraints when child tables are found' +) or diag($output); + +is( + $exit, + 1, + 'Exit code 1 with --where and child tables' +); + +$output = `/tmp/12345/use -N -e "select count(*) from pt_osc.address"`; +is( + $output + 0, + 5, + 'Table address not modified' +) or diag($output); + +$sb->load_file('master', "$sample/basic_with_fks.sql"); + +($output, $exit) = full_output( + sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=city", + '--alter', 'drop column last_update', '--where', 'city_id >= 3', '--execute', + '--alter-foreign-keys-method', 'auto', + '--no-drop-new-table', '--no-swap-tables') } +); + +like( + $output, + qr/Child tables found and option --where specified. Rebuilding foreign key constraints may lead to errors./i, + 'Option --where does not run without --force and --alter-foreign-keys-method=auto when child tables are found' +) or diag($output); + +is( + $exit, + 1, + 'Exit code 1 with --where and child tables' +); + +# ############################################################################# +# Done. +# ############################################################################# + +$sb->wipe_clean($master_dbh); +ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox"); +# +done_testing;