From 39fe787fddfcbe04dd1135dc4a5713c95b21133a Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Mon, 20 Feb 2012 11:09:36 -0700 Subject: [PATCH 1/3] Test and quote all idents, including reserved words and those with spaces, in OSCCaptureSync.pm. --- lib/OSCCaptureSync.pm | 18 +++-- t/lib/OSCCaptureSync.t | 127 ++++++++++++++++++------------- t/lib/samples/osc/capsync001.txt | 6 +- t/lib/samples/osc/capsync002.txt | 3 + t/lib/samples/osc/capsync003.txt | 3 + t/lib/samples/osc/tbl002.sql | 13 ++++ t/lib/samples/osc/tbl003.sql | 13 ++++ 7 files changed, 120 insertions(+), 63 deletions(-) create mode 100644 t/lib/samples/osc/capsync002.txt create mode 100644 t/lib/samples/osc/capsync003.txt create mode 100644 t/lib/samples/osc/tbl002.sql create mode 100644 t/lib/samples/osc/tbl003.sql diff --git a/lib/OSCCaptureSync.pm b/lib/OSCCaptureSync.pm index eafb4e08..aa679c4e 100644 --- a/lib/OSCCaptureSync.pm +++ b/lib/OSCCaptureSync.pm @@ -37,13 +37,13 @@ use constant PTDEBUG => $ENV{PTDEBUG} || 0; # OSCCaptureSync object sub new { my ( $class, %args ) = @_; - my @required_args = qw(); + my @required_args = qw(Quoter); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } my $self = { - %args, + Quoter => $args{Quoter}, }; return bless $self, $class; @@ -73,11 +73,14 @@ sub _make_triggers { die "I need a $arg argument" unless $args{$arg}; } my ($db, $tbl, $tmp_tbl, $chunk_column) = @args{@required_args}; + my $q = $self->{Quoter}; - my $old_table = "`$db`.`$tbl`"; - my $new_table = "`$db`.`$tmp_tbl`"; - my $new_values = join(', ', map { "NEW.$_" } @{$args{columns}}); - my $columns = join(', ', @{$args{columns}}); + $chunk_column = $q->quote($chunk_column); + + my $old_table = $q->quote($db, $tbl); + my $new_table = $q->quote($db, $tmp_tbl); + my $new_values = join(', ', map { "NEW.".$q->quote($_) } @{$args{columns}}); + my $columns = join(', ', map { $q->quote($_) } @{$args{columns}}); my $delete_trigger = "CREATE TRIGGER mk_osc_del AFTER DELETE ON $old_table " . "FOR EACH ROW " @@ -113,9 +116,10 @@ sub cleanup { die "I need a $arg argument" unless $args{$arg}; } my ($dbh, $db, $msg) = @args{@required_args}; + my $q = $self->{Quoter}; foreach my $trigger ( qw(del ins upd) ) { - my $sql = "DROP TRIGGER IF EXISTS `$db`.`mk_osc_$trigger`"; + my $sql = "DROP TRIGGER IF EXISTS " . $q->quote($db, "mk_osc_$trigger"); $msg->($sql); $dbh->do($sql) unless $args{print}; } diff --git a/t/lib/OSCCaptureSync.t b/t/lib/OSCCaptureSync.t index 829ebaba..261c861e 100644 --- a/t/lib/OSCCaptureSync.t +++ b/t/lib/OSCCaptureSync.t @@ -14,6 +14,7 @@ use Test::More; use DSNParser; use Sandbox; use PerconaTest; +use Quoter; use OSCCaptureSync; use Data::Dumper; @@ -30,72 +31,92 @@ if ( !$dbh ) { } else { - plan tests => 4; + plan tests => 10; } -$sb->load_file("master", "t/lib/samples/osc/tbl001.sql"); -$dbh->do("USE osc"); - -my $osc = new OSCCaptureSync(); - +my $q = new Quoter(); +my $osc = new OSCCaptureSync(Quoter => $q); my $msg = sub { print "$_[0]\n"; }; +my $output; -my $output = output( - sub { - $osc->capture( - dbh => $dbh, - db => 'osc', - tbl => 't', - tmp_tbl => '__new_t', - columns => [qw(id c)], - chunk_column => 'id', - msg => $msg, - ) - }, -); +sub test_table { + my (%args) = @_; + my ($tbl, $col, $expect) = @args{qw(tbl col expect)}; -ok( - no_diff( - $output, - "t/lib/samples/osc/capsync001.txt", - cmd_output => 1, - ), - "SQL statments to create triggers" -); + $sb->load_file("master", "t/lib/samples/osc/$tbl"); + PerconaTest::wait_for_table($dbh, "osc.t", "id=5"); + $dbh->do("USE osc"); -$dbh->do('insert into t values (6, "f")'); -$dbh->do('update t set c="z" where id=1'); -$dbh->do('delete from t where id=3'); - -my $rows = $dbh->selectall_arrayref("select id, c from __new_t order by id"); -is_deeply( - $rows, - [ - [1, 'z'], # update t set c="z" where id=1 - [6, 'f'], # insert into t values (6, "f") - ], - "Triggers work" -) or print Dumper($rows); - -output(sub { - $osc->cleanup( - dbh => $dbh, - db => 'osc', - msg => $msg, + ok( + no_diff( + sub { + $osc->capture( + dbh => $dbh, + db => 'osc', + tbl => 't', + tmp_tbl => '__new_t', + columns => ['id', $col], + chunk_column => 'id', + msg => $msg, + ) + }, + "t/lib/samples/osc/$expect", + stderr => 1, + ), + "$tbl: SQL statments to create triggers" ); -}); -$rows = $dbh->selectall_arrayref("show triggers from `osc` like 't'"); -is_deeply( - $rows, - [], - "Cleanup removes the triggers" + $dbh->do("insert into t values (6, 'f')"); + $dbh->do("update t set `$col`='z' where id=1"); + $dbh->do("delete from t where id=3"); + + my $rows = $dbh->selectall_arrayref("select id, `$col` from __new_t order by id"); + is_deeply( + $rows, + [ + [1, 'z'], # update t set c="z" where id=1 + [6, 'f'], # insert into t values (6, "f") + ], + "$tbl: Triggers work" + ) or print Dumper($rows); + + output(sub { + $osc->cleanup( + dbh => $dbh, + db => 'osc', + msg => $msg, + ); + }); + + $rows = $dbh->selectall_arrayref("show triggers from `osc` like 't'"); + is_deeply( + $rows, + [], + "$tbl: Cleanup removes the triggers" + ); +} + +test_table( + tbl => "tbl001.sql", + col => "c", + expect => "capsync001.txt", +); + +test_table( + tbl => "tbl002.sql", + col => "default", + expect => "capsync002.txt", +); + +test_table( + tbl => "tbl003.sql", + col => "space col", + expect => "capsync003.txt", ); # ############################################################################# # Done. # ############################################################################# -$output = ''; { local *STDERR; open STDERR, '>', \$output; diff --git a/t/lib/samples/osc/capsync001.txt b/t/lib/samples/osc/capsync001.txt index 8f16663e..8dd2c941 100644 --- a/t/lib/samples/osc/capsync001.txt +++ b/t/lib/samples/osc/capsync001.txt @@ -1,3 +1,3 @@ -CREATE TRIGGER mk_osc_del AFTER DELETE ON `osc`.`t` FOR EACH ROW DELETE IGNORE FROM `osc`.`__new_t` WHERE `osc`.`__new_t`.id = OLD.id -CREATE TRIGGER mk_osc_upd AFTER UPDATE ON `osc`.`t` FOR EACH ROW REPLACE INTO `osc`.`__new_t` (id, c) VALUES (NEW.id, NEW.c) -CREATE TRIGGER mk_osc_ins AFTER INSERT ON `osc`.`t` FOR EACH ROW REPLACE INTO `osc`.`__new_t` (id, c) VALUES(NEW.id, NEW.c) +CREATE TRIGGER mk_osc_del AFTER DELETE ON `osc`.`t` FOR EACH ROW DELETE IGNORE FROM `osc`.`__new_t` WHERE `osc`.`__new_t`.`id` = OLD.`id` +CREATE TRIGGER mk_osc_upd AFTER UPDATE ON `osc`.`t` FOR EACH ROW REPLACE INTO `osc`.`__new_t` (`id`, `c`) VALUES (NEW.`id`, NEW.`c`) +CREATE TRIGGER mk_osc_ins AFTER INSERT ON `osc`.`t` FOR EACH ROW REPLACE INTO `osc`.`__new_t` (`id`, `c`) VALUES(NEW.`id`, NEW.`c`) diff --git a/t/lib/samples/osc/capsync002.txt b/t/lib/samples/osc/capsync002.txt new file mode 100644 index 00000000..846c074d --- /dev/null +++ b/t/lib/samples/osc/capsync002.txt @@ -0,0 +1,3 @@ +CREATE TRIGGER mk_osc_del AFTER DELETE ON `osc`.`t` FOR EACH ROW DELETE IGNORE FROM `osc`.`__new_t` WHERE `osc`.`__new_t`.`id` = OLD.`id` +CREATE TRIGGER mk_osc_upd AFTER UPDATE ON `osc`.`t` FOR EACH ROW REPLACE INTO `osc`.`__new_t` (`id`, `default`) VALUES (NEW.`id`, NEW.`default`) +CREATE TRIGGER mk_osc_ins AFTER INSERT ON `osc`.`t` FOR EACH ROW REPLACE INTO `osc`.`__new_t` (`id`, `default`) VALUES(NEW.`id`, NEW.`default`) diff --git a/t/lib/samples/osc/capsync003.txt b/t/lib/samples/osc/capsync003.txt new file mode 100644 index 00000000..e971e4e0 --- /dev/null +++ b/t/lib/samples/osc/capsync003.txt @@ -0,0 +1,3 @@ +CREATE TRIGGER mk_osc_del AFTER DELETE ON `osc`.`t` FOR EACH ROW DELETE IGNORE FROM `osc`.`__new_t` WHERE `osc`.`__new_t`.`id` = OLD.`id` +CREATE TRIGGER mk_osc_upd AFTER UPDATE ON `osc`.`t` FOR EACH ROW REPLACE INTO `osc`.`__new_t` (`id`, `space col`) VALUES (NEW.`id`, NEW.`space col`) +CREATE TRIGGER mk_osc_ins AFTER INSERT ON `osc`.`t` FOR EACH ROW REPLACE INTO `osc`.`__new_t` (`id`, `space col`) VALUES(NEW.`id`, NEW.`space col`) diff --git a/t/lib/samples/osc/tbl002.sql b/t/lib/samples/osc/tbl002.sql new file mode 100644 index 00000000..075823b5 --- /dev/null +++ b/t/lib/samples/osc/tbl002.sql @@ -0,0 +1,13 @@ +DROP DATABASE IF EXISTS osc; +CREATE DATABASE osc; +USE osc; + +CREATE TABLE t ( + id INT UNSIGNED PRIMARY KEY, + `default` VARCHAR(16) +) ENGINE=InnoDB; + +CREATE TABLE __new_t LIKE t; + +INSERT INTO t VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e'); + diff --git a/t/lib/samples/osc/tbl003.sql b/t/lib/samples/osc/tbl003.sql new file mode 100644 index 00000000..b0085f42 --- /dev/null +++ b/t/lib/samples/osc/tbl003.sql @@ -0,0 +1,13 @@ +DROP DATABASE IF EXISTS osc; +CREATE DATABASE osc; +USE osc; + +CREATE TABLE t ( + id INT UNSIGNED PRIMARY KEY, + `space col` VARCHAR(16) +) ENGINE=InnoDB; + +CREATE TABLE __new_t LIKE t; + +INSERT INTO t VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e'); + From 17fe96f361c26418b2eff8828ad11382bb9e0695 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Mon, 20 Feb 2012 11:37:34 -0700 Subject: [PATCH 2/3] Test and quote all idents, including reserved words and those with spaces, in CopyRowsInsertSelect.pm. --- lib/CopyRowsInsertSelect.pm | 8 +- t/lib/CopyRowsInsertSelect.t | 168 ++++++++++++++++++------------- t/lib/samples/osc/copyins001.txt | 4 +- t/lib/samples/osc/copyins002.txt | 2 + t/lib/samples/osc/copyins003.txt | 2 + 5 files changed, 111 insertions(+), 73 deletions(-) create mode 100644 t/lib/samples/osc/copyins002.txt create mode 100644 t/lib/samples/osc/copyins003.txt diff --git a/lib/CopyRowsInsertSelect.pm b/lib/CopyRowsInsertSelect.pm index 7e6b2c1f..b210d902 100644 --- a/lib/CopyRowsInsertSelect.pm +++ b/lib/CopyRowsInsertSelect.pm @@ -37,13 +37,14 @@ use constant PTDEBUG => $ENV{PTDEBUG} || 0; # CopyRowsInsertSelect object sub new { my ( $class, %args ) = @_; - my @required_args = qw(Retry); + my @required_args = qw(Retry Quoter); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } my $self = { - %args, + Retry => $args{Retry}, + Quoter => $args{Quoter}, }; return bless $self, $class; @@ -56,9 +57,10 @@ sub copy { die "I need a $arg argument" unless $args{$arg}; } my ($dbh, $msg, $from_table, $to_table, $chunks) = @args{@required_args}; + my $q = $self->{Quoter}; my $pr = $args{Progress}; my $sleep = $args{sleep}; - my $columns = join(', ', @{$args{columns}}); + my $columns = join(', ', map { $q->quote($_) } @{$args{columns}}); my $n_chunks = @$chunks - 1; for my $chunkno ( 0..$n_chunks ) { diff --git a/t/lib/CopyRowsInsertSelect.t b/t/lib/CopyRowsInsertSelect.t index b5a4e0a1..361775df 100644 --- a/t/lib/CopyRowsInsertSelect.t +++ b/t/lib/CopyRowsInsertSelect.t @@ -17,6 +17,7 @@ use PerconaTest; use Progress; use Transformers; use Retry; +use Quoter; use CopyRowsInsertSelect; Transformers->import(qw(secs_to_time)); @@ -38,64 +39,124 @@ elsif ( !@{$dbh->selectcol_arrayref('SHOW DATABASES LIKE "sakila"')} ) { plan skip_all => "Sandbox master does not have the sakila database"; } else { - plan tests => 8; + plan tests => 14; } +my $q = new Quoter(); my $rr = new Retry(); -my $osc = new CopyRowsInsertSelect(Retry => $rr); +my $osc = new CopyRowsInsertSelect(Retry => $rr, Quoter => $q); my $msg = sub { print "$_[0]\n"; }; my $output = ""; +my $rows; -$sb->load_file("master", "t/lib/samples/osc/tbl001.sql"); -$dbh->do("USE osc"); +# ########################################################################### +# Copy simple tables. +# ########################################################################### -$osc->copy( - dbh => $dbh, - from_table => 'osc.t', - to_table => 'osc.__new_t', - columns => [qw(id c)], - chunks => ['1=1'], - msg => $msg, +sub test_copy_table { + my (%args) = @_; + my ($tbl, $col, $expect) = @args{qw(tbl col expect)}; + + $sb->load_file("master", "t/lib/samples/osc/$tbl"); + PerconaTest::wait_for_table($dbh, "osc.t", "id=5"); + $dbh->do("USE osc"); + + $osc->copy( + dbh => $dbh, + from_table => 'osc.t', + to_table => 'osc.__new_t', + columns => ['id', $col], + chunks => ['1=1'], + msg => $msg, + ); + + $rows = $dbh->selectall_arrayref("select id, `$col` from __new_t order by id"); + is_deeply( + $rows, + [ [1, 'a'], [2, 'b'], [3, 'c'], [4, 'd'], [5, 'e'], ], + "$tbl: One chunk copy" + ) or print Dumper($rows); + + $dbh->do("truncate table osc.__new_t"); + + ok( + no_diff( + sub { + $osc->copy( + dbh => $dbh, + from_table => 'osc.t', + to_table => 'osc.__new_t', + columns => ['id', $col], + chunks => ['id < 4', 'id >= 4 AND id < 6'], + msg => $msg, + print => 1, + engine_flags => 'LOCK IN SHARE MODE', + ); + }, + "t/lib/samples/osc/$expect", + stderr => 1, + ), + "$tbl: 2 chunk copy" + ); + + $rows = $dbh->selectall_arrayref("select id, `$col` from __new_t order by id"); + is_deeply( + $rows, + [], + "$tbl: print doesn't exec statements" + ); +} + +test_copy_table( + tbl => "tbl001.sql", + col => "c", + expect => "copyins001.txt", ); -my $rows = $dbh->selectall_arrayref("select id, c from __new_t order by id"); -is_deeply( - $rows, - [ [1, 'a'], [2, 'b'], [3, 'c'], [4, 'd'], [5, 'e'], ], - "One chunk copy" -) or print Dumper($rows); - - +# Sleep callback. +my $sleep_cnt = 0; $dbh->do("truncate table osc.__new_t"); -$output = output( sub { +output( sub { $osc->copy( - dbh => $dbh, - from_table => 'osc.t', - to_table => 'osc.__new_t', - columns => [qw(id c)], - chunks => ['id < 4', 'id >= 4 AND id < 6'], - msg => $msg, - print => 1, - engine_flags => 'LOCK IN SHARE MODE', + dbh => $dbh, + from_table => 'osc.t', + to_table => 'osc.__new_t', + columns => [qw(id c)], + chunks => ['id < 4', 'id >= 4 AND id < 6'], + msg => $msg, + sleep => sub { $sleep_cnt++; }, ); }); +is( + $sleep_cnt, + 1, + "Calls sleep callback after each chunk (except last chunk)" +); + +eval { + $output = output(sub { $osc->cleanup(); } ); +}; ok( - no_diff( - $output, - "t/lib/samples/osc/copyins001.txt", - cmd_output => 1, - ), - "Prints 2 SQL statments for the 2 chunks" + !$EVAL_ERROR && !$output, + "cleanup() works but doesn't do anything" ); -$rows = $dbh->selectall_arrayref("select id, c from __new_t order by id"); -is_deeply( - $rows, - [], - "Doesn't exec those statements if print is true" +test_copy_table( + tbl => "tbl002.sql", + col => "default", + expect => "copyins002.txt", ); +test_copy_table( + tbl => "tbl003.sql", + col => "space col", + expect => "copyins003.txt", +); + +# ########################################################################### +# Copy a larger, more complex sakila table. +# ########################################################################### $dbh->do('create table osc.city like sakila.city'); $dbh->do('alter table osc.city engine=myisam'); my $chunks = [ @@ -140,35 +201,6 @@ like( "Reports copy progress if Progress obj given" ); - -my $sleep_cnt = 0; - -$dbh->do("truncate table osc.__new_t"); -output( sub { - $osc->copy( - dbh => $dbh, - from_table => 'osc.t', - to_table => 'osc.__new_t', - columns => [qw(id c)], - chunks => ['id < 4', 'id >= 4 AND id < 6'], - msg => $msg, - sleep => sub { $sleep_cnt++; }, - ); -}); -is( - $sleep_cnt, - 1, - "Calls sleep callback after each chunk (except last chunk)" -); - -eval { - $output = output(sub { $osc->cleanup(); } ); -}; -ok( - !$EVAL_ERROR && !$output, - "cleanup() works but doesn't do anything" -); - # ############################################################################# # Done. # ############################################################################# diff --git a/t/lib/samples/osc/copyins001.txt b/t/lib/samples/osc/copyins001.txt index def1dd77..07200a9a 100644 --- a/t/lib/samples/osc/copyins001.txt +++ b/t/lib/samples/osc/copyins001.txt @@ -1,2 +1,2 @@ -INSERT IGNORE INTO osc.__new_t (id, c) SELECT id, c FROM osc.t WHERE (id < 4) LOCK IN SHARE MODE -INSERT IGNORE INTO osc.__new_t (id, c) SELECT id, c FROM osc.t WHERE (id >= 4 AND id < 6) LOCK IN SHARE MODE +INSERT IGNORE INTO osc.__new_t (`id`, `c`) SELECT `id`, `c` FROM osc.t WHERE (id < 4) LOCK IN SHARE MODE +INSERT IGNORE INTO osc.__new_t (`id`, `c`) SELECT `id`, `c` FROM osc.t WHERE (id >= 4 AND id < 6) LOCK IN SHARE MODE diff --git a/t/lib/samples/osc/copyins002.txt b/t/lib/samples/osc/copyins002.txt new file mode 100644 index 00000000..f20bb523 --- /dev/null +++ b/t/lib/samples/osc/copyins002.txt @@ -0,0 +1,2 @@ +INSERT IGNORE INTO osc.__new_t (`id`, `default`) SELECT `id`, `default` FROM osc.t WHERE (id < 4) LOCK IN SHARE MODE +INSERT IGNORE INTO osc.__new_t (`id`, `default`) SELECT `id`, `default` FROM osc.t WHERE (id >= 4 AND id < 6) LOCK IN SHARE MODE diff --git a/t/lib/samples/osc/copyins003.txt b/t/lib/samples/osc/copyins003.txt new file mode 100644 index 00000000..1ad162d3 --- /dev/null +++ b/t/lib/samples/osc/copyins003.txt @@ -0,0 +1,2 @@ +INSERT IGNORE INTO osc.__new_t (`id`, `space col`) SELECT `id`, `space col` FROM osc.t WHERE (id < 4) LOCK IN SHARE MODE +INSERT IGNORE INTO osc.__new_t (`id`, `space col`) SELECT `id`, `space col` FROM osc.t WHERE (id >= 4 AND id < 6) LOCK IN SHARE MODE From 75457eaadd4f5642b6c5f4bad0494fecbd2093a3 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Tue, 21 Feb 2012 09:06:08 -0700 Subject: [PATCH 3/3] Test, update modules, and fix bug 873598 in pt-online-schema-change. --- bin/pt-online-schema-change | 604 +++++++---------------- t/pt-online-schema-change/basics.t | 47 +- t/pt-online-schema-change/check_tables.t | 6 +- 3 files changed, 236 insertions(+), 421 deletions(-) diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index c899fa09..e39042ec 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -959,7 +959,7 @@ sub _parse_size { $opt->{value} = ($pre || '') . $num; } else { - $self->save_error("Invalid size for --$opt->{long}"); + $self->save_error("Invalid size for --$opt->{long}: $val"); } return; } @@ -1249,12 +1249,14 @@ sub parse_options { sub as_string { my ( $self, $dsn, $props ) = @_; return $dsn unless ref $dsn; - my %allowed = $props ? map { $_=>1 } @$props : (); + my @keys = $props ? @$props : sort keys %$dsn; return join(',', - map { "$_=" . ($_ eq 'p' ? '...' : $dsn->{$_}) } - grep { defined $dsn->{$_} && $self->{opts}->{$_} } - grep { !$props || $allowed{$_} } - sort keys %$dsn ); + map { "$_=" . ($_ eq 'p' ? '...' : $dsn->{$_}) } + grep { + exists $self->{opts}->{$_} + && exists $dsn->{$_} + && defined $dsn->{$_} + } @keys); } sub usage { @@ -1729,6 +1731,48 @@ sub join_quote { return $db ? "$db.$tbl" : $tbl; } +sub serialize_list { + my ( $self, @args ) = @_; + return unless @args; + + return $args[0] if @args == 1 && !defined $args[0]; + + die "Cannot serialize multiple values with undef/NULL" + if grep { !defined $_ } @args; + + return join ',', map { quotemeta } @args; +} + +sub deserialize_list { + my ( $self, $string ) = @_; + return $string unless defined $string; + my @escaped_parts = $string =~ / + \G # Start of string, or end of previous match. + ( # Each of these is an element in the original list. + [^\\,]* # Anything not a backslash or a comma + (?: # When we get here, we found one of the above. + \\. # A backslash followed by something so we can continue + [^\\,]* # Same as above. + )* # Repeat zero of more times. + ) + , # Comma dividing elements + /sxgc; + + push @escaped_parts, pos($string) ? substr( $string, pos($string) ) : $string; + + my @unescaped_parts = map { + my $part = $_; + + my $char_class = utf8::is_utf8($part) # If it's a UTF-8 string, + ? qr/(?=\p{ASCII})\W/ # We only care about non-word + : qr/(?=\p{ASCII})\W|[\x{80}-\x{FF}]/; # Otherwise, + $part =~ s/\\($char_class)/$1/g; + $part; + } @escaped_parts; + + return @unescaped_parts; +} + 1; } # ########################################################################### @@ -2022,19 +2066,56 @@ sub new { return bless $self, $class; } +sub get_create_table { + my ( $self, $dbh, $db, $tbl ) = @_; + die "I need a dbh parameter" unless $dbh; + die "I need a db parameter" unless $db; + die "I need a tbl parameter" unless $tbl; + my $q = $self->{Quoter}; + + my $sql = '/*!40101 SET @OLD_SQL_MODE := @@SQL_MODE, ' + . q{@@SQL_MODE := REPLACE(REPLACE(@@SQL_MODE, 'ANSI_QUOTES', ''), ',,', ','), } + . '@OLD_QUOTE := @@SQL_QUOTE_SHOW_CREATE, ' + . '@@SQL_QUOTE_SHOW_CREATE := 1 */'; + PTDEBUG && _d($sql); + eval { $dbh->do($sql); }; + PTDEBUG && $EVAL_ERROR && _d($EVAL_ERROR); + + $sql = 'USE ' . $q->quote($db); + PTDEBUG && _d($dbh, $sql); + $dbh->do($sql); + + $sql = "SHOW CREATE TABLE " . $q->quote($db, $tbl); + PTDEBUG && _d($sql); + my $href; + eval { $href = $dbh->selectrow_hashref($sql); }; + if ( $EVAL_ERROR ) { + PTDEBUG && _d($EVAL_ERROR); + return; + } + + $sql = '/*!40101 SET @@SQL_MODE := @OLD_SQL_MODE, ' + . '@@SQL_QUOTE_SHOW_CREATE := @OLD_QUOTE */'; + PTDEBUG && _d($sql); + $dbh->do($sql); + + my ($key) = grep { m/create table/i } keys %$href; + if ( $key ) { + PTDEBUG && _d('This table is a base table'); + $href->{$key} =~ s/\b[ ]{2,}/ /g; + $href->{$key} .= "\n"; + } + else { + PTDEBUG && _d('This table is a view'); + ($key) = grep { m/create view/i } keys %$href; + } + + return $href->{$key}; +} + sub parse { my ( $self, $ddl, $opts ) = @_; return unless $ddl; - if ( ref $ddl eq 'ARRAY' ) { - if ( lc $ddl->[0] eq 'table' ) { - $ddl = $ddl->[1]; - } - else { - return { - engine => 'VIEW', - }; - } - } if ( $ddl !~ m/CREATE (?:TEMPORARY )?TABLE `/ ) { die "Cannot parse table definition; is ANSI quoting " @@ -2341,41 +2422,31 @@ sub remove_auto_increment { return $ddl; } -sub remove_secondary_indexes { - my ( $self, $ddl ) = @_; - my $sec_indexes_ddl; - my $tbl_struct = $self->parse($ddl); - - if ( ($tbl_struct->{engine} || '') =~ m/InnoDB/i ) { - my $clustered_key = $tbl_struct->{clustered_key}; - $clustered_key ||= ''; - - my @sec_indexes = map { - my $key_def = $_->{ddl}; - $key_def =~ s/([\(\)])/\\$1/g; - $ddl =~ s/\s+$key_def//i; - - my $key_ddl = "ADD $_->{ddl}"; - $key_ddl .= ',' unless $key_ddl =~ m/,$/; - $key_ddl; - } - grep { $_->{name} ne $clustered_key } - values %{$tbl_struct->{keys}}; - PTDEBUG && _d('Secondary indexes:', Dumper(\@sec_indexes)); - - if ( @sec_indexes ) { - $sec_indexes_ddl = join(' ', @sec_indexes); - $sec_indexes_ddl =~ s/,$//; - } - - $ddl =~ s/,(\n\) )/$1/s; +sub get_table_status { + my ( $self, $dbh, $db, $like ) = @_; + my $q = $self->{Quoter}; + my $sql = "SHOW TABLE STATUS FROM " . $q->quote($db); + my @params; + if ( $like ) { + $sql .= ' LIKE ?'; + push @params, $like; } - else { - PTDEBUG && _d('Not removing secondary indexes from', - $tbl_struct->{engine}, 'table'); + PTDEBUG && _d($sql, @params); + my $sth = $dbh->prepare($sql); + eval { $sth->execute(@params); }; + if ($EVAL_ERROR) { + PTDEBUG && _d($EVAL_ERROR); + return; } - - return $ddl, $sec_indexes_ddl, $tbl_struct; + my @tables = @{$sth->fetchall_arrayref({})}; + @tables = map { + my %tbl; # Make a copy with lowercased keys + @tbl{ map { lc $_ } keys %$_ } = values %$_; + $tbl{engine} ||= $tbl{type} || $tbl{comment}; + delete $tbl{type}; + \%tbl; + } @tables; + return @tables; } sub _d { @@ -2392,311 +2463,6 @@ sub _d { # End TableParser package # ########################################################################### -# ########################################################################### -# MySQLDump package -# This package is a copy without comments from the original. The original -# with comments and its test file can be found in the Bazaar repository at, -# lib/MySQLDump.pm -# t/lib/MySQLDump.t -# See https://launchpad.net/percona-toolkit for more information. -# ########################################################################### -{ -package MySQLDump; - -use strict; -use warnings FATAL => 'all'; -use English qw(-no_match_vars); -use constant PTDEBUG => $ENV{PTDEBUG} || 0; - -( our $before = <<'EOF') =~ s/^ //gm; - /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; - /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; - /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; - /*!40101 SET NAMES utf8 */; - /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; - /*!40103 SET TIME_ZONE='+00:00' */; - /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; - /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; - /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; - /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; -EOF - -( our $after = <<'EOF') =~ s/^ //gm; - /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; - /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; - /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; - /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; - /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; - /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; - /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; - /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; -EOF - -sub new { - my ( $class, %args ) = @_; - my $self = { - cache => 0, # Afaik no script uses this cache any longer because - }; - return bless $self, $class; -} - -sub dump { - my ( $self, $dbh, $quoter, $db, $tbl, $what ) = @_; - - if ( $what eq 'table' ) { - my $ddl = $self->get_create_table($dbh, $quoter, $db, $tbl); - return unless $ddl; - if ( $ddl->[0] eq 'table' ) { - return $before - . 'DROP TABLE IF EXISTS ' . $quoter->quote($tbl) . ";\n" - . $ddl->[1] . ";\n"; - } - else { - return 'DROP TABLE IF EXISTS ' . $quoter->quote($tbl) . ";\n" - . '/*!50001 DROP VIEW IF EXISTS ' - . $quoter->quote($tbl) . "*/;\n/*!50001 " - . $self->get_tmp_table($dbh, $quoter, $db, $tbl) . "*/;\n"; - } - } - elsif ( $what eq 'triggers' ) { - my $trgs = $self->get_triggers($dbh, $quoter, $db, $tbl); - if ( $trgs && @$trgs ) { - my $result = $before . "\nDELIMITER ;;\n"; - foreach my $trg ( @$trgs ) { - if ( $trg->{sql_mode} ) { - $result .= qq{/*!50003 SET SESSION SQL_MODE='$trg->{sql_mode}' */;;\n}; - } - $result .= "/*!50003 CREATE */ "; - if ( $trg->{definer} ) { - my ( $user, $host ) - = map { s/'/''/g; "'$_'"; } - split('@', $trg->{definer}, 2); - $result .= "/*!50017 DEFINER=$user\@$host */ "; - } - $result .= sprintf("/*!50003 TRIGGER %s %s %s ON %s\nFOR EACH ROW %s */;;\n\n", - $quoter->quote($trg->{trigger}), - @{$trg}{qw(timing event)}, - $quoter->quote($trg->{table}), - $trg->{statement}); - } - $result .= "DELIMITER ;\n\n/*!50003 SET SESSION SQL_MODE=\@OLD_SQL_MODE */;\n\n"; - return $result; - } - else { - return undef; - } - } - elsif ( $what eq 'view' ) { - my $ddl = $self->get_create_table($dbh, $quoter, $db, $tbl); - return '/*!50001 DROP TABLE IF EXISTS ' . $quoter->quote($tbl) . "*/;\n" - . '/*!50001 DROP VIEW IF EXISTS ' . $quoter->quote($tbl) . "*/;\n" - . '/*!50001 ' . $ddl->[1] . "*/;\n"; - } - else { - die "You didn't say what to dump."; - } -} - -sub _use_db { - my ( $self, $dbh, $quoter, $new ) = @_; - if ( !$new ) { - PTDEBUG && _d('No new DB to use'); - return; - } - my $sql = 'USE ' . $quoter->quote($new); - PTDEBUG && _d($dbh, $sql); - $dbh->do($sql); - return; -} - -sub get_create_table { - my ( $self, $dbh, $quoter, $db, $tbl ) = @_; - if ( !$self->{cache} || !$self->{tables}->{$db}->{$tbl} ) { - my $sql = '/*!40101 SET @OLD_SQL_MODE := @@SQL_MODE, ' - . q{@@SQL_MODE := REPLACE(REPLACE(@@SQL_MODE, 'ANSI_QUOTES', ''), ',,', ','), } - . '@OLD_QUOTE := @@SQL_QUOTE_SHOW_CREATE, ' - . '@@SQL_QUOTE_SHOW_CREATE := 1 */'; - PTDEBUG && _d($sql); - eval { $dbh->do($sql); }; - PTDEBUG && $EVAL_ERROR && _d($EVAL_ERROR); - $self->_use_db($dbh, $quoter, $db); - $sql = "SHOW CREATE TABLE " . $quoter->quote($db, $tbl); - PTDEBUG && _d($sql); - my $href; - eval { $href = $dbh->selectrow_hashref($sql); }; - if ( $EVAL_ERROR ) { - warn "Failed to $sql. The table may be damaged.\nError: $EVAL_ERROR"; - return; - } - - $sql = '/*!40101 SET @@SQL_MODE := @OLD_SQL_MODE, ' - . '@@SQL_QUOTE_SHOW_CREATE := @OLD_QUOTE */'; - PTDEBUG && _d($sql); - $dbh->do($sql); - my ($key) = grep { m/create table/i } keys %$href; - if ( $key ) { - PTDEBUG && _d('This table is a base table'); - $self->{tables}->{$db}->{$tbl} = [ 'table', $href->{$key} ]; - } - else { - PTDEBUG && _d('This table is a view'); - ($key) = grep { m/create view/i } keys %$href; - $self->{tables}->{$db}->{$tbl} = [ 'view', $href->{$key} ]; - } - } - return $self->{tables}->{$db}->{$tbl}; -} - -sub get_columns { - my ( $self, $dbh, $quoter, $db, $tbl ) = @_; - PTDEBUG && _d('Get columns for', $db, $tbl); - if ( !$self->{cache} || !$self->{columns}->{$db}->{$tbl} ) { - $self->_use_db($dbh, $quoter, $db); - my $sql = "SHOW COLUMNS FROM " . $quoter->quote($db, $tbl); - PTDEBUG && _d($sql); - my $cols = $dbh->selectall_arrayref($sql, { Slice => {} }); - - $self->{columns}->{$db}->{$tbl} = [ - map { - my %row; - @row{ map { lc $_ } keys %$_ } = values %$_; - \%row; - } @$cols - ]; - } - return $self->{columns}->{$db}->{$tbl}; -} - -sub get_tmp_table { - my ( $self, $dbh, $quoter, $db, $tbl ) = @_; - my $result = 'CREATE TABLE ' . $quoter->quote($tbl) . " (\n"; - $result .= join(",\n", - map { ' ' . $quoter->quote($_->{field}) . ' ' . $_->{type} } - @{$self->get_columns($dbh, $quoter, $db, $tbl)}); - $result .= "\n)"; - PTDEBUG && _d($result); - return $result; -} - -sub get_triggers { - my ( $self, $dbh, $quoter, $db, $tbl ) = @_; - if ( !$self->{cache} || !$self->{triggers}->{$db} ) { - $self->{triggers}->{$db} = {}; - my $sql = '/*!40101 SET @OLD_SQL_MODE := @@SQL_MODE, ' - . q{@@SQL_MODE := REPLACE(REPLACE(@@SQL_MODE, 'ANSI_QUOTES', ''), ',,', ','), } - . '@OLD_QUOTE := @@SQL_QUOTE_SHOW_CREATE, ' - . '@@SQL_QUOTE_SHOW_CREATE := 1 */'; - PTDEBUG && _d($sql); - eval { $dbh->do($sql); }; - PTDEBUG && $EVAL_ERROR && _d($EVAL_ERROR); - $sql = "SHOW TRIGGERS FROM " . $quoter->quote($db); - PTDEBUG && _d($sql); - my $sth = $dbh->prepare($sql); - $sth->execute(); - if ( $sth->rows ) { - my $trgs = $sth->fetchall_arrayref({}); - foreach my $trg (@$trgs) { - my %trg; - @trg{ map { lc $_ } keys %$trg } = values %$trg; - push @{ $self->{triggers}->{$db}->{ $trg{table} } }, \%trg; - } - } - $sql = '/*!40101 SET @@SQL_MODE := @OLD_SQL_MODE, ' - . '@@SQL_QUOTE_SHOW_CREATE := @OLD_QUOTE */'; - PTDEBUG && _d($sql); - $dbh->do($sql); - } - if ( $tbl ) { - return $self->{triggers}->{$db}->{$tbl}; - } - return values %{$self->{triggers}->{$db}}; -} - -sub get_databases { - my ( $self, $dbh, $quoter, $like ) = @_; - if ( !$self->{cache} || !$self->{databases} || $like ) { - my $sql = 'SHOW DATABASES'; - my @params; - if ( $like ) { - $sql .= ' LIKE ?'; - push @params, $like; - } - my $sth = $dbh->prepare($sql); - PTDEBUG && _d($sql, @params); - $sth->execute( @params ); - my @dbs = map { $_->[0] } @{$sth->fetchall_arrayref()}; - $self->{databases} = \@dbs unless $like; - return @dbs; - } - return @{$self->{databases}}; -} - -sub get_table_status { - my ( $self, $dbh, $quoter, $db, $like ) = @_; - if ( !$self->{cache} || !$self->{table_status}->{$db} || $like ) { - my $sql = "SHOW TABLE STATUS FROM " . $quoter->quote($db); - my @params; - if ( $like ) { - $sql .= ' LIKE ?'; - push @params, $like; - } - PTDEBUG && _d($sql, @params); - my $sth = $dbh->prepare($sql); - $sth->execute(@params); - my @tables = @{$sth->fetchall_arrayref({})}; - @tables = map { - my %tbl; # Make a copy with lowercased keys - @tbl{ map { lc $_ } keys %$_ } = values %$_; - $tbl{engine} ||= $tbl{type} || $tbl{comment}; - delete $tbl{type}; - \%tbl; - } @tables; - $self->{table_status}->{$db} = \@tables unless $like; - return @tables; - } - return @{$self->{table_status}->{$db}}; -} - -sub get_table_list { - my ( $self, $dbh, $quoter, $db, $like ) = @_; - if ( !$self->{cache} || !$self->{table_list}->{$db} || $like ) { - my $sql = "SHOW /*!50002 FULL*/ TABLES FROM " . $quoter->quote($db); - my @params; - if ( $like ) { - $sql .= ' LIKE ?'; - push @params, $like; - } - PTDEBUG && _d($sql, @params); - my $sth = $dbh->prepare($sql); - $sth->execute(@params); - my @tables = @{$sth->fetchall_arrayref()}; - @tables = map { - my %tbl = ( - name => $_->[0], - engine => ($_->[1] || '') eq 'VIEW' ? 'VIEW' : '', - ); - \%tbl; - } @tables; - $self->{table_list}->{$db} = \@tables unless $like; - return @tables; - } - return @{$self->{table_list}->{$db}}; -} - -sub _d { - my ($package, undef, $line) = caller 0; - @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } - map { defined $_ ? $_ : 'undef' } - @_; - print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; -} - -1; -} -# ########################################################################### -# End MySQLDump package -# ########################################################################### - # ########################################################################### # TableChunker package # This package is a copy without comments from the original. The original @@ -3698,14 +3464,21 @@ sub set_callback { sub start { my ( $self, $start ) = @_; $self->{start} = $self->{last_reported} = $start || time(); + $self->{first_report} = 0; } sub update { - my ( $self, $callback, $now ) = @_; + my ( $self, $callback, %args ) = @_; my $jobsize = $self->{jobsize}; - $now ||= time(); + my $now ||= $args{now} || time; + $self->{iterations}++; # How many updates have happened; + if ( !$self->{first_report} && $args{first_report} ) { + $args{first_report}->(); + $self->{first_report} = 1; + } + if ( $self->{report} eq 'time' && $self->{interval} > $now - $self->{last_reported} ) { @@ -3786,13 +3559,13 @@ use constant PTDEBUG => $ENV{PTDEBUG} || 0; sub new { my ( $class, %args ) = @_; - my @required_args = qw(); + my @required_args = qw(Quoter); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } my $self = { - %args, + Quoter => $args{Quoter}, }; return bless $self, $class; @@ -3822,11 +3595,14 @@ sub _make_triggers { die "I need a $arg argument" unless $args{$arg}; } my ($db, $tbl, $tmp_tbl, $chunk_column) = @args{@required_args}; + my $q = $self->{Quoter}; - my $old_table = "`$db`.`$tbl`"; - my $new_table = "`$db`.`$tmp_tbl`"; - my $new_values = join(', ', map { "NEW.$_" } @{$args{columns}}); - my $columns = join(', ', @{$args{columns}}); + $chunk_column = $q->quote($chunk_column); + + my $old_table = $q->quote($db, $tbl); + my $new_table = $q->quote($db, $tmp_tbl); + my $new_values = join(', ', map { "NEW.".$q->quote($_) } @{$args{columns}}); + my $columns = join(', ', map { $q->quote($_) } @{$args{columns}}); my $delete_trigger = "CREATE TRIGGER mk_osc_del AFTER DELETE ON $old_table " . "FOR EACH ROW " @@ -3862,9 +3638,10 @@ sub cleanup { die "I need a $arg argument" unless $args{$arg}; } my ($dbh, $db, $msg) = @args{@required_args}; + my $q = $self->{Quoter}; foreach my $trigger ( qw(del ins upd) ) { - my $sql = "DROP TRIGGER IF EXISTS `$db`.`mk_osc_$trigger`"; + my $sql = "DROP TRIGGER IF EXISTS " . $q->quote($db, "mk_osc_$trigger"); $msg->($sql); $dbh->do($sql) unless $args{print}; } @@ -3904,13 +3681,14 @@ use constant PTDEBUG => $ENV{PTDEBUG} || 0; sub new { my ( $class, %args ) = @_; - my @required_args = qw(Retry); + my @required_args = qw(Retry Quoter); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } my $self = { - %args, + Retry => $args{Retry}, + Quoter => $args{Quoter}, }; return bless $self, $class; @@ -3923,9 +3701,10 @@ sub copy { die "I need a $arg argument" unless $args{$arg}; } my ($dbh, $msg, $from_table, $to_table, $chunks) = @args{@required_args}; + my $q = $self->{Quoter}; my $pr = $args{Progress}; my $sleep = $args{sleep}; - my $columns = join(', ', @{$args{columns}}); + my $columns = join(', ', map { $q->quote($_) } @{$args{columns}}); my $n_chunks = @$chunks - 1; for my $chunkno ( 0..$n_chunks ) { @@ -3950,24 +3729,23 @@ sub copy { wait => sub { sleep 1; }, tries => 3, try => sub { - my ( %args ) = @_; - eval { - $dbh->do($sql); - }; - if ( $EVAL_ERROR ) { - PTDEBUG && _d($EVAL_ERROR); - if ( $EVAL_ERROR =~ m/Lock wait timeout exceeded/ ) { - $error = $EVAL_ERROR; - if ( $args{tryno} > 1 ) { - $msg->("Lock wait timeout exceeded; retrying $sql"); - } - return; - } - die $EVAL_ERROR; - } - return 1; + $dbh->do($sql); + return; + }, + fail => sub { + my (%args) = @_; + my $error = $args{error}; + PTDEBUG && _d($error); + if ( $error =~ m/Lock wait timeout exceeded/ ) { + $msg->("Lock wait timeout exceeded; retrying $sql"); + return 1; # call wait, call try + } + return 0; # call final_fail + }, + final_fail => sub { + my (%args) = @_; + die $args{error}; }, - on_failure => sub { die $error; }, ); } @@ -4024,48 +3802,42 @@ sub new { sub retry { my ( $self, %args ) = @_; - my @required_args = qw(try wait); + my @required_args = qw(try fail final_fail); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; }; - my ($try, $wait) = @args{@required_args}; + my ($try, $fail, $final_fail) = @args{@required_args}; + my $wait = $args{wait} || sub { sleep 1; }; my $tries = $args{tries} || 3; + my $last_error; my $tryno = 0; + TRY: while ( ++$tryno <= $tries ) { - PTDEBUG && _d("Retry", $tryno, "of", $tries); + PTDEBUG && _d("Try", $tryno, "of", $tries); my $result; eval { $result = $try->(tryno=>$tryno); }; + if ( $EVAL_ERROR ) { + PTDEBUG && _d("Try code failed:", $EVAL_ERROR); + $last_error = $EVAL_ERROR; - if ( defined $result ) { - PTDEBUG && _d("Try code succeeded"); - if ( my $on_success = $args{on_success} ) { - PTDEBUG && _d("Calling on_success code"); - $on_success->(tryno=>$tryno, result=>$result); + if ( $tryno < $tries ) { # more retries + my $retry = $fail->(tryno=>$tryno, error=>$last_error); + last TRY unless $retry; + PTDEBUG && _d("Calling wait code"); + $wait->(tryno=>$tryno); } + } + else { + PTDEBUG && _d("Try code succeeded"); return $result; } - - if ( $EVAL_ERROR ) { - PTDEBUG && _d("Try code died:", $EVAL_ERROR); - die $EVAL_ERROR unless $args{retry_on_die}; - } - - if ( $tryno < $tries ) { - PTDEBUG && _d("Try code failed, calling wait code"); - $wait->(tryno=>$tryno); - } } - PTDEBUG && _d("Try code did not succeed"); - if ( my $on_failure = $args{on_failure} ) { - PTDEBUG && _d("Calling on_failure code"); - $on_failure->(); - } - - return; + PTDEBUG && _d('Try code did not succeed'); + return $final_fail->(error=>$last_error); } sub _d { @@ -4110,7 +3882,6 @@ sub main { my $vp = new VersionParser(); my $q = new Quoter(); my $tp = new TableParser(Quoter => $q); - my $du = new MySQLDump(); my $chunker = new TableChunker(Quoter => $q, TableParser => $tp); # ######################################################################## @@ -4225,7 +3996,6 @@ sub main { Quoter => $q, TableParser => $tp, TableChunker => $chunker, - MySQLDump => $du, VersionParser => $vp, ); @@ -4233,8 +4003,11 @@ sub main { # Create the capture-sync and copy-rows plugins. Currently, we just have # one method for each. # ######################################################################## - my $capture_sync = new OSCCaptureSync(); - my $copy_rows = new CopyRowsInsertSelect(Retry => new Retry()); + my $capture_sync = new OSCCaptureSync(Quoter => $q); + my $copy_rows = new CopyRowsInsertSelect( + Retry => new Retry(), + Quoter => $q, + ); # More values are added later. These are the minimum need to do --cleanup. my %plugin_args = ( @@ -4379,7 +4152,7 @@ sub main { # it manually. if ( !$o->get('print') ) { my $tmp_tbl_struct = $tp->parse( - $du->get_create_table($dbh, $q, $db, $tmp_tbl)); + $tp->get_create_table($dbh, $db, $tmp_tbl)); @columns = intersection([ $plugin_args{tbl_struct}->{is_col}, @@ -4489,7 +4262,7 @@ sub main { # ############################################################################ sub check_tables { my ( %args ) = @_; - my @required_args = qw(dbh db tbl tmp_tbl old_tbl VersionParser Quoter TableParser OptionParser TableChunker MySQLDump); + my @required_args = qw(dbh db tbl tmp_tbl old_tbl VersionParser Quoter TableParser OptionParser TableChunker); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } @@ -4528,8 +4301,7 @@ sub check_tables { # For now, we require that the old table has an exact-chunkable # column (i.e. unique single-column). - $tbl_info{tbl_struct} = $tp->parse( - $args{MySQLDump}->get_create_table($dbh, $args{Quoter}, $db, $tbl)); + $tbl_info{tbl_struct} = $tp->parse($tp->get_create_table($dbh, $db, $tbl)); my ($exact, @chunkable_cols) = $args{TableChunker}->find_chunk_columns( tbl_struct => $tbl_info{tbl_struct}, exact => 1, diff --git a/t/pt-online-schema-change/basics.t b/t/pt-online-schema-change/basics.t index c39c9526..fc9aacb1 100644 --- a/t/pt-online-schema-change/basics.t +++ b/t/pt-online-schema-change/basics.t @@ -25,7 +25,7 @@ if ( !$dbh ) { plan skip_all => 'Cannot connect to sandbox master'; } else { - plan tests => 18; + plan tests => 22; } my $output = ""; @@ -232,6 +232,51 @@ is( "Updated child table foreign key constraint (drop_old_table method)" ); +# ############################################################################# +# Alter tables with columns with resvered words and spaces. +# ############################################################################# +sub test_table { + my (%args) = @_; + my ($file, $name) = @args{qw(file name)}; + + $sb->load_file('master', "t/lib/samples/osc/$file"); + PerconaTest::wait_for_table($dbh, "osc.t", "id=5"); + PerconaTest::wait_for_table($dbh, "osc.__new_t"); + $dbh->do('use osc'); + $dbh->do("DROP TABLE IF EXISTS osc.__new_t"); + + $org_rows = $dbh->selectall_arrayref('select * from osc.t order by id'); + + output( + sub { $exit = pt_online_schema_change::main(@args, + 'D=osc,t=t', qw(--alter ENGINE=InnoDB)) }, + ); + + $new_rows = $dbh->selectall_arrayref('select * from osc.t order by id'); + + is_deeply( + $new_rows, + $org_rows, + "$name rows" + ); + + is( + $exit, + 0, + "$name exit status 0" + ); +} + +test_table( + file => "tbl002.sql", + name => "Reserved word column", +); + +test_table( + file => "tbl003.sql", + name => "Space column", +); + # ############################################################################# # Done. # ############################################################################# diff --git a/t/pt-online-schema-change/check_tables.t b/t/pt-online-schema-change/check_tables.t index f2d6b435..650870c1 100644 --- a/t/pt-online-schema-change/check_tables.t +++ b/t/pt-online-schema-change/check_tables.t @@ -29,7 +29,6 @@ else { my $vp = new VersionParser(); my $q = new Quoter(); my $tp = new TableParser(Quoter => $q); -my $du = new MySQLDump(); my $chunker = new TableChunker(Quoter => $q, TableParser => $tp); my $o = new OptionParser(); @@ -40,7 +39,7 @@ pt_online_schema_change::__set_quiet(1); $sb->load_file('master', "t/pt-online-schema-change/samples/small_table.sql"); $dbh->do('use mkosc'); -my $old_tbl_struct = $tp->parse($du->get_create_table($dbh, $q, 'mkosc', 'a')); +my $old_tbl_struct = $tp->parse($tp->get_create_table($dbh, 'mkosc', 'a')); my %args = ( dbh => $dbh, @@ -53,7 +52,6 @@ my %args = ( TableParser => $tp, OptionParser => $o, TableChunker => $chunker, - MySQLDump => $du, ); my %tbl_info = pt_online_schema_change::check_tables(%args); @@ -112,7 +110,7 @@ throws_ok( $dbh->do('DROP TRIGGER mkosc.foo'); $dbh->do('ALTER TABLE mkosc.a DROP COLUMN i'); -my $tmp_struct = $tp->parse($du->get_create_table($dbh, $q, 'mkosc', 'a')); +my $tmp_struct = $tp->parse($tp->get_create_table($dbh, 'mkosc', 'a')); throws_ok( sub { pt_online_schema_change::check_tables( %args,