diff --git a/lib/PerconaTest.pm b/lib/PerconaTest.pm index ea78972f..ed443da1 100644 --- a/lib/PerconaTest.pm +++ b/lib/PerconaTest.pm @@ -753,6 +753,27 @@ sub full_output { return ($output, $status); } +sub tables_used { + my ($file) = @_; + local $INPUT_RECORD_SEPARATOR = ''; + open my $fh, '<', $file or die "Cannot open $file: $OS_ERROR"; + my %tables; + while ( defined(my $chunk = <$fh>) ) { + map { + my $db_tbl = $_; + $db_tbl =~ s/^\s*`?//; # strip leading space and ` + $db_tbl =~ s/\s*`?$//; # strip trailing space and ` + $db_tbl =~ s/`\.`/./; # strip inner `.` + $tables{$db_tbl} = 1; + } + grep { + m/(?:\w\.\w|`\.`)/ # only db.tbl, not just db + } + $chunk =~ m/(?:FROM|INTO|UPDATE)\s+(\S+)/gi; + } + return [ sort keys %tables ]; +} + 1; } # ########################################################################### diff --git a/lib/Sandbox.pm b/lib/Sandbox.pm index d8eada7a..8b0712ad 100644 --- a/lib/Sandbox.pm +++ b/lib/Sandbox.pm @@ -374,6 +374,28 @@ sub verify_test_data { return; } +sub dsn_for { + my ($self, $host) = @_; + _check_server($host); + return "h=127.1,P=$port_for{$host},u=msandbox,p=msandbox"; +} + +sub genlog { + my ($self, $host) = @_; + _check_server($host); + return "/tmp/$port_for{$host}/data/genlog"; +} + +sub clear_genlogs { + my ($self, @hosts) = @_; + @hosts = qw(master slave1 slave2) unless scalar @hosts; + foreach my $host ( @hosts ) { + PTDEVDEBUG && _d('Clearing general log on', $host); + Test::More::diag(`echo > /tmp/$port_for{$host}/data/genlog`); + } + return; +} + 1; } # ########################################################################### diff --git a/sandbox/servers/5.0/my.sandbox.cnf b/sandbox/servers/5.0/my.sandbox.cnf index 8d8c1074..858b9f62 100644 --- a/sandbox/servers/5.0/my.sandbox.cnf +++ b/sandbox/servers/5.0/my.sandbox.cnf @@ -24,4 +24,4 @@ report-host = 127.0.0.1 report-port = PORT log-error = mysqld.log innodb_lock_wait_timeout = 3 -log +log = genlog diff --git a/sandbox/servers/5.1/my.sandbox.cnf b/sandbox/servers/5.1/my.sandbox.cnf index 8d8c1074..858b9f62 100644 --- a/sandbox/servers/5.1/my.sandbox.cnf +++ b/sandbox/servers/5.1/my.sandbox.cnf @@ -24,4 +24,4 @@ report-host = 127.0.0.1 report-port = PORT log-error = mysqld.log innodb_lock_wait_timeout = 3 -log +log = genlog diff --git a/sandbox/servers/5.5/my.sandbox.cnf b/sandbox/servers/5.5/my.sandbox.cnf index 8adc6fa6..d7f23f8c 100644 --- a/sandbox/servers/5.5/my.sandbox.cnf +++ b/sandbox/servers/5.5/my.sandbox.cnf @@ -24,4 +24,4 @@ report-host = 127.0.0.1 report-port = PORT log-error = mysqld.log innodb_lock_wait_timeout = 3 -general_log +general_log = genlog diff --git a/t/lib/samples/SchemaIterator.sql b/t/lib/samples/SchemaIterator.sql index 5969a46f..c06922d0 100644 --- a/t/lib/samples/SchemaIterator.sql +++ b/t/lib/samples/SchemaIterator.sql @@ -1,24 +1,43 @@ +-- +-- d1 +-- DROP DATABASE IF EXISTS d1; -DROP DATABASE IF EXISTS d2; -DROP DATABASE IF EXISTS d3; - CREATE DATABASE d1; -CREATE DATABASE d2; -CREATE DATABASE d3; CREATE TABLE d1.t1 ( - i int + id int auto_increment primary key, + c char(8) ) ENGINE=MyISAM CHARSET=utf8; -INSERT INTO d1.t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9); + CREATE TABLE d1.t2 ( - c varchar(255) + id int auto_increment primary key, + c varchar(8) ) ENGINE=InnoDB; + CREATE TABLE d1.t3 ( - i int + id int auto_increment primary key, + c char(8) ) ENGINE=MEMORY; -CREATE TABLE d2.t1 ( - i int -); +INSERT INTO d1.t1 VALUES (null, 'a'), (null, 'b'), (null, 'c'), (null, 'd'), (null, 'e'), (null, 'f'), (null, 'g'), (null, 'h'), (null, 'i'); +INSERT INTO d1.t2 SELECT * FROM d1.t1; +INSERT INTO d1.t3 SELECT * FROM d1.t1; +-- +-- d2 +-- +DROP DATABASE IF EXISTS d2; +CREATE DATABASE d2; + +CREATE TABLE d2.t1 ( + id int auto_increment primary key, + c char(8) +); +-- d2.t1 is an empty table. + +-- +-- d3 +-- +DROP DATABASE IF EXISTS d3; +CREATE DATABASE d3; -- d3 is an empty database. diff --git a/t/pt-table-sync/filters.t b/t/pt-table-sync/filters.t index 9c2d432a..3804f473 100644 --- a/t/pt-table-sync/filters.t +++ b/t/pt-table-sync/filters.t @@ -10,12 +10,12 @@ use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use Test::More; +use Data::Dumper; use PerconaTest; use Sandbox; require "$trunk/bin/pt-table-sync"; - my $vp = new VersionParser(); my $dp = new DSNParser(opts=>$dsn_opts); my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); @@ -29,81 +29,145 @@ elsif ( !$slave_dbh ) { plan skip_all => 'Cannot connect to sandbox slave'; } else { - plan tests => 9; + plan tests => 18; } -# Previous tests slave 12347 to 12346 which makes pt-table-checksum -# complain that it cannot connect to 12347 for checking repl filters -# and such. 12347 isn't present but SHOW SLAVE HOSTS on 12346 hasn't -# figured that out yet, so we restart 12346 to refresh this list. -#diag(`/tmp/12346/stop >/dev/null`); -#diag(`/tmp/12346/start >/dev/null`); -$slave_dbh = $sb->get_dbh_for('slave1'); - +my $master_dsn = $sb->dsn_for('master'); +my $slave1_dsn = $sb->dsn_for('slave1'); my $output; -my $cnf = "/tmp/12345/my.sandbox.cnf"; -my $cmd = "$trunk/bin/pt-table-sync -F $cnf"; -my $t = qr/\d\d:\d\d:\d\d/; -$sb->wipe_clean($master_dbh); -$sb->load_file('master', 't/pt-table-sync/samples/filter_tables.sql'); +# See this SQL file because it has a number of simple dbs and tbls +# that are used to checking schema object filters. +$sb->load_file('master', "t/lib/samples/SchemaIterator.sql"); -$output = `$cmd h=127.1,P=12345 P=12346 --no-check-slave --dry-run -t issue_806_1.t2 | tail -n 2`; -$output =~ s/$t/00:00:00/g; -$output =~ s/[ ]{2,}/ /g; -is( - $output, -"# DELETE REPLACE INSERT UPDATE ALGORITHM START END EXIT DATABASE.TABLE -# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_1.t2 -", - "db-qualified --tables (issue 806)" +sub test_filters { + my (%args) = @_; + + $sb->clear_genlogs(); + + my $output = output( + sub { pt_table_sync::main(@{$args{cmds}}, + qw(--print)) + }, + ); + + my $tables_used = PerconaTest::tables_used($sb->genlog('master')); + is_deeply( + $tables_used, + $args{res}, + "$args{name}: tables used" + ) or diag(Dumper($tables_used)); +} + +# ############################################################################# +# Basic schema object filters: --databases, --tables, etc. +# ############################################################################# + +# Not really a filter, but only the specified tables should be used. +test_filters( + name => "Sync d1.t1 to d1.t2", + cmds => ["$master_dsn,D=d1,t=t1", "t=t2"], + res => [qw(d1.t1 d1.t2)], +); + +# Use slave1 like it's another master, ok becuase we're not actually +# syncing anything, so --no-check-slave is required else pt-table-sync +# won't run (because it doesn't like to sync directly to a slave). +test_filters( + name => "-t d1.t1", + cmds => [$master_dsn, $slave1_dsn, qw(--no-check-slave), + qw(-t d1.t1)], + res => [qw(d1.t1)], +); + +# Like the previous test, but now any table called "t1". +test_filters( + name => "-t t1", + cmds => [$master_dsn, $slave1_dsn, qw(--no-check-slave), + qw(-t t1)], + res => [qw(d1.t1 d2.t1)], +); + +# Only the given db, all its tables: there's only 1 tbl in d2. +test_filters( + name => "--databases d2", + cmds => [$master_dsn, $slave1_dsn, qw(--no-check-slave), + qw(--databases d2)], + res => [qw(d2.t1)], +); + +# --database with --tables. +test_filters( + name => "--databases d1 --tables t2,t3", + cmds => [$master_dsn, $slave1_dsn, qw(--no-check-slave), + qw(--databases d1), "--tables", "t2,t3"], + res => [qw(d1.t2 d1.t3)], ); # ############################################################################# -# Issue 820: Make mk-table-sync honor schema filters with --replicate +# Filters with --replicate and --sync-to-master. # ############################################################################# -$master_dbh->do('DROP DATABASE IF EXISTS test'); -$master_dbh->do('CREATE DATABASE test'); + +# Checksum the filter tables. +$master_dbh->do("DROP DATABASE IF EXISTS percona"); +$sb->wait_for_slaves(); +diag(`$trunk/bin/pt-table-checksum $master_dsn -d d1,d2,d3 --chunk-size 100 --quiet --lock-wait-timeout 3 --max-load ''`); -$slave_dbh->do('insert into issue_806_1.t1 values (41)'); -$slave_dbh->do('insert into issue_806_2.t2 values (42)'); +my $rows = $master_dbh->selectall_arrayref("SELECT CONCAT(db, '.', tbl) FROM percona.checksums ORDER BY db, tbl"); +is_deeply( + $rows, + [ + ["d1.t1"], + ["d1.t2"], + ["d1.t3"], + ["d2.t1"], + ], + "Checksummed all tables" +) or diag(Dumper($rows)); -my $mk_table_checksum = "$trunk/bin/pt-table-checksum --lock-wait-time 3"; +# Make all checksums on the slave different than the master +# so that pt-table-sync would sync all the tables if there +# were no filters. +$slave_dbh->do("UPDATE percona.checksums SET this_cnt=999 WHERE 1=1"); -`$mk_table_checksum --replicate test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d issue_806_1,issue_806_2 --quiet`; - -$output = `$cmd h=127.1,P=12345 --replicate test.checksum --dry-run | tail -n 2`; -$output =~ s/$t/00:00:00/g; -$output =~ s/[ ]{2,}/ /g; -is( - $output, -"# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_1.t1 -# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_2.t2 -", - "--replicate with no filters" +# Verify that that ^ is true. +test_filters( + name => "All tables are different on the slave", + cmds => [$master_dsn, qw(--replicate percona.checksums)], + res => [qw(d1.t1 d1.t2 d1.t3 d2.t1 percona.checksums)], ); -$output = `$cmd h=127.1,P=12345 --replicate test.checksum --dry-run -t t1 | tail -n 2`; -$output =~ s/$t/00:00:00/g; -$output =~ s/[ ]{2,}/ /g; -is( - $output, -"# DELETE REPLACE INSERT UPDATE ALGORITHM START END EXIT DATABASE.TABLE -# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_1.t1 -", - "--replicate with --tables" -); +# Sync with --replicate, --sync-to-master, and some filters. +# --replicate and --sync-to-master have different code paths, +# but the filter results should be the same. +foreach my $args ( + [$master_dsn, qw(--replicate percona.checksums)], + [$slave1_dsn, qw(--replicate percona.checksums --sync-to-master)] +) { -$output = `$cmd h=127.1,P=12345 --replicate test.checksum --dry-run -d issue_806_2 | tail -n 2`; -$output =~ s/$t/00:00:00/g; -$output =~ s/[ ]{2,}/ /g; -is( - $output, -"# DELETE REPLACE INSERT UPDATE ALGORITHM START END EXIT DATABASE.TABLE -# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_2.t2 -", - "--replicate with --databases" -); + my $stm = $args->[-1] eq '--sync-to-master' ? ' --sync-to-master' : ''; + + test_filters( + name => $stm . "--replicate --tables t1", + cmds => [@$args, + qw(--tables t1)], + res => [qw(d1.t1 d2.t1 percona.checksums)], + ); + + test_filters( + name => $stm . "--replicate --databases d2", + cmds => [@$args, + qw(--databases d2)], + res => [qw(d2.t1 percona.checksums)], + ); + + test_filters( + name => $stm . "--replicate --databases d1 --tables t1,t3", + cmds => [@$args, + qw(--databases d1), "--tables", "t1,t3"], + res => [qw(d1.t1 d1.t3 percona.checksums)], + ); +} # ############################################################################# # pt-table-sync --ignore-* options don't work with --replicate @@ -120,7 +184,7 @@ $slave_dbh->do("INSERT INTO test.empty_it VALUES (null,11,11,'eleven')"); diag(`$trunk/bin/pt-table-checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test --quiet --quiet --lock-wait-timeout 3 --max-load ''`); # Make sure all the tables were checksummed. -my $rows = $master_dbh->selectall_arrayref("SELECT DISTINCT db, tbl FROM percona.checksums ORDER BY db, tbl"); +$rows = $master_dbh->selectall_arrayref("SELECT DISTINCT db, tbl FROM percona.checksums ORDER BY db, tbl"); is_deeply( $rows, [ [qw(test empty_it) ], diff --git a/t/pt-table-sync/issue_79.t b/t/pt-table-sync/issue_79.t deleted file mode 100644 index 0b9f2ad0..00000000 --- a/t/pt-table-sync/issue_79.t +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env perl - -BEGIN { - die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" - unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; - unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; -}; - -use strict; -use warnings FATAL => 'all'; -use English qw(-no_match_vars); -use Test::More; - -use PerconaTest; -use Sandbox; -require "$trunk/bin/pt-table-sync"; - -my $output; -my $dp = new DSNParser(opts=>$dsn_opts); -my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); -my $master_dbh = $sb->get_dbh_for('master'); -my $slave_dbh = $sb->get_dbh_for('slave1'); - -if ( !$master_dbh ) { - plan skip_all => 'Cannot connect to sandbox master'; -} -elsif ( !$slave_dbh ) { - plan skip_all => 'Cannot connect to sandbox slave'; -} -else { - plan tests => 7; -} - -$sb->wipe_clean($master_dbh); -$sb->create_dbs($master_dbh, [qw(test)]); -$sb->load_file('master', 't/pt-table-sync/samples/before.sql'); -PerconaTest::wait_for_table($slave_dbh, "test.test4", "id=15034"); - -# ############################################################################# -# Issue 79: mk-table-sync with --replicate doesn't honor --tables -# ############################################################################# - -$master_dbh->do('create table test.messages (i int)'); -PerconaTest::wait_for_table($slave_dbh, "test.messages", "i=3"); -$slave_dbh->do('insert into test.messages values (1), (2), (3)'); - -# The previous test should have left test.messages on the slave (12346) -# out of sync. Now we also unsync test2 on the slave and then re-sync only -# it. If --tables is honored, only test2 on the slave will be synced. -$sb->use('master', "-D test -e \"SET SQL_LOG_BIN=0; INSERT INTO test2 VALUES (1,'a'),(2,'b')\""); -diag(`$trunk/bin/pt-table-checksum --replicate=test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test --lock-wait-time 3 > /dev/null`); - -is_deeply( - $master_dbh->selectall_arrayref('select * from test.messages'), - [], - 'test.messages on master empty' -); -is_deeply( - $slave_dbh->selectall_arrayref('select * from test.messages'), - [[1],[2],[3]], - 'test.messages on slave has values' -); - -# Test that what the doc says about --tables is true: -# "Table names may be qualified with the database name." -# In the code, a qualified db.tbl name is used. -# So we'll test first an unqualified tbl name. -$output = `$trunk/bin/pt-table-sync h=127.1,P=12345,u=msandbox,p=msandbox --replicate test.checksum --execute -d test -t test2 -v`; -unlike($output, qr/messages/, '--replicate honors --tables (1/4)'); -like($output, qr/test2/, '--replicate honors --tables (2/4)'); - -# Now we'll test with a qualified db.tbl name. -$sb->use('slave1', '-D test -e "TRUNCATE TABLE test2; TRUNCATE TABLE messages"'); -diag(`$trunk/bin/pt-table-checksum --replicate=test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test --lock-wait-time 3 > /dev/null`); - -$output = `$trunk/bin/pt-table-sync h=127.1,P=12345,u=msandbox,p=msandbox --replicate test.checksum --execute -d test -t test.test2 -v`; -unlike($output, qr/messages/, '--replicate honors --tables (3/4)'); -like($output, qr/test2/, '--replicate honors --tables (4/4)'); - -# ############################################################################# -# Done. -# ############################################################################# -$sb->wipe_clean($master_dbh); -ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox"); -exit; diff --git a/t/pt-table-sync/samples/filter_tables.sql b/t/pt-table-sync/samples/filter_tables.sql deleted file mode 100644 index e0f16b13..00000000 --- a/t/pt-table-sync/samples/filter_tables.sql +++ /dev/null @@ -1,13 +0,0 @@ -drop database if exists issue_806_1; -drop database if exists issue_806_2; -create database issue_806_1; -create database issue_806_2; - -use issue_806_1; -create table t1 (i int, unique index (i)); -create table t2 (i int, unique index (i)); - -use issue_806_2; -create table t1 (i int, unique index (i)); -create table t2 (i int, unique index (i)); -create table t3 (i int, unique index (i));