From 5c80f55874cbf5c6e6fca39c125f50fbe5dc5fc7 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Thu, 20 Oct 2011 13:46:33 -0600 Subject: [PATCH] USE repl db when deleting from repl db. Move and update issue_982.t tests to replication_filters.t and remove issue_982.t. --- bin/pt-table-checksum | 67 +++---- t/pt-table-checksum/issue_982.t | 217 ---------------------- t/pt-table-checksum/replication_filters.t | 171 ++++++++++++++++- 3 files changed, 202 insertions(+), 253 deletions(-) delete mode 100644 t/pt-table-checksum/issue_982.t diff --git a/bin/pt-table-checksum b/bin/pt-table-checksum index a4781046..2ffebbb3 100755 --- a/bin/pt-table-checksum +++ b/bin/pt-table-checksum @@ -5975,10 +5975,29 @@ sub main { $last_chunk = undef; } elsif ( $o->get('empty-replicate-table') ) { + use_repl_db( + dbh => $master_cxn->dbh(), + repl_table => $repl_table, + OptionParser => $o, + Quoter => $q, + ); MKDEBUG && _d($delete_sth->{Statement}); $delete_sth->execute($tbl->{db}, $tbl->{tbl}); } + # USE the correct db while checksumming this table. The "correct" + # db is a complicated subject; see sub for comments. + use_repl_db( + dbh => $master_cxn->dbh(), + tbl => $tbl, # XXX working on this table + repl_table => $repl_table, + OptionParser => $o, + Quoter => $q, + ); + # ######################################################### + # XXX DO NOT CHANGE THE DB UNTIL THIS TABLE IS FINISHED XXX + # ######################################################### + return $oktonibble; # continue nibbling table? }, next_boundaries => sub { @@ -6294,16 +6313,6 @@ sub main { # be saved here. print_checksum_results() uses this info. $tbl->{checksum_results} = {}; - # USE the correct db while checksumming this table. The "correct" - # db is a complicated subject; see sub for comments. - use_repl_db( - dbh => $master_cxn->dbh(), - tbl => $tbl, - repl_table => $repl_table, - OptionParser => $o, - Quoter => $q, - ); - # Set table's initial chunk size. If this is the first table, # then total rate will be zero, so use --chunk-size. Or, if # --chunk-time=0, then only use --chunk-size for every table. @@ -6737,9 +6746,6 @@ sub check_repl_table { # # Returns: # Nothing or dies on error -{ -my $current_db; - sub use_repl_db { my ( %args ) = @_; my @required_args = qw(dbh repl_table OptionParser Quoter); @@ -6747,6 +6753,7 @@ sub use_repl_db { die "I need a $arg argument" unless $args{$arg}; } my ($dbh, $repl_table, $o, $q) = @args{@required_args}; + MKDEBUG && _d('use_repl_db'); my ($db, $tbl) = $q->split_unquote($repl_table); if ( my $tbl = $args{tbl} ) { @@ -6765,30 +6772,26 @@ sub use_repl_db { $db = $o->get('replicate-database') if $o->get('replicate-database'); } - if ( !$current_db || $current_db ne $db ) { - eval { - my $sql = "USE " . $q->quote($db); - MKDEBUG && _d($sql); - $dbh->do($sql); - }; - if ( $EVAL_ERROR ) { - # Report which option db really came from. - my $opt = $o->get('replicate-database') ? "--replicate-database" - : "--replicate database"; - if ( $EVAL_ERROR =~ m/unknown database/i ) { - die "$opt $db does not exist. You need to create the " - . "database or specify a database for $opt that exists.\n"; - } - else { - die "Error using $opt $db: $EVAL_ERROR\n"; - } + eval { + my $sql = "USE " . $q->quote($db); + MKDEBUG && _d($sql); + $dbh->do($sql); + }; + if ( $EVAL_ERROR ) { + # Report which option db really came from. + my $opt = $o->get('replicate-database') ? "--replicate-database" + : "--replicate database"; + if ( $EVAL_ERROR =~ m/unknown database/i ) { + die "$opt $db does not exist. You need to create the " + . "database or specify a database for $opt that exists.\n"; + } + else { + die "Error using $opt $db: $EVAL_ERROR\n"; } - $current_db = $db; } return; } -} sub create_repl_table { my ( %args ) = @_; diff --git a/t/pt-table-checksum/issue_982.t b/t/pt-table-checksum/issue_982.t deleted file mode 100644 index 3bc927a2..00000000 --- a/t/pt-table-checksum/issue_982.t +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env perl - -BEGIN { - die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" - unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; - unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; -}; - -use strict; -use warnings FATAL => 'all'; -use English qw(-no_match_vars); -use Test::More skip_all => 'Finish updating issue_982.t'; - -use PerconaTest; -use Sandbox; -require "$trunk/bin/pt-table-checksum"; - -my $dp = new DSNParser(opts=>$dsn_opts); -my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); -my $master_dbh = $sb->get_dbh_for('master'); -my $slave1_dbh = $sb->get_dbh_for('slave1'); - -if ( !$master_dbh ) { - plan skip_all => 'Cannot connect to sandbox master'; -} -if ( !$slave1_dbh ) { - plan skip_all => 'Cannot connect to sandbox slave'; -} -else { - plan tests => 8; -} - -# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic -# so we need to specify --lock-wait-timeout=3 else the tool will die. -# And --max-load "" prevents waiting for status variables. -my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox'; -my @args = ($master_dsn, qw(--lock-wait-timeout 3), '--max-load', ''); -my $row; -my $output; - -# ############################################################################# -# Issue 982: --empty-replicate-table does not work with binlog-ignore-db -# ############################################################################# - -$sb->wipe_clean($master_dbh); -pt_table_checksum::main(@args, qw(-t sakila.country --quiet)); -$master_dbh->do("insert into percona.checksums values ('sakila', 'country', 99, null, null, null, null, '', 0, '', 0, null)"); -PerconaTest::wait_for_table($slave1_dbh, 'percona.checksums', "db='sakila' and tbl='country' and chunk=99"); - -$master_dbh->disconnect(); -$slave1_dbh->disconnect(); - -# Add a replication filter to the slave. -diag(`/tmp/12346/stop >/dev/null`); -diag(`/tmp/12345/stop >/dev/null`); -diag(`cp /tmp/12345/my.sandbox.cnf /tmp/12345/orig.cnf`); -diag(`echo "binlog-ignore-db=sakila" >> /tmp/12345/my.sandbox.cnf`); -diag(`echo "binlog-ignore-db=mysql" >> /tmp/12345/my.sandbox.cnf`); -diag(`/tmp/12345/start >/dev/null`); -diag(`/tmp/12346/start >/dev/null`); - -$output = output( - sub { pt_table_checksum::main(@args, qw(--no-check-replication-filters), - qw(-d mysql -t user)) }, - stderr => 1, -); - -$master_dbh = $sb->get_dbh_for('master'); -$slave1_dbh = $sb->get_dbh_for('slave1'); - -$row = $slave1_dbh->selectall_arrayref("select * from percona.checksums where db='sakila' and tbl='country' and chunk=99"); -ok( - @$row == 0, - "Slave checksum table deleted" -); - -# Clear checksum table for next tests. -$master_dbh->do("truncate table percona.checksums"); -wait_until( - sub { - $row = $slave1_dbh->selectall_arrayref("select * from percona.checksums"); - return !@$row; - } -); - -$master_dbh->disconnect(); -$slave1_dbh->disconnect(); - -# Restore original config. -diag(`/tmp/12346/stop >/dev/null`); -diag(`/tmp/12345/stop >/dev/null`); -diag(`cp /tmp/12345/orig.cnf /tmp/12345/my.sandbox.cnf`); - -# ############################################################################# -# Test --replicate-database which resulted from this issue. -# ############################################################################# - -# Add a binlog-do-db filter so master will only replicate -# statements when USE mysql is in effect. -diag(`echo "binlog-do-db=mysql" >> /tmp/12345/my.sandbox.cnf`); -diag(`/tmp/12345/start >/dev/null`); -diag(`/tmp/12346/start >/dev/null`); - -$master_dbh = $sb->get_dbh_for('master'); -$slave1_dbh = $sb->get_dbh_for('slave1'); - -$output = output( - sub { pt_table_checksum::main(@args, qw(--no-check-replication-filters), - qw(--replicate=percona.checksums -d mysql -t user)) - }, - stderr => 1, -); - -# Because we did not use --replicate-database, mk-table-checksum should -# have done USE mysql before updating the checksum table. Thus, the -# checksums should show up on the slave. -sleep 1; -$row = $slave1_dbh->selectall_arrayref("select * from percona.checksums where db='mysql' AND tbl='user'"); -ok( - @$row == 1, - "Checksum replicated with binlog-do-db, without --replicate-database" -); - -# Now force --replicate-database test and the checksums should not replicate. - -$master_dbh->do("use mysql"); -$master_dbh->do("truncate table percona.checksums"); -sleep 1; -$row = $slave1_dbh->selectall_arrayref("select * from percona.checksums"); -ok( - !@$row, - "Checksum table empty on slave" -); - -$output = output( - sub { pt_table_checksum::main(@args, qw(--no-check-replication-filters), - qw(--replicate=percona.checksums -d mysql -t user), - qw(--replicate-database test)) - }, - stderr => 1, -); -sleep 1; -$row = $slave1_dbh->selectall_arrayref("select * from percona.checksums where db='mysql' AND tbl='user'"); -ok( - !@$row, - "Checksum did not replicated with binlog-do-db, with --replicate-database" -); - -# ############################################################################# -# Restore original config. -# ############################################################################# -$master_dbh->disconnect(); -$slave1_dbh->disconnect(); - -diag(`/tmp/12346/stop >/dev/null`); -diag(`/tmp/12345/stop >/dev/null`); -diag(`mv /tmp/12345/orig.cnf /tmp/12345/my.sandbox.cnf`); -diag(`/tmp/12345/start >/dev/null`); -diag(`/tmp/12346/start >/dev/null`); - -$master_dbh = $sb->get_dbh_for('master'); -$slave1_dbh = $sb->get_dbh_for('slave1'); - -# ############################################################################# -# Test it again by looking at binlog to see that the db didn't change. -# ############################################################################# -diag(`$trunk/sandbox/test-env reset`); -sleep 1; - -# To speed this test up, ignore these tables. -# http://code.google.com/p/maatkit/issues/detail?id=1027 -my $it = "payment,rental,help_topic,help_keyword,inventory,film_actor"; - -$output = output( - sub { pt_table_checksum::main(@args, - qw(--replicate=percona.checksums), '--ignore-tables', $it, qw(--chunk-size 20k)) - }, - stderr => 1, -); -sleep 1; - -$row = $master_dbh->selectrow_hashref('show master status'); -$output = `$ENV{PERCONA_TOOLKIT_SANDBOX}/bin/mysqlbinlog /tmp/12345/data/$row->{file} | grep 'use ' | grep -v '^# Warning' | sort -u`; -is( - $output, -"use mysql/*!*/; -use sakila/*!*/; -use test/*!*/; -", - "USE each table's db (binlog dump)" -); - -diag(`$trunk/sandbox/test-env reset`); -sleep 1; - -$output = output( - sub { pt_table_checksum::main(@args, qw(--replicate-database test), - qw(--replicate=percona.checksums), '--ignore-tables', $it, qw(--chunk-size 20k)) - }, - stderr => 1, -); -sleep 1; - -$row = $master_dbh->selectrow_hashref('show master status'); -$output = `$ENV{PERCONA_TOOLKIT_SANDBOX}/bin/mysqlbinlog /tmp/12345/data/$row->{file} | grep 'use ' | grep -v '^# Warning'`; -is( - $output, -"use test/*!*/; -", - "USE only --replicate-database db (binlog dump)" -); - -# ############################################################################# -# Done. -# ############################################################################# -$sb->wipe_clean($master_dbh); -exit; diff --git a/t/pt-table-checksum/replication_filters.t b/t/pt-table-checksum/replication_filters.t index 6cc014e0..e208fd8c 100644 --- a/t/pt-table-checksum/replication_filters.t +++ b/t/pt-table-checksum/replication_filters.t @@ -36,7 +36,7 @@ elsif ( !$slave2_dbh ) { plan skip_all => 'Cannot connect to sandbox slave2'; } else { - plan tests => 4; + plan tests => 10; } # The sandbox servers run with lock_wait_timeout=3 and it's not dynamic @@ -93,14 +93,177 @@ like( "--no-check-replication-filters" ); -# ############################################################################# -# Done. -# ############################################################################# # Remove the replication filter from the slave. for my $port ( qw(12346 12347) ) { diag(`/tmp/$port/stop >/dev/null`); diag(`mv /tmp/$port/orig.cnf /tmp/$port/my.sandbox.cnf`); diag(`/tmp/$port/start >/dev/null`); } + +# ############################################################################# +# Issue 982: --empty-replicate-table does not work with binlog-ignore-db +# ############################################################################# + +# Write some results to master and slave for dbs mysql and sakila. +$sb->wipe_clean($master_dbh); +pt_table_checksum::main(@args, qw(--chunk-time 0 --chunk-size 100), + '-t', 'mysql.user,sakila.city', qw(--quiet)); +PerconaTest::wait_for_table($slave1_dbh, 'percona.checksums', "db='sakila' and tbl='city' and chunk=6"); + +$master_dbh->disconnect(); +$slave1_dbh->disconnect(); + +# Add a replication filter to the slave: ignore db mysql. +diag(`/tmp/12346/stop >/dev/null`); +diag(`/tmp/12345/stop >/dev/null`); + +diag(`cp /tmp/12345/my.sandbox.cnf /tmp/12345/orig.cnf`); +diag(`echo "binlog-ignore-db=mysql" >> /tmp/12345/my.sandbox.cnf`); + +diag(`/tmp/12345/start >/dev/null`); +diag(`/tmp/12346/start >/dev/null`); +$master_dbh = $sb->get_dbh_for('master'); +$slave1_dbh = $sb->get_dbh_for('slave1'); + +# Checksum the tables again in 1 chunk. Since db percona isn't being +# ignored, deleting old results in the repl table should replicate. +# But since db mysql is ignored, the new results for mysql.user should +# not replicate. +pt_table_checksum::main(@args, qw(--no-check-replication-filters), + '-t', 'mysql.user,sakila.city', qw(--quiet --no-replicate-check)); + +PerconaTest::wait_for_table($slave1_dbh, 'percona.checksums', "db='sakila' and tbl='city' and chunk=1"); + +$row = $slave1_dbh->selectall_arrayref("select db,tbl,chunk from percona.checksums order by db,tbl,chunk"); +is_deeply( + $row, + [[qw(sakila city 1)]], + "binlog-ignore-db" +) or print STDERR Dumper($row); + +$master_dbh->do("use percona"); +$master_dbh->do("truncate table percona.checksums"); +wait_until( + sub { + $row = $slave1_dbh->selectall_arrayref("select * from percona.checksums"); + return !@$row; + } +); + +$master_dbh->disconnect(); +$slave1_dbh->disconnect(); + +# Restore original config. +diag(`/tmp/12346/stop >/dev/null`); +diag(`/tmp/12345/stop >/dev/null`); +diag(`cp /tmp/12345/orig.cnf /tmp/12345/my.sandbox.cnf`); + +# ############################################################################# +# Test --replicate-database which resulted from this issue. +# ############################################################################# + +# Add a binlog-do-db filter so master will only replicate +# statements when USE mysql is in effect. +diag(`echo "binlog-do-db=mysql" >> /tmp/12345/my.sandbox.cnf`); +diag(`/tmp/12345/start >/dev/null`); +diag(`/tmp/12346/start >/dev/null`); + +$master_dbh = $sb->get_dbh_for('master'); +$slave1_dbh = $sb->get_dbh_for('slave1'); + +$output = output( + sub { pt_table_checksum::main(@args, qw(--no-check-replication-filters), + qw(-d mysql -t user)) + }, + stderr => 1, +); + +# Because we did not use --replicate-database, pt-table-checksum should +# have done USE mysql before updating the repl table. Thus, the +# checksums should show up on the slave. +PerconaTest::wait_for_table($slave1_dbh, 'percona.checksums', "db='mysql' and tbl='user' and chunk=1"); + +$row = $slave1_dbh->selectall_arrayref("select db,tbl,chunk from percona.checksums order by db,tbl,chunk"); +is_deeply( + $row, + [[qw(mysql user 1)]], + "binlog-do-do, without --replicate-database" +) or print STDERR Dumper($row); + +# Now force --replicate-database test and the checksums should not replicate. +$master_dbh->do("use mysql"); +$master_dbh->do("truncate table percona.checksums"); +wait_until( + sub { + $row = $slave1_dbh->selectall_arrayref("select * from percona.checksums"); + return !@$row; + } +); + +$pos = PerconaTest::get_master_binlog_pos($master_dbh); + +pt_table_checksum::main(@args, qw(--quiet --no-check-replication-filters), + qw(-t mysql.user --replicate-database sakila --no-replicate-check)); + +sleep 1; + +$row = $slave1_dbh->selectall_arrayref("select * from percona.checksums where db='mysql' AND tbl='user'"); +ok( + !@$row, + "binlog-do-db, with --replicate-database" +) or print STDERR Dumper($row); + +is( + PerconaTest::get_master_binlog_pos($master_dbh), + $pos, + "Master pos did not change" +); + +# ############################################################################# +# Restore original config. +# ############################################################################# +$master_dbh->disconnect(); +$slave1_dbh->disconnect(); + +diag(`/tmp/12346/stop >/dev/null`); +diag(`/tmp/12345/stop >/dev/null`); +diag(`mv /tmp/12345/orig.cnf /tmp/12345/my.sandbox.cnf`); +diag(`/tmp/12345/start >/dev/null`); +diag(`/tmp/12346/start >/dev/null`); + +diag(`$trunk/sandbox/test-env reset`); + +$master_dbh = $sb->get_dbh_for('master'); +$slave1_dbh = $sb->get_dbh_for('slave1'); + +pt_table_checksum::main(@args, qw(--quiet)); + +$row = $master_dbh->selectrow_hashref('show master status'); +$output = `$ENV{PERCONA_TOOLKIT_SANDBOX}/bin/mysqlbinlog /tmp/12345/data/$row->{file} | grep 'use ' | grep -v '^# Warning' | sort -u`; + +is( + $output, +"use mysql/*!*/; +use percona/*!*/; +use sakila/*!*/; +", + "USE each table's database (binlog dump)" +); + +diag(`$trunk/sandbox/test-env reset`); + +pt_table_checksum::main(@args, qw(--quiet --replicate-database percona)); + +$output = `$ENV{PERCONA_TOOLKIT_SANDBOX}/bin/mysqlbinlog /tmp/12345/data/$row->{file} | grep 'use ' | grep -v '^# Warning'`; +is( + $output, +"use percona/*!*/; +", + "USE only --replicate-database (binlog dump)" +); + +# ############################################################################# +# Done. +# ############################################################################# $sb->wipe_clean($master_dbh); exit;