percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-24 05:15:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

USE repl db when deleting from repl db. Move and update issue_982.t tests to replication_filters.t and remove issue_982.t.

Browse Source
This commit is contained in:
Daniel Nichter
2011-10-20 13:46:33 -06:00
parent 41821380b6
commit 5c80f55874
3 changed files with 202 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
bin/pt-table-checksum
View File

@@ -5975,10 +5975,29 @@ sub main {
$last_chunk = undef; $last_chunk = undef;
} }
elsif ( $o->get('empty-replicate-table') ) { elsif ( $o->get('empty-replicate-table') ) {
use_repl_db(
dbh => $master_cxn->dbh(),
repl_table => $repl_table,
OptionParser => $o,
Quoter => $q,
);
MKDEBUG && _d($delete_sth->{Statement}); MKDEBUG && _d($delete_sth->{Statement});
$delete_sth->execute($tbl->{db}, $tbl->{tbl}); $delete_sth->execute($tbl->{db}, $tbl->{tbl});
} }
# USE the correct db while checksumming this table. The "correct"
# db is a complicated subject; see sub for comments.
use_repl_db(
dbh => $master_cxn->dbh(),
tbl => $tbl, # XXX working on this table
repl_table => $repl_table,
OptionParser => $o,
Quoter => $q,
);
# #########################################################
# XXX DO NOT CHANGE THE DB UNTIL THIS TABLE IS FINISHED XXX
# #########################################################
return $oktonibble; # continue nibbling table? return $oktonibble; # continue nibbling table?
}, },
next_boundaries => sub { next_boundaries => sub {
@@ -6294,16 +6313,6 @@ sub main {
# be saved here. print_checksum_results() uses this info. # be saved here. print_checksum_results() uses this info.
$tbl->{checksum_results} = {}; $tbl->{checksum_results} = {};
# USE the correct db while checksumming this table. The "correct"
# db is a complicated subject; see sub for comments.
use_repl_db(
dbh => $master_cxn->dbh(),
tbl => $tbl,
repl_table => $repl_table,
OptionParser => $o,
Quoter => $q,
);
# Set table's initial chunk size. If this is the first table, # Set table's initial chunk size. If this is the first table,
# then total rate will be zero, so use --chunk-size. Or, if # then total rate will be zero, so use --chunk-size. Or, if
# --chunk-time=0, then only use --chunk-size for every table. # --chunk-time=0, then only use --chunk-size for every table.
@@ -6737,9 +6746,6 @@ sub check_repl_table {
# #
# Returns: # Returns:
# Nothing or dies on error # Nothing or dies on error
{
my $current_db;
sub use_repl_db { sub use_repl_db {
my ( %args ) = @_; my ( %args ) = @_;
my @required_args = qw(dbh repl_table OptionParser Quoter); my @required_args = qw(dbh repl_table OptionParser Quoter);
@@ -6747,6 +6753,7 @@ sub use_repl_db {
die "I need a $arg argument" unless $args{$arg}; die "I need a $arg argument" unless $args{$arg};
} }
my ($dbh, $repl_table, $o, $q) = @args{@required_args}; my ($dbh, $repl_table, $o, $q) = @args{@required_args};
MKDEBUG && _d('use_repl_db');
my ($db, $tbl) = $q->split_unquote($repl_table); my ($db, $tbl) = $q->split_unquote($repl_table);
if ( my $tbl = $args{tbl} ) { if ( my $tbl = $args{tbl} ) {
@@ -6765,30 +6772,26 @@ sub use_repl_db {
$db = $o->get('replicate-database') if $o->get('replicate-database'); $db = $o->get('replicate-database') if $o->get('replicate-database');
} }
if ( !$current_db || $current_db ne $db ) { eval {
eval { my $sql = "USE " . $q->quote($db);
my $sql = "USE " . $q->quote($db); MKDEBUG && _d($sql);
MKDEBUG && _d($sql); $dbh->do($sql);
$dbh->do($sql); };
}; if ( $EVAL_ERROR ) {
if ( $EVAL_ERROR ) { # Report which option db really came from.
# Report which option db really came from. my $opt = $o->get('replicate-database') ? "--replicate-database"
my $opt = $o->get('replicate-database') ? "--replicate-database" : "--replicate database";
: "--replicate database"; if ( $EVAL_ERROR =~ m/unknown database/i ) {
if ( $EVAL_ERROR =~ m/unknown database/i ) { die "$opt $db does not exist. You need to create the "
die "$opt $db does not exist. You need to create the " . "database or specify a database for $opt that exists.\n";
. "database or specify a database for $opt that exists.\n"; }
} else {
else { die "Error using $opt $db: $EVAL_ERROR\n";
die "Error using $opt $db: $EVAL_ERROR\n";
}
} }
$current_db = $db;
} }
return; return;
} }
}
sub create_repl_table { sub create_repl_table {
my ( %args ) = @_; my ( %args ) = @_;

217
t/pt-table-checksum/issue_982.t
View File

@@ -1,217 +0,0 @@
#!/usr/bin/env perl
BEGIN {
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
};
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Test::More skip_all => 'Finish updating issue_982.t';
use PerconaTest;
use Sandbox;
require "$trunk/bin/pt-table-checksum";
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $master_dbh = $sb->get_dbh_for('master');
my $slave1_dbh = $sb->get_dbh_for('slave1');
if ( !$master_dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
if ( !$slave1_dbh ) {
plan skip_all => 'Cannot connect to sandbox slave';
}
else {
plan tests => 8;
}
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
# so we need to specify --lock-wait-timeout=3 else the tool will die.
# And --max-load "" prevents waiting for status variables.
my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox';
my @args = ($master_dsn, qw(--lock-wait-timeout 3), '--max-load', '');
my $row;
my $output;
# #############################################################################
# Issue 982: --empty-replicate-table does not work with binlog-ignore-db
# #############################################################################
$sb->wipe_clean($master_dbh);
pt_table_checksum::main(@args, qw(-t sakila.country --quiet));
$master_dbh->do("insert into percona.checksums values ('sakila', 'country', 99, null, null, null, null, '', 0, '', 0, null)");
PerconaTest::wait_for_table($slave1_dbh, 'percona.checksums', "db='sakila' and tbl='country' and chunk=99");
$master_dbh->disconnect();
$slave1_dbh->disconnect();
# Add a replication filter to the slave.
diag(`/tmp/12346/stop >/dev/null`);
diag(`/tmp/12345/stop >/dev/null`);
diag(`cp /tmp/12345/my.sandbox.cnf /tmp/12345/orig.cnf`);
diag(`echo "binlog-ignore-db=sakila" >> /tmp/12345/my.sandbox.cnf`);
diag(`echo "binlog-ignore-db=mysql" >> /tmp/12345/my.sandbox.cnf`);
diag(`/tmp/12345/start >/dev/null`);
diag(`/tmp/12346/start >/dev/null`);
$output = output(
sub { pt_table_checksum::main(@args, qw(--no-check-replication-filters),
qw(-d mysql -t user)) },
stderr => 1,
);
$master_dbh = $sb->get_dbh_for('master');
$slave1_dbh = $sb->get_dbh_for('slave1');
$row = $slave1_dbh->selectall_arrayref("select * from percona.checksums where db='sakila' and tbl='country' and chunk=99");
ok(
@$row == 0,
"Slave checksum table deleted"
);
# Clear checksum table for next tests.
$master_dbh->do("truncate table percona.checksums");
wait_until(
sub {
$row = $slave1_dbh->selectall_arrayref("select * from percona.checksums");
return !@$row;
}
);
$master_dbh->disconnect();
$slave1_dbh->disconnect();
# Restore original config.
diag(`/tmp/12346/stop >/dev/null`);
diag(`/tmp/12345/stop >/dev/null`);
diag(`cp /tmp/12345/orig.cnf /tmp/12345/my.sandbox.cnf`);
# #############################################################################
# Test --replicate-database which resulted from this issue.
# #############################################################################
# Add a binlog-do-db filter so master will only replicate
# statements when USE mysql is in effect.
diag(`echo "binlog-do-db=mysql" >> /tmp/12345/my.sandbox.cnf`);
diag(`/tmp/12345/start >/dev/null`);
diag(`/tmp/12346/start >/dev/null`);
$master_dbh = $sb->get_dbh_for('master');
$slave1_dbh = $sb->get_dbh_for('slave1');
$output = output(
sub { pt_table_checksum::main(@args, qw(--no-check-replication-filters),
qw(--replicate=percona.checksums -d mysql -t user))
},
stderr => 1,
);
# Because we did not use --replicate-database, mk-table-checksum should
# have done USE mysql before updating the checksum table. Thus, the
# checksums should show up on the slave.
sleep 1;
$row = $slave1_dbh->selectall_arrayref("select * from percona.checksums where db='mysql' AND tbl='user'");
ok(
@$row == 1,
"Checksum replicated with binlog-do-db, without --replicate-database"
);
# Now force --replicate-database test and the checksums should not replicate.
$master_dbh->do("use mysql");
$master_dbh->do("truncate table percona.checksums");
sleep 1;
$row = $slave1_dbh->selectall_arrayref("select * from percona.checksums");
ok(
!@$row,
"Checksum table empty on slave"
);
$output = output(
sub { pt_table_checksum::main(@args, qw(--no-check-replication-filters),
qw(--replicate=percona.checksums -d mysql -t user),
qw(--replicate-database test))
},
stderr => 1,
);
sleep 1;
$row = $slave1_dbh->selectall_arrayref("select * from percona.checksums where db='mysql' AND tbl='user'");
ok(
!@$row,
"Checksum did not replicated with binlog-do-db, with --replicate-database"
);
# #############################################################################
# Restore original config.
# #############################################################################
$master_dbh->disconnect();
$slave1_dbh->disconnect();
diag(`/tmp/12346/stop >/dev/null`);
diag(`/tmp/12345/stop >/dev/null`);
diag(`mv /tmp/12345/orig.cnf /tmp/12345/my.sandbox.cnf`);
diag(`/tmp/12345/start >/dev/null`);
diag(`/tmp/12346/start >/dev/null`);
$master_dbh = $sb->get_dbh_for('master');
$slave1_dbh = $sb->get_dbh_for('slave1');
# #############################################################################
# Test it again by looking at binlog to see that the db didn't change.
# #############################################################################
diag(`$trunk/sandbox/test-env reset`);
sleep 1;
# To speed this test up, ignore these tables.
# http://code.google.com/p/maatkit/issues/detail?id=1027
my $it = "payment,rental,help_topic,help_keyword,inventory,film_actor";
$output = output(
sub { pt_table_checksum::main(@args,
qw(--replicate=percona.checksums), '--ignore-tables', $it, qw(--chunk-size 20k))
},
stderr => 1,
);
sleep 1;
$row = $master_dbh->selectrow_hashref('show master status');
$output = `$ENV{PERCONA_TOOLKIT_SANDBOX}/bin/mysqlbinlog /tmp/12345/data/$row->{file} | grep 'use ' | grep -v '^# Warning' | sort -u`;
is(
$output,
"use mysql/*!*/;
use sakila/*!*/;
use test/*!*/;
",
"USE each table's db (binlog dump)"
);
diag(`$trunk/sandbox/test-env reset`);
sleep 1;
$output = output(
sub { pt_table_checksum::main(@args, qw(--replicate-database test),
qw(--replicate=percona.checksums), '--ignore-tables', $it, qw(--chunk-size 20k))
},
stderr => 1,
);
sleep 1;
$row = $master_dbh->selectrow_hashref('show master status');
$output = `$ENV{PERCONA_TOOLKIT_SANDBOX}/bin/mysqlbinlog /tmp/12345/data/$row->{file} | grep 'use ' | grep -v '^# Warning'`;
is(
$output,
"use test/*!*/;
",
"USE only --replicate-database db (binlog dump)"
);
# #############################################################################
# Done.
# #############################################################################
$sb->wipe_clean($master_dbh);
exit;

171
t/pt-table-checksum/replication_filters.t
View File

@@ -36,7 +36,7 @@ elsif ( !$slave2_dbh ) {
plan skip_all => 'Cannot connect to sandbox slave2'; plan skip_all => 'Cannot connect to sandbox slave2';
} }
else { else {
plan tests => 4; plan tests => 10;
} }
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic # The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
@@ -93,14 +93,177 @@ like(
"--no-check-replication-filters" "--no-check-replication-filters"
); );
# #############################################################################
# Done.
# #############################################################################
# Remove the replication filter from the slave. # Remove the replication filter from the slave.
for my $port ( qw(12346 12347) ) { for my $port ( qw(12346 12347) ) {
diag(`/tmp/$port/stop >/dev/null`); diag(`/tmp/$port/stop >/dev/null`);
diag(`mv /tmp/$port/orig.cnf /tmp/$port/my.sandbox.cnf`); diag(`mv /tmp/$port/orig.cnf /tmp/$port/my.sandbox.cnf`);
diag(`/tmp/$port/start >/dev/null`); diag(`/tmp/$port/start >/dev/null`);
} }
# #############################################################################
# Issue 982: --empty-replicate-table does not work with binlog-ignore-db
# #############################################################################
# Write some results to master and slave for dbs mysql and sakila.
$sb->wipe_clean($master_dbh);
pt_table_checksum::main(@args, qw(--chunk-time 0 --chunk-size 100),
'-t', 'mysql.user,sakila.city', qw(--quiet));
PerconaTest::wait_for_table($slave1_dbh, 'percona.checksums', "db='sakila' and tbl='city' and chunk=6");
$master_dbh->disconnect();
$slave1_dbh->disconnect();
# Add a replication filter to the slave: ignore db mysql.
diag(`/tmp/12346/stop >/dev/null`);
diag(`/tmp/12345/stop >/dev/null`);
diag(`cp /tmp/12345/my.sandbox.cnf /tmp/12345/orig.cnf`);
diag(`echo "binlog-ignore-db=mysql" >> /tmp/12345/my.sandbox.cnf`);
diag(`/tmp/12345/start >/dev/null`);
diag(`/tmp/12346/start >/dev/null`);
$master_dbh = $sb->get_dbh_for('master');
$slave1_dbh = $sb->get_dbh_for('slave1');
# Checksum the tables again in 1 chunk. Since db percona isn't being
# ignored, deleting old results in the repl table should replicate.
# But since db mysql is ignored, the new results for mysql.user should
# not replicate.
pt_table_checksum::main(@args, qw(--no-check-replication-filters),
'-t', 'mysql.user,sakila.city', qw(--quiet --no-replicate-check));
PerconaTest::wait_for_table($slave1_dbh, 'percona.checksums', "db='sakila' and tbl='city' and chunk=1");
$row = $slave1_dbh->selectall_arrayref("select db,tbl,chunk from percona.checksums order by db,tbl,chunk");
is_deeply(
$row,
[[qw(sakila city 1)]],
"binlog-ignore-db"
) or print STDERR Dumper($row);
$master_dbh->do("use percona");
$master_dbh->do("truncate table percona.checksums");
wait_until(
sub {
$row = $slave1_dbh->selectall_arrayref("select * from percona.checksums");
return !@$row;
}
);
$master_dbh->disconnect();
$slave1_dbh->disconnect();
# Restore original config.
diag(`/tmp/12346/stop >/dev/null`);
diag(`/tmp/12345/stop >/dev/null`);
diag(`cp /tmp/12345/orig.cnf /tmp/12345/my.sandbox.cnf`);
# #############################################################################
# Test --replicate-database which resulted from this issue.
# #############################################################################
# Add a binlog-do-db filter so master will only replicate
# statements when USE mysql is in effect.
diag(`echo "binlog-do-db=mysql" >> /tmp/12345/my.sandbox.cnf`);
diag(`/tmp/12345/start >/dev/null`);
diag(`/tmp/12346/start >/dev/null`);
$master_dbh = $sb->get_dbh_for('master');
$slave1_dbh = $sb->get_dbh_for('slave1');
$output = output(
sub { pt_table_checksum::main(@args, qw(--no-check-replication-filters),
qw(-d mysql -t user))
},
stderr => 1,
);
# Because we did not use --replicate-database, pt-table-checksum should
# have done USE mysql before updating the repl table. Thus, the
# checksums should show up on the slave.
PerconaTest::wait_for_table($slave1_dbh, 'percona.checksums', "db='mysql' and tbl='user' and chunk=1");
$row = $slave1_dbh->selectall_arrayref("select db,tbl,chunk from percona.checksums order by db,tbl,chunk");
is_deeply(
$row,
[[qw(mysql user 1)]],
"binlog-do-do, without --replicate-database"
) or print STDERR Dumper($row);
# Now force --replicate-database test and the checksums should not replicate.
$master_dbh->do("use mysql");
$master_dbh->do("truncate table percona.checksums");
wait_until(
sub {
$row = $slave1_dbh->selectall_arrayref("select * from percona.checksums");
return !@$row;
}
);
$pos = PerconaTest::get_master_binlog_pos($master_dbh);
pt_table_checksum::main(@args, qw(--quiet --no-check-replication-filters),
qw(-t mysql.user --replicate-database sakila --no-replicate-check));
sleep 1;
$row = $slave1_dbh->selectall_arrayref("select * from percona.checksums where db='mysql' AND tbl='user'");
ok(
!@$row,
"binlog-do-db, with --replicate-database"
) or print STDERR Dumper($row);
is(
PerconaTest::get_master_binlog_pos($master_dbh),
$pos,
"Master pos did not change"
);
# #############################################################################
# Restore original config.
# #############################################################################
$master_dbh->disconnect();
$slave1_dbh->disconnect();
diag(`/tmp/12346/stop >/dev/null`);
diag(`/tmp/12345/stop >/dev/null`);
diag(`mv /tmp/12345/orig.cnf /tmp/12345/my.sandbox.cnf`);
diag(`/tmp/12345/start >/dev/null`);
diag(`/tmp/12346/start >/dev/null`);
diag(`$trunk/sandbox/test-env reset`);
$master_dbh = $sb->get_dbh_for('master');
$slave1_dbh = $sb->get_dbh_for('slave1');
pt_table_checksum::main(@args, qw(--quiet));
$row = $master_dbh->selectrow_hashref('show master status');
$output = `$ENV{PERCONA_TOOLKIT_SANDBOX}/bin/mysqlbinlog /tmp/12345/data/$row->{file} | grep 'use ' | grep -v '^# Warning' | sort -u`;
is(
$output,
"use mysql/*!*/;
use percona/*!*/;
use sakila/*!*/;
",
"USE each table's database (binlog dump)"
);
diag(`$trunk/sandbox/test-env reset`);
pt_table_checksum::main(@args, qw(--quiet --replicate-database percona));
$output = `$ENV{PERCONA_TOOLKIT_SANDBOX}/bin/mysqlbinlog /tmp/12345/data/$row->{file} | grep 'use ' | grep -v '^# Warning'`;
is(
$output,
"use percona/*!*/;
",
"USE only --replicate-database (binlog dump)"
);
# #############################################################################
# Done.
# #############################################################################
$sb->wipe_clean($master_dbh); $sb->wipe_clean($master_dbh);
exit; exit;