percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-12-20 02:20:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove pt-table-sync/issue_79.t (it was broken but coincidentally working) and write better tests in filters.t. Add subs to Sandbox and PerconaTest to clear and parse tables used from the general logs. Set log=genlog in the sandbox configs.

Browse Source
This commit is contained in:
Daniel Nichter
2012-06-07 12:38:45 -04:00
parent 8d4354090e
commit 79c716d358
9 changed files with 203 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
lib/PerconaTest.pm
View File

@@ -753,6 +753,27 @@ sub full_output {
return ($output, $status); return ($output, $status);
} }
sub tables_used {
my ($file) = @_;
local $INPUT_RECORD_SEPARATOR = '';
open my $fh, '<', $file or die "Cannot open $file: $OS_ERROR";
my %tables;
while ( defined(my $chunk = <$fh>) ) {
map {
my $db_tbl = $_;
$db_tbl =~ s/^\s*`?//; # strip leading space and `
$db_tbl =~ s/\s*`?$//; # strip trailing space and `
$db_tbl =~ s/`\.`/./; # strip inner `.`
$tables{$db_tbl} = 1;
}
grep {
m/(?:\w\.\w|`\.`)/ # only db.tbl, not just db
}
$chunk =~ m/(?:FROM|INTO|UPDATE)\s+(\S+)/gi;
}
return [ sort keys %tables ];
}
1; 1;
} }
# ########################################################################### # ###########################################################################

22
lib/Sandbox.pm
View File

@@ -374,6 +374,28 @@ sub verify_test_data {
return; return;
} }
sub dsn_for {
my ($self, $host) = @_;
_check_server($host);
return "h=127.1,P=$port_for{$host},u=msandbox,p=msandbox";
}
sub genlog {
my ($self, $host) = @_;
_check_server($host);
return "/tmp/$port_for{$host}/data/genlog";
}
sub clear_genlogs {
my ($self, @hosts) = @_;
@hosts = qw(master slave1 slave2) unless scalar @hosts;
foreach my $host ( @hosts ) {
PTDEVDEBUG && _d('Clearing general log on', $host);
Test::More::diag(`echo > /tmp/$port_for{$host}/data/genlog`);
}
return;
}
1; 1;
} }
# ########################################################################### # ###########################################################################

2
sandbox/servers/5.0/my.sandbox.cnf
View File

@@ -24,4 +24,4 @@ report-host = 127.0.0.1
report-port = PORT report-port = PORT
log-error = mysqld.log log-error = mysqld.log
innodb_lock_wait_timeout = 3 innodb_lock_wait_timeout = 3
log log = genlog

2
sandbox/servers/5.1/my.sandbox.cnf
View File

@@ -24,4 +24,4 @@ report-host = 127.0.0.1
report-port = PORT report-port = PORT
log-error = mysqld.log log-error = mysqld.log
innodb_lock_wait_timeout = 3 innodb_lock_wait_timeout = 3
log log = genlog

2
sandbox/servers/5.5/my.sandbox.cnf
View File

@@ -24,4 +24,4 @@ report-host = 127.0.0.1
report-port = PORT report-port = PORT
log-error = mysqld.log log-error = mysqld.log
innodb_lock_wait_timeout = 3 innodb_lock_wait_timeout = 3
general_log general_log = genlog

43
t/lib/samples/SchemaIterator.sql
View File

@@ -1,24 +1,43 @@
--
-- d1
--
DROP DATABASE IF EXISTS d1; DROP DATABASE IF EXISTS d1;
DROP DATABASE IF EXISTS d2;
DROP DATABASE IF EXISTS d3;
CREATE DATABASE d1; CREATE DATABASE d1;
CREATE DATABASE d2;
CREATE DATABASE d3;
CREATE TABLE d1.t1 ( CREATE TABLE d1.t1 (
i int id int auto_increment primary key,
c char(8)
) ENGINE=MyISAM CHARSET=utf8; ) ENGINE=MyISAM CHARSET=utf8;
INSERT INTO d1.t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9);
CREATE TABLE d1.t2 ( CREATE TABLE d1.t2 (
c varchar(255) id int auto_increment primary key,
c varchar(8)
) ENGINE=InnoDB; ) ENGINE=InnoDB;
CREATE TABLE d1.t3 ( CREATE TABLE d1.t3 (
i int id int auto_increment primary key,
c char(8)
) ENGINE=MEMORY; ) ENGINE=MEMORY;
CREATE TABLE d2.t1 ( INSERT INTO d1.t1 VALUES (null, 'a'), (null, 'b'), (null, 'c'), (null, 'd'), (null, 'e'), (null, 'f'), (null, 'g'), (null, 'h'), (null, 'i');
i int INSERT INTO d1.t2 SELECT * FROM d1.t1;
); INSERT INTO d1.t3 SELECT * FROM d1.t1;
--
-- d2
--
DROP DATABASE IF EXISTS d2;
CREATE DATABASE d2;
CREATE TABLE d2.t1 (
id int auto_increment primary key,
c char(8)
);
-- d2.t1 is an empty table.
--
-- d3
--
DROP DATABASE IF EXISTS d3;
CREATE DATABASE d3;
-- d3 is an empty database. -- d3 is an empty database.

188
t/pt-table-sync/filters.t
View File

@@ -10,12 +10,12 @@ use strict;
use warnings FATAL => 'all'; use warnings FATAL => 'all';
use English qw(-no_match_vars); use English qw(-no_match_vars);
use Test::More; use Test::More;
use Data::Dumper;
use PerconaTest; use PerconaTest;
use Sandbox; use Sandbox;
require "$trunk/bin/pt-table-sync"; require "$trunk/bin/pt-table-sync";
my $vp = new VersionParser(); my $vp = new VersionParser();
my $dp = new DSNParser(opts=>$dsn_opts); my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
@@ -29,81 +29,145 @@ elsif ( !$slave_dbh ) {
plan skip_all => 'Cannot connect to sandbox slave'; plan skip_all => 'Cannot connect to sandbox slave';
} }
else { else {
plan tests => 9; plan tests => 18;
} }
# Previous tests slave 12347 to 12346 which makes pt-table-checksum my $master_dsn = $sb->dsn_for('master');
# complain that it cannot connect to 12347 for checking repl filters my $slave1_dsn = $sb->dsn_for('slave1');
# and such. 12347 isn't present but SHOW SLAVE HOSTS on 12346 hasn't
# figured that out yet, so we restart 12346 to refresh this list.
#diag(`/tmp/12346/stop >/dev/null`);
#diag(`/tmp/12346/start >/dev/null`);
$slave_dbh = $sb->get_dbh_for('slave1');
my $output; my $output;
my $cnf = "/tmp/12345/my.sandbox.cnf";
my $cmd = "$trunk/bin/pt-table-sync -F $cnf";
my $t = qr/\d\d:\d\d:\d\d/;
$sb->wipe_clean($master_dbh); # See this SQL file because it has a number of simple dbs and tbls
$sb->load_file('master', 't/pt-table-sync/samples/filter_tables.sql'); # that are used to checking schema object filters.
$sb->load_file('master', "t/lib/samples/SchemaIterator.sql");
$output = `$cmd h=127.1,P=12345 P=12346 --no-check-slave --dry-run -t issue_806_1.t2 | tail -n 2`; sub test_filters {
$output =~ s/$t/00:00:00/g; my (%args) = @_;
$output =~ s/[ ]{2,}/ /g;
is( $sb->clear_genlogs();
$output,
"# DELETE REPLACE INSERT UPDATE ALGORITHM START END EXIT DATABASE.TABLE my $output = output(
# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_1.t2 sub { pt_table_sync::main(@{$args{cmds}},
", qw(--print))
"db-qualified --tables (issue 806)" },
);
my $tables_used = PerconaTest::tables_used($sb->genlog('master'));
is_deeply(
$tables_used,
$args{res},
"$args{name}: tables used"
) or diag(Dumper($tables_used));
}
# #############################################################################
# Basic schema object filters: --databases, --tables, etc.
# #############################################################################
# Not really a filter, but only the specified tables should be used.
test_filters(
name => "Sync d1.t1 to d1.t2",
cmds => ["$master_dsn,D=d1,t=t1", "t=t2"],
res => [qw(d1.t1 d1.t2)],
);
# Use slave1 like it's another master, ok becuase we're not actually
# syncing anything, so --no-check-slave is required else pt-table-sync
# won't run (because it doesn't like to sync directly to a slave).
test_filters(
name => "-t d1.t1",
cmds => [$master_dsn, $slave1_dsn, qw(--no-check-slave),
qw(-t d1.t1)],
res => [qw(d1.t1)],
);
# Like the previous test, but now any table called "t1".
test_filters(
name => "-t t1",
cmds => [$master_dsn, $slave1_dsn, qw(--no-check-slave),
qw(-t t1)],
res => [qw(d1.t1 d2.t1)],
);
# Only the given db, all its tables: there's only 1 tbl in d2.
test_filters(
name => "--databases d2",
cmds => [$master_dsn, $slave1_dsn, qw(--no-check-slave),
qw(--databases d2)],
res => [qw(d2.t1)],
);
# --database with --tables.
test_filters(
name => "--databases d1 --tables t2,t3",
cmds => [$master_dsn, $slave1_dsn, qw(--no-check-slave),
qw(--databases d1), "--tables", "t2,t3"],
res => [qw(d1.t2 d1.t3)],
); );
# ############################################################################# # #############################################################################
# Issue 820: Make mk-table-sync honor schema filters with --replicate # Filters with --replicate and --sync-to-master.
# ############################################################################# # #############################################################################
$master_dbh->do('DROP DATABASE IF EXISTS test');
$master_dbh->do('CREATE DATABASE test'); # Checksum the filter tables.
$master_dbh->do("DROP DATABASE IF EXISTS percona");
$sb->wait_for_slaves();
diag(`$trunk/bin/pt-table-checksum $master_dsn -d d1,d2,d3 --chunk-size 100 --quiet --lock-wait-timeout 3 --max-load ''`);
$slave_dbh->do('insert into issue_806_1.t1 values (41)'); my $rows = $master_dbh->selectall_arrayref("SELECT CONCAT(db, '.', tbl) FROM percona.checksums ORDER BY db, tbl");
$slave_dbh->do('insert into issue_806_2.t2 values (42)'); is_deeply(
$rows,
[
["d1.t1"],
["d1.t2"],
["d1.t3"],
["d2.t1"],
],
"Checksummed all tables"
) or diag(Dumper($rows));
my $mk_table_checksum = "$trunk/bin/pt-table-checksum --lock-wait-time 3"; # Make all checksums on the slave different than the master
# so that pt-table-sync would sync all the tables if there
# were no filters.
$slave_dbh->do("UPDATE percona.checksums SET this_cnt=999 WHERE 1=1");
`$mk_table_checksum --replicate test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d issue_806_1,issue_806_2 --quiet`; # Verify that that ^ is true.
test_filters(
$output = `$cmd h=127.1,P=12345 --replicate test.checksum --dry-run | tail -n 2`; name => "All tables are different on the slave",
$output =~ s/$t/00:00:00/g; cmds => [$master_dsn, qw(--replicate percona.checksums)],
$output =~ s/[ ]{2,}/ /g; res => [qw(d1.t1 d1.t2 d1.t3 d2.t1 percona.checksums)],
is(
$output,
"# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_1.t1
# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_2.t2
",
"--replicate with no filters"
); );
$output = `$cmd h=127.1,P=12345 --replicate test.checksum --dry-run -t t1 | tail -n 2`; # Sync with --replicate, --sync-to-master, and some filters.
$output =~ s/$t/00:00:00/g; # --replicate and --sync-to-master have different code paths,
$output =~ s/[ ]{2,}/ /g; # but the filter results should be the same.
is( foreach my $args (
$output, [$master_dsn, qw(--replicate percona.checksums)],
"# DELETE REPLACE INSERT UPDATE ALGORITHM START END EXIT DATABASE.TABLE [$slave1_dsn, qw(--replicate percona.checksums --sync-to-master)]
# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_1.t1 ) {
",
"--replicate with --tables"
);
$output = `$cmd h=127.1,P=12345 --replicate test.checksum --dry-run -d issue_806_2 | tail -n 2`; my $stm = $args->[-1] eq '--sync-to-master' ? ' --sync-to-master' : '';
$output =~ s/$t/00:00:00/g;
$output =~ s/[ ]{2,}/ /g; test_filters(
is( name => $stm . "--replicate --tables t1",
$output, cmds => [@$args,
"# DELETE REPLACE INSERT UPDATE ALGORITHM START END EXIT DATABASE.TABLE qw(--tables t1)],
# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_2.t2 res => [qw(d1.t1 d2.t1 percona.checksums)],
", );
"--replicate with --databases"
); test_filters(
name => $stm . "--replicate --databases d2",
cmds => [@$args,
qw(--databases d2)],
res => [qw(d2.t1 percona.checksums)],
);
test_filters(
name => $stm . "--replicate --databases d1 --tables t1,t3",
cmds => [@$args,
qw(--databases d1), "--tables", "t1,t3"],
res => [qw(d1.t1 d1.t3 percona.checksums)],
);
}
# ############################################################################# # #############################################################################
# pt-table-sync --ignore-* options don't work with --replicate # pt-table-sync --ignore-* options don't work with --replicate
@@ -120,7 +184,7 @@ $slave_dbh->do("INSERT INTO test.empty_it VALUES (null,11,11,'eleven')");
diag(`$trunk/bin/pt-table-checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test --quiet --quiet --lock-wait-timeout 3 --max-load ''`); diag(`$trunk/bin/pt-table-checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test --quiet --quiet --lock-wait-timeout 3 --max-load ''`);
# Make sure all the tables were checksummed. # Make sure all the tables were checksummed.
my $rows = $master_dbh->selectall_arrayref("SELECT DISTINCT db, tbl FROM percona.checksums ORDER BY db, tbl"); $rows = $master_dbh->selectall_arrayref("SELECT DISTINCT db, tbl FROM percona.checksums ORDER BY db, tbl");
is_deeply( is_deeply(
$rows, $rows,
[ [qw(test empty_it) ], [ [qw(test empty_it) ],

85
t/pt-table-sync/issue_79.t
View File

@@ -1,85 +0,0 @@
#!/usr/bin/env perl
BEGIN {
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
};
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Test::More;
use PerconaTest;
use Sandbox;
require "$trunk/bin/pt-table-sync";
my $output;
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $master_dbh = $sb->get_dbh_for('master');
my $slave_dbh = $sb->get_dbh_for('slave1');
if ( !$master_dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
elsif ( !$slave_dbh ) {
plan skip_all => 'Cannot connect to sandbox slave';
}
else {
plan tests => 7;
}
$sb->wipe_clean($master_dbh);
$sb->create_dbs($master_dbh, [qw(test)]);
$sb->load_file('master', 't/pt-table-sync/samples/before.sql');
PerconaTest::wait_for_table($slave_dbh, "test.test4", "id=15034");
# #############################################################################
# Issue 79: mk-table-sync with --replicate doesn't honor --tables
# #############################################################################
$master_dbh->do('create table test.messages (i int)');
PerconaTest::wait_for_table($slave_dbh, "test.messages", "i=3");
$slave_dbh->do('insert into test.messages values (1), (2), (3)');
# The previous test should have left test.messages on the slave (12346)
# out of sync. Now we also unsync test2 on the slave and then re-sync only
# it. If --tables is honored, only test2 on the slave will be synced.
$sb->use('master', "-D test -e \"SET SQL_LOG_BIN=0; INSERT INTO test2 VALUES (1,'a'),(2,'b')\"");
diag(`$trunk/bin/pt-table-checksum --replicate=test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test --lock-wait-time 3 > /dev/null`);
is_deeply(
$master_dbh->selectall_arrayref('select * from test.messages'),
[],
'test.messages on master empty'
);
is_deeply(
$slave_dbh->selectall_arrayref('select * from test.messages'),
[[1],[2],[3]],
'test.messages on slave has values'
);
# Test that what the doc says about --tables is true:
# "Table names may be qualified with the database name."
# In the code, a qualified db.tbl name is used.
# So we'll test first an unqualified tbl name.
$output = `$trunk/bin/pt-table-sync h=127.1,P=12345,u=msandbox,p=msandbox --replicate test.checksum --execute -d test -t test2 -v`;
unlike($output, qr/messages/, '--replicate honors --tables (1/4)');
like($output, qr/test2/, '--replicate honors --tables (2/4)');
# Now we'll test with a qualified db.tbl name.
$sb->use('slave1', '-D test -e "TRUNCATE TABLE test2; TRUNCATE TABLE messages"');
diag(`$trunk/bin/pt-table-checksum --replicate=test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test --lock-wait-time 3 > /dev/null`);
$output = `$trunk/bin/pt-table-sync h=127.1,P=12345,u=msandbox,p=msandbox --replicate test.checksum --execute -d test -t test.test2 -v`;
unlike($output, qr/messages/, '--replicate honors --tables (3/4)');
like($output, qr/test2/, '--replicate honors --tables (4/4)');
# #############################################################################
# Done.
# #############################################################################
$sb->wipe_clean($master_dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
exit;

13
t/pt-table-sync/samples/filter_tables.sql
View File

@@ -1,13 +0,0 @@
drop database if exists issue_806_1;
drop database if exists issue_806_2;
create database issue_806_1;
create database issue_806_2;
use issue_806_1;
create table t1 (i int, unique index (i));
create table t2 (i int, unique index (i));
use issue_806_2;
create table t1 (i int, unique index (i));
create table t2 (i int, unique index (i));
create table t3 (i int, unique index (i));