mirror of
https://github.com/percona/percona-toolkit.git
synced 2025-09-10 21:19:59 +00:00
Remove pt-table-sync/issue_79.t (it was broken but coincidentally working) and write better tests in filters.t. Add subs to Sandbox and PerconaTest to clear and parse tables used from the general logs. Set log=genlog in the sandbox configs.
This commit is contained in:
Daniel Nichter
@@ -753,6 +753,27 @@ sub full_output {
|
||||
return ($output, $status);
|
||||
}
|
||||
|
||||
sub tables_used {
|
||||
my ($file) = @_;
|
||||
local $INPUT_RECORD_SEPARATOR = '';
|
||||
open my $fh, '<', $file or die "Cannot open $file: $OS_ERROR";
|
||||
my %tables;
|
||||
while ( defined(my $chunk = <$fh>) ) {
|
||||
map {
|
||||
my $db_tbl = $_;
|
||||
$db_tbl =~ s/^\s*`?//; # strip leading space and `
|
||||
$db_tbl =~ s/\s*`?$//; # strip trailing space and `
|
||||
$db_tbl =~ s/`\.`/./; # strip inner `.`
|
||||
$tables{$db_tbl} = 1;
|
||||
}
|
||||
grep {
|
||||
m/(?:\w\.\w|`\.`)/ # only db.tbl, not just db
|
||||
}
|
||||
$chunk =~ m/(?:FROM|INTO|UPDATE)\s+(\S+)/gi;
|
||||
}
|
||||
return [ sort keys %tables ];
|
||||
}
|
||||
|
||||
1;
|
||||
}
|
||||
# ###########################################################################
|
||||
|
||||
@@ -374,6 +374,28 @@ sub verify_test_data {
|
||||
return;
|
||||
}
|
||||
|
||||
sub dsn_for {
|
||||
my ($self, $host) = @_;
|
||||
_check_server($host);
|
||||
return "h=127.1,P=$port_for{$host},u=msandbox,p=msandbox";
|
||||
}
|
||||
|
||||
sub genlog {
|
||||
my ($self, $host) = @_;
|
||||
_check_server($host);
|
||||
return "/tmp/$port_for{$host}/data/genlog";
|
||||
}
|
||||
|
||||
sub clear_genlogs {
|
||||
my ($self, @hosts) = @_;
|
||||
@hosts = qw(master slave1 slave2) unless scalar @hosts;
|
||||
foreach my $host ( @hosts ) {
|
||||
PTDEVDEBUG && _d('Clearing general log on', $host);
|
||||
Test::More::diag(`echo > /tmp/$port_for{$host}/data/genlog`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
1;
|
||||
}
|
||||
# ###########################################################################
|
||||
|
||||
@@ -24,4 +24,4 @@ report-host = 127.0.0.1
|
||||
report-port = PORT
|
||||
log-error = mysqld.log
|
||||
innodb_lock_wait_timeout = 3
|
||||
log
|
||||
log = genlog
|
||||
|
||||
@@ -24,4 +24,4 @@ report-host = 127.0.0.1
|
||||
report-port = PORT
|
||||
log-error = mysqld.log
|
||||
innodb_lock_wait_timeout = 3
|
||||
log
|
||||
log = genlog
|
||||
|
||||
@@ -24,4 +24,4 @@ report-host = 127.0.0.1
|
||||
report-port = PORT
|
||||
log-error = mysqld.log
|
||||
innodb_lock_wait_timeout = 3
|
||||
general_log
|
||||
general_log = genlog
|
||||
|
||||
@@ -1,24 +1,43 @@
|
||||
--
|
||||
-- d1
|
||||
--
|
||||
DROP DATABASE IF EXISTS d1;
|
||||
DROP DATABASE IF EXISTS d2;
|
||||
DROP DATABASE IF EXISTS d3;
|
||||
|
||||
CREATE DATABASE d1;
|
||||
CREATE DATABASE d2;
|
||||
CREATE DATABASE d3;
|
||||
|
||||
CREATE TABLE d1.t1 (
|
||||
i int
|
||||
id int auto_increment primary key,
|
||||
c char(8)
|
||||
) ENGINE=MyISAM CHARSET=utf8;
|
||||
INSERT INTO d1.t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9);
|
||||
|
||||
CREATE TABLE d1.t2 (
|
||||
c varchar(255)
|
||||
id int auto_increment primary key,
|
||||
c varchar(8)
|
||||
) ENGINE=InnoDB;
|
||||
|
||||
CREATE TABLE d1.t3 (
|
||||
i int
|
||||
id int auto_increment primary key,
|
||||
c char(8)
|
||||
) ENGINE=MEMORY;
|
||||
|
||||
CREATE TABLE d2.t1 (
|
||||
i int
|
||||
);
|
||||
INSERT INTO d1.t1 VALUES (null, 'a'), (null, 'b'), (null, 'c'), (null, 'd'), (null, 'e'), (null, 'f'), (null, 'g'), (null, 'h'), (null, 'i');
|
||||
INSERT INTO d1.t2 SELECT * FROM d1.t1;
|
||||
INSERT INTO d1.t3 SELECT * FROM d1.t1;
|
||||
|
||||
--
|
||||
-- d2
|
||||
--
|
||||
DROP DATABASE IF EXISTS d2;
|
||||
CREATE DATABASE d2;
|
||||
|
||||
CREATE TABLE d2.t1 (
|
||||
id int auto_increment primary key,
|
||||
c char(8)
|
||||
);
|
||||
-- d2.t1 is an empty table.
|
||||
|
||||
--
|
||||
-- d3
|
||||
--
|
||||
DROP DATABASE IF EXISTS d3;
|
||||
CREATE DATABASE d3;
|
||||
-- d3 is an empty database.
|
||||
|
||||
@@ -10,12 +10,12 @@ use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use English qw(-no_match_vars);
|
||||
use Test::More;
|
||||
use Data::Dumper;
|
||||
|
||||
use PerconaTest;
|
||||
use Sandbox;
|
||||
require "$trunk/bin/pt-table-sync";
|
||||
|
||||
|
||||
my $vp = new VersionParser();
|
||||
my $dp = new DSNParser(opts=>$dsn_opts);
|
||||
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
|
||||
@@ -29,81 +29,145 @@ elsif ( !$slave_dbh ) {
|
||||
plan skip_all => 'Cannot connect to sandbox slave';
|
||||
}
|
||||
else {
|
||||
plan tests => 9;
|
||||
plan tests => 18;
|
||||
}
|
||||
|
||||
# Previous tests slave 12347 to 12346 which makes pt-table-checksum
|
||||
# complain that it cannot connect to 12347 for checking repl filters
|
||||
# and such. 12347 isn't present but SHOW SLAVE HOSTS on 12346 hasn't
|
||||
# figured that out yet, so we restart 12346 to refresh this list.
|
||||
#diag(`/tmp/12346/stop >/dev/null`);
|
||||
#diag(`/tmp/12346/start >/dev/null`);
|
||||
$slave_dbh = $sb->get_dbh_for('slave1');
|
||||
|
||||
my $master_dsn = $sb->dsn_for('master');
|
||||
my $slave1_dsn = $sb->dsn_for('slave1');
|
||||
my $output;
|
||||
my $cnf = "/tmp/12345/my.sandbox.cnf";
|
||||
my $cmd = "$trunk/bin/pt-table-sync -F $cnf";
|
||||
my $t = qr/\d\d:\d\d:\d\d/;
|
||||
|
||||
$sb->wipe_clean($master_dbh);
|
||||
$sb->load_file('master', 't/pt-table-sync/samples/filter_tables.sql');
|
||||
# See this SQL file because it has a number of simple dbs and tbls
|
||||
# that are used to checking schema object filters.
|
||||
$sb->load_file('master', "t/lib/samples/SchemaIterator.sql");
|
||||
|
||||
$output = `$cmd h=127.1,P=12345 P=12346 --no-check-slave --dry-run -t issue_806_1.t2 | tail -n 2`;
|
||||
$output =~ s/$t/00:00:00/g;
|
||||
$output =~ s/[ ]{2,}/ /g;
|
||||
is(
|
||||
$output,
|
||||
"# DELETE REPLACE INSERT UPDATE ALGORITHM START END EXIT DATABASE.TABLE
|
||||
# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_1.t2
|
||||
",
|
||||
"db-qualified --tables (issue 806)"
|
||||
sub test_filters {
|
||||
my (%args) = @_;
|
||||
|
||||
$sb->clear_genlogs();
|
||||
|
||||
my $output = output(
|
||||
sub { pt_table_sync::main(@{$args{cmds}},
|
||||
qw(--print))
|
||||
},
|
||||
);
|
||||
|
||||
my $tables_used = PerconaTest::tables_used($sb->genlog('master'));
|
||||
is_deeply(
|
||||
$tables_used,
|
||||
$args{res},
|
||||
"$args{name}: tables used"
|
||||
) or diag(Dumper($tables_used));
|
||||
}
|
||||
|
||||
# #############################################################################
|
||||
# Basic schema object filters: --databases, --tables, etc.
|
||||
# #############################################################################
|
||||
|
||||
# Not really a filter, but only the specified tables should be used.
|
||||
test_filters(
|
||||
name => "Sync d1.t1 to d1.t2",
|
||||
cmds => ["$master_dsn,D=d1,t=t1", "t=t2"],
|
||||
res => [qw(d1.t1 d1.t2)],
|
||||
);
|
||||
|
||||
# Use slave1 like it's another master, ok becuase we're not actually
|
||||
# syncing anything, so --no-check-slave is required else pt-table-sync
|
||||
# won't run (because it doesn't like to sync directly to a slave).
|
||||
test_filters(
|
||||
name => "-t d1.t1",
|
||||
cmds => [$master_dsn, $slave1_dsn, qw(--no-check-slave),
|
||||
qw(-t d1.t1)],
|
||||
res => [qw(d1.t1)],
|
||||
);
|
||||
|
||||
# Like the previous test, but now any table called "t1".
|
||||
test_filters(
|
||||
name => "-t t1",
|
||||
cmds => [$master_dsn, $slave1_dsn, qw(--no-check-slave),
|
||||
qw(-t t1)],
|
||||
res => [qw(d1.t1 d2.t1)],
|
||||
);
|
||||
|
||||
# Only the given db, all its tables: there's only 1 tbl in d2.
|
||||
test_filters(
|
||||
name => "--databases d2",
|
||||
cmds => [$master_dsn, $slave1_dsn, qw(--no-check-slave),
|
||||
qw(--databases d2)],
|
||||
res => [qw(d2.t1)],
|
||||
);
|
||||
|
||||
# --database with --tables.
|
||||
test_filters(
|
||||
name => "--databases d1 --tables t2,t3",
|
||||
cmds => [$master_dsn, $slave1_dsn, qw(--no-check-slave),
|
||||
qw(--databases d1), "--tables", "t2,t3"],
|
||||
res => [qw(d1.t2 d1.t3)],
|
||||
);
|
||||
|
||||
# #############################################################################
|
||||
# Issue 820: Make mk-table-sync honor schema filters with --replicate
|
||||
# Filters with --replicate and --sync-to-master.
|
||||
# #############################################################################
|
||||
$master_dbh->do('DROP DATABASE IF EXISTS test');
|
||||
$master_dbh->do('CREATE DATABASE test');
|
||||
|
||||
# Checksum the filter tables.
|
||||
$master_dbh->do("DROP DATABASE IF EXISTS percona");
|
||||
$sb->wait_for_slaves();
|
||||
diag(`$trunk/bin/pt-table-checksum $master_dsn -d d1,d2,d3 --chunk-size 100 --quiet --lock-wait-timeout 3 --max-load ''`);
|
||||
|
||||
$slave_dbh->do('insert into issue_806_1.t1 values (41)');
|
||||
$slave_dbh->do('insert into issue_806_2.t2 values (42)');
|
||||
my $rows = $master_dbh->selectall_arrayref("SELECT CONCAT(db, '.', tbl) FROM percona.checksums ORDER BY db, tbl");
|
||||
is_deeply(
|
||||
$rows,
|
||||
[
|
||||
["d1.t1"],
|
||||
["d1.t2"],
|
||||
["d1.t3"],
|
||||
["d2.t1"],
|
||||
],
|
||||
"Checksummed all tables"
|
||||
) or diag(Dumper($rows));
|
||||
|
||||
my $mk_table_checksum = "$trunk/bin/pt-table-checksum --lock-wait-time 3";
|
||||
# Make all checksums on the slave different than the master
|
||||
# so that pt-table-sync would sync all the tables if there
|
||||
# were no filters.
|
||||
$slave_dbh->do("UPDATE percona.checksums SET this_cnt=999 WHERE 1=1");
|
||||
|
||||
`$mk_table_checksum --replicate test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d issue_806_1,issue_806_2 --quiet`;
|
||||
|
||||
$output = `$cmd h=127.1,P=12345 --replicate test.checksum --dry-run | tail -n 2`;
|
||||
$output =~ s/$t/00:00:00/g;
|
||||
$output =~ s/[ ]{2,}/ /g;
|
||||
is(
|
||||
$output,
|
||||
"# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_1.t1
|
||||
# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_2.t2
|
||||
",
|
||||
"--replicate with no filters"
|
||||
# Verify that that ^ is true.
|
||||
test_filters(
|
||||
name => "All tables are different on the slave",
|
||||
cmds => [$master_dsn, qw(--replicate percona.checksums)],
|
||||
res => [qw(d1.t1 d1.t2 d1.t3 d2.t1 percona.checksums)],
|
||||
);
|
||||
|
||||
$output = `$cmd h=127.1,P=12345 --replicate test.checksum --dry-run -t t1 | tail -n 2`;
|
||||
$output =~ s/$t/00:00:00/g;
|
||||
$output =~ s/[ ]{2,}/ /g;
|
||||
is(
|
||||
$output,
|
||||
"# DELETE REPLACE INSERT UPDATE ALGORITHM START END EXIT DATABASE.TABLE
|
||||
# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_1.t1
|
||||
",
|
||||
"--replicate with --tables"
|
||||
);
|
||||
# Sync with --replicate, --sync-to-master, and some filters.
|
||||
# --replicate and --sync-to-master have different code paths,
|
||||
# but the filter results should be the same.
|
||||
foreach my $args (
|
||||
[$master_dsn, qw(--replicate percona.checksums)],
|
||||
[$slave1_dsn, qw(--replicate percona.checksums --sync-to-master)]
|
||||
) {
|
||||
|
||||
$output = `$cmd h=127.1,P=12345 --replicate test.checksum --dry-run -d issue_806_2 | tail -n 2`;
|
||||
$output =~ s/$t/00:00:00/g;
|
||||
$output =~ s/[ ]{2,}/ /g;
|
||||
is(
|
||||
$output,
|
||||
"# DELETE REPLACE INSERT UPDATE ALGORITHM START END EXIT DATABASE.TABLE
|
||||
# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_2.t2
|
||||
",
|
||||
"--replicate with --databases"
|
||||
);
|
||||
my $stm = $args->[-1] eq '--sync-to-master' ? ' --sync-to-master' : '';
|
||||
|
||||
test_filters(
|
||||
name => $stm . "--replicate --tables t1",
|
||||
cmds => [@$args,
|
||||
qw(--tables t1)],
|
||||
res => [qw(d1.t1 d2.t1 percona.checksums)],
|
||||
);
|
||||
|
||||
test_filters(
|
||||
name => $stm . "--replicate --databases d2",
|
||||
cmds => [@$args,
|
||||
qw(--databases d2)],
|
||||
res => [qw(d2.t1 percona.checksums)],
|
||||
);
|
||||
|
||||
test_filters(
|
||||
name => $stm . "--replicate --databases d1 --tables t1,t3",
|
||||
cmds => [@$args,
|
||||
qw(--databases d1), "--tables", "t1,t3"],
|
||||
res => [qw(d1.t1 d1.t3 percona.checksums)],
|
||||
);
|
||||
}
|
||||
|
||||
# #############################################################################
|
||||
# pt-table-sync --ignore-* options don't work with --replicate
|
||||
@@ -120,7 +184,7 @@ $slave_dbh->do("INSERT INTO test.empty_it VALUES (null,11,11,'eleven')");
|
||||
diag(`$trunk/bin/pt-table-checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test --quiet --quiet --lock-wait-timeout 3 --max-load ''`);
|
||||
|
||||
# Make sure all the tables were checksummed.
|
||||
my $rows = $master_dbh->selectall_arrayref("SELECT DISTINCT db, tbl FROM percona.checksums ORDER BY db, tbl");
|
||||
$rows = $master_dbh->selectall_arrayref("SELECT DISTINCT db, tbl FROM percona.checksums ORDER BY db, tbl");
|
||||
is_deeply(
|
||||
$rows,
|
||||
[ [qw(test empty_it) ],
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
#!/usr/bin/env perl
|
||||
|
||||
BEGIN {
|
||||
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
|
||||
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
|
||||
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
|
||||
};
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use English qw(-no_match_vars);
|
||||
use Test::More;
|
||||
|
||||
use PerconaTest;
|
||||
use Sandbox;
|
||||
require "$trunk/bin/pt-table-sync";
|
||||
|
||||
my $output;
|
||||
my $dp = new DSNParser(opts=>$dsn_opts);
|
||||
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
|
||||
my $master_dbh = $sb->get_dbh_for('master');
|
||||
my $slave_dbh = $sb->get_dbh_for('slave1');
|
||||
|
||||
if ( !$master_dbh ) {
|
||||
plan skip_all => 'Cannot connect to sandbox master';
|
||||
}
|
||||
elsif ( !$slave_dbh ) {
|
||||
plan skip_all => 'Cannot connect to sandbox slave';
|
||||
}
|
||||
else {
|
||||
plan tests => 7;
|
||||
}
|
||||
|
||||
$sb->wipe_clean($master_dbh);
|
||||
$sb->create_dbs($master_dbh, [qw(test)]);
|
||||
$sb->load_file('master', 't/pt-table-sync/samples/before.sql');
|
||||
PerconaTest::wait_for_table($slave_dbh, "test.test4", "id=15034");
|
||||
|
||||
# #############################################################################
|
||||
# Issue 79: mk-table-sync with --replicate doesn't honor --tables
|
||||
# #############################################################################
|
||||
|
||||
$master_dbh->do('create table test.messages (i int)');
|
||||
PerconaTest::wait_for_table($slave_dbh, "test.messages", "i=3");
|
||||
$slave_dbh->do('insert into test.messages values (1), (2), (3)');
|
||||
|
||||
# The previous test should have left test.messages on the slave (12346)
|
||||
# out of sync. Now we also unsync test2 on the slave and then re-sync only
|
||||
# it. If --tables is honored, only test2 on the slave will be synced.
|
||||
$sb->use('master', "-D test -e \"SET SQL_LOG_BIN=0; INSERT INTO test2 VALUES (1,'a'),(2,'b')\"");
|
||||
diag(`$trunk/bin/pt-table-checksum --replicate=test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test --lock-wait-time 3 > /dev/null`);
|
||||
|
||||
is_deeply(
|
||||
$master_dbh->selectall_arrayref('select * from test.messages'),
|
||||
[],
|
||||
'test.messages on master empty'
|
||||
);
|
||||
is_deeply(
|
||||
$slave_dbh->selectall_arrayref('select * from test.messages'),
|
||||
[[1],[2],[3]],
|
||||
'test.messages on slave has values'
|
||||
);
|
||||
|
||||
# Test that what the doc says about --tables is true:
|
||||
# "Table names may be qualified with the database name."
|
||||
# In the code, a qualified db.tbl name is used.
|
||||
# So we'll test first an unqualified tbl name.
|
||||
$output = `$trunk/bin/pt-table-sync h=127.1,P=12345,u=msandbox,p=msandbox --replicate test.checksum --execute -d test -t test2 -v`;
|
||||
unlike($output, qr/messages/, '--replicate honors --tables (1/4)');
|
||||
like($output, qr/test2/, '--replicate honors --tables (2/4)');
|
||||
|
||||
# Now we'll test with a qualified db.tbl name.
|
||||
$sb->use('slave1', '-D test -e "TRUNCATE TABLE test2; TRUNCATE TABLE messages"');
|
||||
diag(`$trunk/bin/pt-table-checksum --replicate=test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test --lock-wait-time 3 > /dev/null`);
|
||||
|
||||
$output = `$trunk/bin/pt-table-sync h=127.1,P=12345,u=msandbox,p=msandbox --replicate test.checksum --execute -d test -t test.test2 -v`;
|
||||
unlike($output, qr/messages/, '--replicate honors --tables (3/4)');
|
||||
like($output, qr/test2/, '--replicate honors --tables (4/4)');
|
||||
|
||||
# #############################################################################
|
||||
# Done.
|
||||
# #############################################################################
|
||||
$sb->wipe_clean($master_dbh);
|
||||
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
|
||||
exit;
|
||||
@@ -1,13 +0,0 @@
|
||||
drop database if exists issue_806_1;
|
||||
drop database if exists issue_806_2;
|
||||
create database issue_806_1;
|
||||
create database issue_806_2;
|
||||
|
||||
use issue_806_1;
|
||||
create table t1 (i int, unique index (i));
|
||||
create table t2 (i int, unique index (i));
|
||||
|
||||
use issue_806_2;
|
||||
create table t1 (i int, unique index (i));
|
||||
create table t2 (i int, unique index (i));
|
||||
create table t3 (i int, unique index (i));
|
||||
Reference in New Issue
Block a user