mirror of
https://github.com/percona/percona-toolkit.git
synced 2025-12-24 02:01:42 +08:00
Convert pt-table-checksum boundaries to SQL WHERE clauses.
This commit is contained in:
Daniel Nichter
@@ -25,11 +25,11 @@ my $dp = new DSNParser(opts=>$dsn_opts);
|
||||
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
|
||||
my $c1_dbh = $sb->get_dbh_for('master');
|
||||
|
||||
diag(`$trunk/sandbox/start-sandbox master 12347 >/dev/null`);
|
||||
my $r1_dbh = $sb->get_dbh_for('slave2');
|
||||
diag(`$trunk/sandbox/start-sandbox master 2900 >/dev/null`);
|
||||
my $r1_dbh = $sb->get_dbh_for('master3');
|
||||
|
||||
diag(`$trunk/sandbox/start-sandbox master 12348 >/dev/null`);
|
||||
my $r2_dbh = $sb->get_dbh_for('master1');
|
||||
diag(`$trunk/sandbox/start-sandbox master 2901 >/dev/null`);
|
||||
my $r2_dbh = $sb->get_dbh_for('master4');
|
||||
|
||||
if ( !$c1_dbh ) {
|
||||
plan skip_all => 'Cannot connect to sandbox master';
|
||||
@@ -44,16 +44,16 @@ else {
|
||||
|
||||
my $output;
|
||||
my $cnf = '/tmp/12345/my.sandbox.cnf';
|
||||
my @args = ('-F', $cnf, 'h=127.1,P=12345', 'P=12347', qw(-d bidi --bidirectional));
|
||||
my @args = ('-F', $cnf, 'h=127.1,P=12345', 'P=2900', qw(-d bidi --bidirectional));
|
||||
|
||||
$sb->wipe_clean($c1_dbh);
|
||||
$sb->wipe_clean($r1_dbh);
|
||||
|
||||
sub load_bidi_data {
|
||||
$sb->load_file('master', 't/pt-table-sync/samples/bidirectional/table.sql');
|
||||
$sb->load_file('slave2', 't/pt-table-sync/samples/bidirectional/table.sql');
|
||||
$sb->load_file('master3', 't/pt-table-sync/samples/bidirectional/table.sql');
|
||||
$sb->load_file('master', 't/pt-table-sync/samples/bidirectional/master-data.sql');
|
||||
$sb->load_file('slave2', 't/pt-table-sync/samples/bidirectional/remote-1.sql');
|
||||
$sb->load_file('master3', 't/pt-table-sync/samples/bidirectional/remote-1.sql');
|
||||
}
|
||||
|
||||
my $r1_data_synced = [
|
||||
@@ -135,13 +135,13 @@ $output = output(
|
||||
|
||||
is(
|
||||
$output,
|
||||
"/*127.1:12347*/ UPDATE `bidi`.`t` SET `c`='ghi', `d`='5', `ts`='2010-02-01 09:17:52' WHERE `id`='3' LIMIT 1;
|
||||
"/*127.1:2900*/ UPDATE `bidi`.`t` SET `c`='ghi', `d`='5', `ts`='2010-02-01 09:17:52' WHERE `id`='3' LIMIT 1;
|
||||
/*127.1:12345*/ UPDATE `bidi`.`t` SET `c`=NULL, `d`='0', `ts`='2010-02-02 05:10:00' WHERE `id`='5' LIMIT 1;
|
||||
/*127.1:12345*/ INSERT INTO `bidi`.`t`(`id`, `c`, `d`, `ts`) VALUES ('11', '?', '0', '2010-01-29 11:17:12');
|
||||
/*127.1:12345*/ UPDATE `bidi`.`t` SET `c`='hmm', `d`='1', `ts`='2010-02-02 12:17:31' WHERE `id`='13' LIMIT 1;
|
||||
/*127.1:12345*/ UPDATE `bidi`.`t` SET `c`='gtg', `d`='7', `ts`='2010-02-02 06:01:08' WHERE `id`='15' LIMIT 1;
|
||||
/*127.1:12345*/ INSERT INTO `bidi`.`t`(`id`, `c`, `d`, `ts`) VALUES ('17', 'good', '1', '2010-02-02 21:38:03');
|
||||
/*127.1:12347*/ INSERT INTO `bidi`.`t`(`id`, `c`, `d`, `ts`) VALUES ('20', 'new', '100', '2010-02-01 04:15:36');
|
||||
/*127.1:2900*/ INSERT INTO `bidi`.`t`(`id`, `c`, `d`, `ts`) VALUES ('20', 'new', '100', '2010-02-01 04:15:36');
|
||||
",
|
||||
'--print correct SQL for c1<->r1 bidirectional sync'
|
||||
);
|
||||
@@ -181,7 +181,7 @@ is(
|
||||
/*127.1:12345*/ UPDATE `bidi`.`t` SET `c`='hmm', `d`='1', `ts`='2010-02-02 12:17:31' WHERE `id`='13' LIMIT 1;
|
||||
/*127.1:12345*/ UPDATE `bidi`.`t` SET `c`='gtg', `d`='7', `ts`='2010-02-02 06:01:08' WHERE `id`='15' LIMIT 1;
|
||||
/*127.1:12345*/ INSERT INTO `bidi`.`t`(`id`, `c`, `d`, `ts`) VALUES ('17', 'good', '1', '2010-02-02 21:38:03');
|
||||
/*127.1:12347*/ INSERT INTO `bidi`.`t`(`id`, `c`, `d`, `ts`) VALUES ('20', 'new', '100', '2010-02-01 04:15:36');
|
||||
/*127.1:2900*/ INSERT INTO `bidi`.`t`(`id`, `c`, `d`, `ts`) VALUES ('20', 'new', '100', '2010-02-01 04:15:36');
|
||||
",
|
||||
'SQL for c1<->r1 with conflict'
|
||||
);
|
||||
@@ -324,14 +324,14 @@ is_deeply(
|
||||
# Test bidirectional sync with 3 servers.
|
||||
# #############################################################################
|
||||
|
||||
# It's confusing but master1 = 12348, aka our 3rd master server.
|
||||
# It's confusing but master4 = 2901, aka our 3rd master server.
|
||||
|
||||
SKIP: {
|
||||
skip 'Cannot connect to third sandbox master', 9 unless $r2_dbh;
|
||||
|
||||
load_bidi_data();
|
||||
$sb->load_file('master1', 't/pt-table-sync/samples/bidirectional/table.sql');
|
||||
$sb->load_file('master1', 't/pt-table-sync/samples/bidirectional/remote-2.sql');
|
||||
$sb->load_file('master4', 't/pt-table-sync/samples/bidirectional/table.sql');
|
||||
$sb->load_file('master4', 't/pt-table-sync/samples/bidirectional/remote-2.sql');
|
||||
|
||||
$res = $r2_dbh->selectall_arrayref('select * from bidi.t order by id');
|
||||
is_deeply(
|
||||
@@ -361,7 +361,7 @@ SKIP: {
|
||||
local *STDERR;
|
||||
open STDERR, '>', \$err;
|
||||
$output = output(
|
||||
sub { pt_table_sync::main(@args, 'h=127.1,P=12348',
|
||||
sub { pt_table_sync::main(@args, 'h=127.1,P=2901',
|
||||
qw(--print --execute --chunk-size 2),
|
||||
qw(--conflict-column ts --conflict-comparison newest)) }
|
||||
);
|
||||
@@ -456,7 +456,7 @@ SKIP: {
|
||||
local *STDERR;
|
||||
open STDERR, '>', \$err;
|
||||
$output = output(
|
||||
sub { pt_table_sync::main(@args, 'h=127.1,P=12348',
|
||||
sub { pt_table_sync::main(@args, 'h=127.1,P=2901',
|
||||
qw(--print --execute --chunk-size 2),
|
||||
qw(--conflict-column ts --conflict-comparison newest)) }
|
||||
);
|
||||
@@ -534,7 +534,6 @@ SKIP: {
|
||||
# #############################################################################
|
||||
# Done.
|
||||
# #############################################################################
|
||||
diag(`$trunk/sandbox/stop-sandbox 12347 >/dev/null`);
|
||||
diag(`$trunk/sandbox/stop-sandbox 12348 >/dev/null`);
|
||||
$sb->wipe_clean($c1_dbh);
|
||||
diag(`$trunk/sandbox/stop-sandbox 2900 >/dev/null`);
|
||||
diag(`$trunk/sandbox/stop-sandbox 2901 >/dev/null`);
|
||||
exit;
|
||||
|
||||
63
t/pt-table-sync/diff_where.t
Normal file
63
t/pt-table-sync/diff_where.t
Normal file
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env perl
|
||||
|
||||
BEGIN {
|
||||
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
|
||||
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
|
||||
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
|
||||
};
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use English qw(-no_match_vars);
|
||||
use Test::More tests => 1;
|
||||
|
||||
use PerconaTest;
|
||||
use Sandbox;
|
||||
require "$trunk/bin/pt-table-sync";
|
||||
|
||||
my $q = new Quoter();
|
||||
my $tp = new TableParser(Quoter => $q);
|
||||
my $tn = new TableNibbler(Quoter => $q, TableParser => $tp);
|
||||
|
||||
my $sample = "t/pt-table-sync/samples";
|
||||
|
||||
sub test_diff_where {
|
||||
my (%args) = @_;
|
||||
|
||||
my $ddl = load_file($args{file});
|
||||
my $tbl_struct = $tp->parse($ddl);
|
||||
my $where = pt_table_sync::diff_where(
|
||||
tbl_struct => $tbl_struct,
|
||||
diff => $args{diff},
|
||||
TableNibbler => $tn,
|
||||
);
|
||||
is(
|
||||
$where,
|
||||
$args{where},
|
||||
$args{name},
|
||||
);
|
||||
}
|
||||
|
||||
test_diff_where(
|
||||
name => "Single int col",
|
||||
file => "$sample/simple-tbl-ddl.sql",
|
||||
diff => {
|
||||
chunk => '3',
|
||||
chunk_index => 'PRIMARY',
|
||||
cnt_diff => '-1',
|
||||
crc_diff => '1',
|
||||
lower_boundary => '7',
|
||||
master_cnt => '3',
|
||||
master_crc => '1ddd6c71',
|
||||
table => 'test.mt1',
|
||||
this_cnt => '2',
|
||||
this_crc => '4a57d814',
|
||||
upper_boundary => '9'
|
||||
},
|
||||
where => "((`id` >= 7)) AND ((`id` <= 9))",
|
||||
);
|
||||
|
||||
# #############################################################################
|
||||
# Done.
|
||||
# #############################################################################
|
||||
exit;
|
||||
@@ -36,8 +36,8 @@ else {
|
||||
# complain that it cannot connect to 12347 for checking repl filters
|
||||
# and such. 12347 isn't present but SHOW SLAVE HOSTS on 12346 hasn't
|
||||
# figured that out yet, so we restart 12346 to refresh this list.
|
||||
diag(`/tmp/12346/stop >/dev/null`);
|
||||
diag(`/tmp/12346/start >/dev/null`);
|
||||
#diag(`/tmp/12346/stop >/dev/null`);
|
||||
#diag(`/tmp/12346/start >/dev/null`);
|
||||
$slave_dbh = $sb->get_dbh_for('slave1');
|
||||
|
||||
my $output;
|
||||
@@ -64,23 +64,21 @@ is(
|
||||
# #############################################################################
|
||||
$master_dbh->do('DROP DATABASE IF EXISTS test');
|
||||
$master_dbh->do('CREATE DATABASE test');
|
||||
$sb->load_file('master', 't/pt-table-sync/samples/checksum_tbl.sql', 'test');
|
||||
|
||||
$slave_dbh->do('insert into issue_806_1.t1 values (41)');
|
||||
$slave_dbh->do('insert into issue_806_2.t2 values (42)');
|
||||
|
||||
my $mk_table_checksum = "$trunk/bin/pt-table-checksum";
|
||||
my $mk_table_checksum = "$trunk/bin/pt-table-checksum --lock-wait-time 3";
|
||||
|
||||
`$mk_table_checksum -F $cnf --replicate test.checksum h=127.1,P=12345 -d issue_806_1,issue_806_2 --quiet`;
|
||||
`$mk_table_checksum -F $cnf --replicate test.checksum h=127.1,P=12345 -d issue_806_1,issue_806_2 --replicate-check 1 --quiet`;
|
||||
`$mk_table_checksum --replicate test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d issue_806_1,issue_806_2 --quiet`;
|
||||
|
||||
$output = `$cmd h=127.1,P=12345 --replicate test.checksum --dry-run | tail -n 2`;
|
||||
$output =~ s/$t/00:00:00/g;
|
||||
$output =~ s/[ ]{2,}/ /g;
|
||||
is(
|
||||
$output,
|
||||
"# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_2.t2
|
||||
# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_1.t1
|
||||
"# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_1.t1
|
||||
# 0 0 0 0 Chunk 00:00:00 00:00:00 0 issue_806_2.t2
|
||||
",
|
||||
"--replicate with no filters"
|
||||
);
|
||||
|
||||
@@ -39,22 +39,17 @@ $sb->create_dbs($master_dbh, [qw(test)]);
|
||||
# Issue 560: mk-table-sync generates impossible WHERE
|
||||
# #############################################################################
|
||||
diag(`/tmp/12345/use < $trunk/t/pt-table-sync/samples/issue_560.sql`);
|
||||
sleep 1;
|
||||
PerconaTest::wait_for_table($slave_dbh, "issue_560.buddy_list", "player_id=353");
|
||||
|
||||
# Make slave differ.
|
||||
$slave_dbh->do('UPDATE issue_560.buddy_list SET buddy_id=0 WHERE player_id IN (333,334)');
|
||||
$slave_dbh->do('UPDATE issue_560.buddy_list SET buddy_id=0 WHERE player_id=486');
|
||||
|
||||
diag(`$trunk/bin/pt-table-checksum --replicate issue_560.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d issue_560 --chunk-size 50 > /dev/null`);
|
||||
sleep 1;
|
||||
$output = `$trunk/bin/pt-table-checksum --replicate issue_560.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d issue_560 --replicate-check 1 --chunk-size 50`;
|
||||
$output =~ s/\d\d:\d\d:\d\d/00:00:00/g;
|
||||
ok(
|
||||
no_diff(
|
||||
$output,
|
||||
"t/pt-table-sync/samples/issue_560_output_1.txt",
|
||||
cmd_output => 1,
|
||||
),
|
||||
$output = `$trunk/bin/pt-table-checksum --replicate issue_560.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d issue_560 --chunk-size 50 --lock-wait-time 3`;
|
||||
|
||||
is(
|
||||
PerconaTest::count_checksum_results($output, 'diffs'),
|
||||
2,
|
||||
'Found checksum differences (issue 560)'
|
||||
);
|
||||
|
||||
|
||||
@@ -48,8 +48,7 @@ $slave_dbh->do('UPDATE issue_375.t SET foo="z" WHERE id=10');
|
||||
$slave_dbh->do('UPDATE issue_375.t SET foo="zz" WHERE id=100');
|
||||
|
||||
# Checksum and replicate.
|
||||
diag(`$trunk/bin/pt-table-checksum --create-replicate-table --replicate issue_375.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d issue_375 -t t > /dev/null`);
|
||||
diag(`$trunk/bin/pt-table-checksum --replicate issue_375.checksum h=127.1,P=12345,u=msandbox,p=msandbox --replicate-check 1 > /dev/null`);
|
||||
diag(`$trunk/bin/pt-table-checksum --create-replicate-table --replicate issue_375.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d issue_375 -t t --lock-wait-time 3 > /dev/null`);
|
||||
|
||||
# And now sync using the replicated checksum results/differences.
|
||||
$output = output(
|
||||
|
||||
@@ -35,7 +35,6 @@ $sb->wipe_clean($master_dbh);
|
||||
$sb->wipe_clean($slave_dbh);
|
||||
$sb->create_dbs($master_dbh, [qw(test)]);
|
||||
$sb->load_file('master', 't/pt-table-sync/samples/before.sql');
|
||||
$sb->load_file('master', 't/pt-table-sync/samples/checksum_tbl.sql');
|
||||
|
||||
# #############################################################################
|
||||
# Issue 79: mk-table-sync with --replicate doesn't honor --tables
|
||||
@@ -49,7 +48,7 @@ $slave_dbh->do('insert into test.messages values (1), (2), (3)');
|
||||
# out of sync. Now we also unsync test2 on the slave and then re-sync only
|
||||
# it. If --tables is honored, only test2 on the slave will be synced.
|
||||
$sb->use('master', "-D test -e \"SET SQL_LOG_BIN=0; INSERT INTO test2 VALUES (1,'a'),(2,'b')\"");
|
||||
diag(`$trunk/bin/pt-table-checksum --replicate=test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test > /dev/null`);
|
||||
diag(`$trunk/bin/pt-table-checksum --replicate=test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test --lock-wait-time 3 > /dev/null`);
|
||||
|
||||
is_deeply(
|
||||
$master_dbh->selectall_arrayref('select * from test.messages'),
|
||||
@@ -72,7 +71,7 @@ like($output, qr/test2/, '--replicate honors --tables (2/4)');
|
||||
|
||||
# Now we'll test with a qualified db.tbl name.
|
||||
$sb->use('slave1', '-D test -e "TRUNCATE TABLE test2; TRUNCATE TABLE messages"');
|
||||
diag(`$trunk/bin/pt-table-checksum --replicate=test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test > /dev/null`);
|
||||
diag(`$trunk/bin/pt-table-checksum --replicate=test.checksum h=127.1,P=12345,u=msandbox,p=msandbox -d test --lock-wait-time 3 > /dev/null`);
|
||||
|
||||
$output = `$trunk/bin/pt-table-sync h=127.1,P=12345,u=msandbox,p=msandbox --replicate test.checksum --execute -d test -t test.test2 -v`;
|
||||
unlike($output, qr/messages/, '--replicate honors --tables (3/4)');
|
||||
|
||||
@@ -33,7 +33,7 @@ else {
|
||||
my $output;
|
||||
my @args = ('--sync-to-master', 'h=127.1,P=12346,u=msandbox,p=msandbox',
|
||||
qw(-d issue_375 --replicate issue_375.checksums --print));
|
||||
my $mk_table_checksum = "$trunk/bin/pt-table-checksum F=/tmp/12345/my.sandbox.cnf -d issue_375 --chunk-size 20 --chunk-size-limit 0";
|
||||
my $pt_table_checksum = "$trunk/bin/pt-table-checksum h=127.1,P=12345,u=msandbox,p=msandbox -d issue_375 --chunk-size 20 --chunk-size-limit 0 --lock-wait-time 3";
|
||||
|
||||
# #############################################################################
|
||||
# Issue 996: might not chunk inside of mk-table-checksum's boundaries
|
||||
@@ -65,13 +65,10 @@ wait_until(
|
||||
);
|
||||
|
||||
# mk-table-checksum the table with 5 chunks of 20 rows.
|
||||
$output = `$mk_table_checksum --replicate issue_375.checksums --create-replicate-table`;
|
||||
sleep 1;
|
||||
|
||||
$output = `$mk_table_checksum --replicate issue_375.checksums --replicate-check 1`;
|
||||
like(
|
||||
$output,
|
||||
qr/issue_375\s+t\s+2\s+0\s+1\s+`id` >= '21' AND `id` < '41'/,
|
||||
$output = `$pt_table_checksum --replicate issue_375.checksums`;
|
||||
is(
|
||||
PerconaTest::count_checksum_results($output, 'diffs'),
|
||||
1,
|
||||
"Chunk checksum diff"
|
||||
);
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
10
t/pt-table-sync/samples/simple-tbl-ddl.sql
Normal file
10
t/pt-table-sync/samples/simple-tbl-ddl.sql
Normal file
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE `it1` (
|
||||
`id` int(11) NOT NULL AUTO_INCREMENT,
|
||||
`a` int(11) NOT NULL,
|
||||
`b` int(11) NOT NULL,
|
||||
`c` varchar(16) NOT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `c` (`c`),
|
||||
UNIQUE KEY `id` (`id`,`c`),
|
||||
KEY `a` (`a`)
|
||||
) ENGINE=InnoDB
|
||||
@@ -44,8 +44,8 @@ $sb->create_dbs($master_dbh, [qw(test)]);
|
||||
# #############################################################################
|
||||
$sb->load_file('master', 't/pt-table-sync/samples/issue_37.sql');
|
||||
$sb->use('master', '-e "SET SQL_LOG_BIN=0; INSERT INTO test.issue_37 VALUES (1), (2);"');
|
||||
$sb->load_file('master', 't/pt-table-sync/samples/checksum_tbl.sql');
|
||||
`$trunk/bin/pt-table-checksum h=127.0.0.1,P=12345,u=msandbox,p=msandbox --replicate test.checksum -d test 2>&1 > /dev/null`;
|
||||
|
||||
`$trunk/bin/pt-table-checksum h=127.0.0.1,P=12345,u=msandbox,p=msandbox --replicate test.checksum -d test --lock-wait-time 3 2>&1 > /dev/null`;
|
||||
|
||||
$output = `$trunk/bin/pt-table-sync --no-check-slave --execute u=msandbox,p=msandbox,h=127.0.0.1,P=12345,D=test,t=issue_37 h=127.1,P=12346 2>&1`;
|
||||
like($output,
|
||||
@@ -117,10 +117,10 @@ $slave_dbh->do('INSERT INTO db1.t1 VALUES (9)');
|
||||
$slave_dbh->do('DELETE FROM db2.t1 WHERE i > 4');
|
||||
|
||||
# Replicate checksum of db2.t1.
|
||||
$output = `$trunk/bin/pt-table-checksum h=127.1,P=12345,u=msandbox,p=msandbox --replicate db1.checksum --create-replicate-table --databases db1,db2 2>&1`;
|
||||
$output = `$trunk/bin/pt-table-checksum h=127.1,P=12345,u=msandbox,p=msandbox --replicate db1.checksum --create-replicate-table --databases db1,db2 --lock-wait-time 3 2>&1`;
|
||||
like(
|
||||
$output,
|
||||
qr/db2\s+t1\s+0\s+127\.1\s+MyISAM\s+5/,
|
||||
qr/db2.t1/,
|
||||
'Replicated checksums (issue 367)'
|
||||
);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user