percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-07 12:58:45 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
873a18a58965d62de12f6d45afe068ab7cf5ce92
percona-toolkit/t/pt-table-sync/bidirectional.t
Carlos Salguero c5a22407a7 PT-2016 pt-table-sync and CRC32 columns (#521) 
* PT-2016 CRC32 on key

* PT-2016 Added tests

* Removed commented out line
2021-11-03 13:53:30 -03:00

541 lines
19 KiB
Perl
Raw Blame History

#!/usr/bin/env perl
BEGIN {
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
};
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Test::More;
use PerconaTest;
use Sandbox;
require "$trunk/bin/pt-table-sync";
use Data::Dumper;
$Data::Dumper::Indent = 0;
$Data::Dumper::Sortkeys = 1;
$Data::Dumper::Quotekeys = 0;
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $c1_dbh = $sb->get_dbh_for('master');
diag(`$trunk/sandbox/start-sandbox master 2900 >/dev/null`);
my $r1_dbh = $sb->get_dbh_for('master3');
diag(`$trunk/sandbox/start-sandbox master 2901 >/dev/null`);
my $r2_dbh = $sb->get_dbh_for('master4');
if ( !$c1_dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
elsif ( !$r1_dbh ) {
plan skip_all => 'Cannot connect to second sandbox master';
}
else {
plan tests => 23;
}
my $output;
my $cnf = '/tmp/12345/my.sandbox.cnf';
my @args = ('-F', $cnf, 'h=127.1,P=12345', 'P=2900', qw(-d bidi --bidirectional --no-version-check));
$sb->wipe_clean($c1_dbh);
$sb->wipe_clean($r1_dbh);
sub load_bidi_data {
$sb->load_file('master', 't/pt-table-sync/samples/bidirectional/table.sql');
$sb->load_file('master3', 't/pt-table-sync/samples/bidirectional/table.sql');
$sb->load_file('master', 't/pt-table-sync/samples/bidirectional/master-data.sql');
$sb->load_file('master3', 't/pt-table-sync/samples/bidirectional/remote-1.sql');
}
my $r1_data_synced = [
[1, 'abc', 1, '2010-02-01 05:45:30'],
[2, 'def', 2, '2010-01-31 06:11:11'],
[3, 'ghi', 5, '2010-02-01 09:17:52'],
[4, 'jkl', 6, '2010-02-01 10:11:33'],
[5, undef, 0, '2010-02-02 05:10:00'],
[6, 'p', 4, '2010-01-31 10:17:00'],
[7, 'qrs', 5, '2010-02-01 10:11:11'],
[8, 'tuv', 6, '2010-01-31 10:17:20'],
[9, 'wxy', 7, '2010-02-01 10:17:00'],
[10, 'z', 8, '2010-01-31 10:17:08'],
[11, '?', 0, '2010-01-29 11:17:12'],
[12, '', 0, '2010-02-01 11:17:00'],
[13, 'hmm', 1, '2010-02-02 12:17:31'],
[14, undef, 0, '2010-01-31 10:17:00'],
[15, 'gtg', 7, '2010-02-02 06:01:08'],
[17, 'good', 1, '2010-02-02 21:38:03'],
[20, 'new', 100, '2010-02-01 04:15:36'],
];
load_bidi_data();
$c1_dbh->do('use bidi');
$r1_dbh->do('use bidi');
my $res = $c1_dbh->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
[
[1, 'abc', 1, '2010-02-01 05:45:30'],
[2, 'def', 2, '2010-01-31 06:11:11'],
[3, 'ghi', 5, '2010-02-01 09:17:52'],
[4, 'jkl', 6, '2010-02-01 10:11:33'],
[5, 'mno', 3, '2010-02-01 10:17:40'],
[6, 'p', 4, '2010-01-31 10:17:00'],
[7, 'qrs', 5, '2010-02-01 10:11:11'],
[8, 'tuv', 6, '2010-01-31 10:17:20'],
[9, 'wxy', 7, '2010-02-01 10:17:00'],
[10, 'z', 8, '2010-01-31 10:17:08'],
[12, '', 0, '2010-02-01 11:17:00'],
[13, undef, 0, '2010-02-01 12:17:31'],
[14, undef, 0, '2010-01-31 10:17:00'],
[15, 'NA', 0, '2010-01-31 07:00:01'],
[20, 'new', 100, '2010-02-01 04:15:36'],
],
'c1 data before sync'
);
$res = $r1_dbh->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
[
[1, 'abc', 1, '2010-02-01 05:45:30'],
[2, 'def', 2, '2010-01-31 06:11:11'],
[3, 'ghi', 5, '2010-02-01 09:17:51'],
[4, 'jkl', 6, '2010-02-01 10:11:33'],
[5, undef, 0, '2010-02-02 05:10:00'],
[6, 'p', 4, '2010-01-31 10:17:00'],
[7, 'qrs', 5, '2010-02-01 10:11:11'],
[8, 'tuv', 6, '2010-01-31 10:17:20'],
[9, 'wxy', 7, '2010-02-01 10:17:00'],
[10, 'z', 8, '2010-01-31 10:17:08'],
[11, '?', 0, '2010-01-29 11:17:12'],
[12, '', 0, '2010-02-01 11:17:00'],
[13, 'hmm', 1, '2010-02-02 12:17:31'],
[14, undef, 0, '2010-01-31 10:17:00'],
[15, 'gtg', 7, '2010-02-02 06:01:08'],
[17, 'good', 1, '2010-02-02 21:38:03'],
],
'r1 data before sync'
);
$output = output(
sub { pt_table_sync::main(@args, qw(--print --execute),
qw(--conflict-column ts --conflict-comparison newest)) }
);
is(
$output,
"/*127.1:2900*/ UPDATE `bidi`.`t` SET `c`='ghi', `d`='5', `ts`='2010-02-01 09:17:52' WHERE `id`='3' LIMIT 1;
/*127.1:12345*/ UPDATE `bidi`.`t` SET `c`=NULL, `d`='0', `ts`='2010-02-02 05:10:00' WHERE `id`='5' LIMIT 1;
/*127.1:12345*/ INSERT INTO `bidi`.`t`(`id`, `c`, `d`, `ts`) VALUES ('11', '?', '0', '2010-01-29 11:17:12');
/*127.1:12345*/ UPDATE `bidi`.`t` SET `c`='hmm', `d`='1', `ts`='2010-02-02 12:17:31' WHERE `id`='13' LIMIT 1;
/*127.1:12345*/ UPDATE `bidi`.`t` SET `c`='gtg', `d`='7', `ts`='2010-02-02 06:01:08' WHERE `id`='15' LIMIT 1;
/*127.1:12345*/ INSERT INTO `bidi`.`t`(`id`, `c`, `d`, `ts`) VALUES ('17', 'good', '1', '2010-02-02 21:38:03');
/*127.1:2900*/ INSERT INTO `bidi`.`t`(`id`, `c`, `d`, `ts`) VALUES ('20', 'new', '100', '2010-02-01 04:15:36');
",
'--print correct SQL for c1<->r1 bidirectional sync'
);
$res = $c1_dbh->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
$r1_data_synced,
'Synced c1'
);
$res = $r1_dbh->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
$r1_data_synced,
'Synced r1'
);
# Set a threshold which will cause some unresolvable conflicts.
load_bidi_data();
my $err = '';
{
local *STDERR;
open STDERR, '>', \$err;
$output = output(
sub { pt_table_sync::main(@args, qw(--print --execute),
qw(--conflict-column ts --conflict-comparison newest),
qw(--conflict-threshold 30m)) }
);
}
is(
$output,
"/*127.1:12345*/ UPDATE `bidi`.`t` SET `c`=NULL, `d`='0', `ts`='2010-02-02 05:10:00' WHERE `id`='5' LIMIT 1;
/*127.1:12345*/ INSERT INTO `bidi`.`t`(`id`, `c`, `d`, `ts`) VALUES ('11', '?', '0', '2010-01-29 11:17:12');
/*127.1:12345*/ UPDATE `bidi`.`t` SET `c`='hmm', `d`='1', `ts`='2010-02-02 12:17:31' WHERE `id`='13' LIMIT 1;
/*127.1:12345*/ UPDATE `bidi`.`t` SET `c`='gtg', `d`='7', `ts`='2010-02-02 06:01:08' WHERE `id`='15' LIMIT 1;
/*127.1:12345*/ INSERT INTO `bidi`.`t`(`id`, `c`, `d`, `ts`) VALUES ('17', 'good', '1', '2010-02-02 21:38:03');
/*127.1:2900*/ INSERT INTO `bidi`.`t`(`id`, `c`, `d`, `ts`) VALUES ('20', 'new', '100', '2010-02-01 04:15:36');
",
'SQL for c1<->r1 with conflict'
);
is(
$err,
"# Cannot resolve conflict WHERE `id`='3': `ts` values do not differ by the threhold, 30m.
",
'Warns about conflict'
);
$res = $c1_dbh->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
[
[1, 'abc', 1, '2010-02-01 05:45:30'],
[2, 'def', 2, '2010-01-31 06:11:11'],
[3, 'ghi', 5, '2010-02-01 09:17:52'], # not synced
[4, 'jkl', 6, '2010-02-01 10:11:33'],
[5, undef, 0, '2010-02-02 05:10:00'],
[6, 'p', 4, '2010-01-31 10:17:00'],
[7, 'qrs', 5, '2010-02-01 10:11:11'],
[8, 'tuv', 6, '2010-01-31 10:17:20'],
[9, 'wxy', 7, '2010-02-01 10:17:00'],
[10, 'z', 8, '2010-01-31 10:17:08'],
[11, '?', 0, '2010-01-29 11:17:12'],
[12, '', 0, '2010-02-01 11:17:00'],
[13, 'hmm', 1, '2010-02-02 12:17:31'],
[14, undef, 0, '2010-01-31 10:17:00'],
[15, 'gtg', 7, '2010-02-02 06:01:08'],
[17, 'good', 1, '2010-02-02 21:38:03'],
[20, 'new', 100, '2010-02-01 04:15:36'],
],
'Synced c1 except conflict'
);
$res = $r1_dbh->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
[
[1, 'abc', 1, '2010-02-01 05:45:30'],
[2, 'def', 2, '2010-01-31 06:11:11'],
[3, 'ghi', 5, '2010-02-01 09:17:51'], # not synced
[4, 'jkl', 6, '2010-02-01 10:11:33'],
[5, undef, 0, '2010-02-02 05:10:00'],
[6, 'p', 4, '2010-01-31 10:17:00'],
[7, 'qrs', 5, '2010-02-01 10:11:11'],
[8, 'tuv', 6, '2010-01-31 10:17:20'],
[9, 'wxy', 7, '2010-02-01 10:17:00'],
[10, 'z', 8, '2010-01-31 10:17:08'],
[11, '?', 0, '2010-01-29 11:17:12'],
[12, '', 0, '2010-02-01 11:17:00'],
[13, 'hmm', 1, '2010-02-02 12:17:31'],
[14, undef, 0, '2010-01-31 10:17:00'],
[15, 'gtg', 7, '2010-02-02 06:01:08'],
[17, 'good', 1, '2010-02-02 21:38:03'],
[20, 'new', 100, '2010-02-01 04:15:36'],
],
'Synced r1 except conflict'
);
# Now die on conflict error.
load_bidi_data();
{
$err = '';
local *STDERR;
open STDERR, '>', \$err;
$output = output(
sub { pt_table_sync::main(@args, qw(--print --execute),
qw(--conflict-column ts --conflict-comparison newest),
qw(--conflict-threshold 30m --conflict-error die)) }
);
};
is(
$output,
"",
'No SQL for c1<->r1 with die on conflict'
);
# mk-table-sync catches death and warns instead so it can continue
# syncing other tables.
is(
$err,
"# Cannot resolve conflict WHERE `id`='3': `ts` values do not differ by the threhold, 30m. while doing bidi.t on 127.1
",
'Die/warn about conflict'
);
$res = $c1_dbh->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
[
[1, 'abc', 1, '2010-02-01 05:45:30'],
[2, 'def', 2, '2010-01-31 06:11:11'],
[3, 'ghi', 5, '2010-02-01 09:17:52'],
[4, 'jkl', 6, '2010-02-01 10:11:33'],
[5, 'mno', 3, '2010-02-01 10:17:40'],
[6, 'p', 4, '2010-01-31 10:17:00'],
[7, 'qrs', 5, '2010-02-01 10:11:11'],
[8, 'tuv', 6, '2010-01-31 10:17:20'],
[9, 'wxy', 7, '2010-02-01 10:17:00'],
[10, 'z', 8, '2010-01-31 10:17:08'],
[12, '', 0, '2010-02-01 11:17:00'],
[13, undef, 0, '2010-02-01 12:17:31'],
[14, undef, 0, '2010-01-31 10:17:00'],
[15, 'NA', 0, '2010-01-31 07:00:01'],
[20, 'new', 100, '2010-02-01 04:15:36'],
],
'c1 not synced due to die on conflict'
);
$res = $r1_dbh->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
[
[1, 'abc', 1, '2010-02-01 05:45:30'],
[2, 'def', 2, '2010-01-31 06:11:11'],
[3, 'ghi', 5, '2010-02-01 09:17:51'],
[4, 'jkl', 6, '2010-02-01 10:11:33'],
[5, undef, 0, '2010-02-02 05:10:00'],
[6, 'p', 4, '2010-01-31 10:17:00'],
[7, 'qrs', 5, '2010-02-01 10:11:11'],
[8, 'tuv', 6, '2010-01-31 10:17:20'],
[9, 'wxy', 7, '2010-02-01 10:17:00'],
[10, 'z', 8, '2010-01-31 10:17:08'],
[11, '?', 0, '2010-01-29 11:17:12'],
[12, '', 0, '2010-02-01 11:17:00'],
[13, 'hmm', 1, '2010-02-02 12:17:31'],
[14, undef, 0, '2010-01-31 10:17:00'],
[15, 'gtg', 7, '2010-02-02 06:01:08'],
[17, 'good', 1, '2010-02-02 21:38:03'],
],
'r1 not synced due to die on conflict'
);
# #############################################################################
# Test bidirectional sync with 3 servers.
# #############################################################################
# It's confusing but master4 = 2901, aka our 3rd master server.
SKIP: {
skip 'Cannot connect to third sandbox master', 9 unless $r2_dbh;
load_bidi_data();
$sb->load_file('master4', 't/pt-table-sync/samples/bidirectional/table.sql');
$sb->load_file('master4', 't/pt-table-sync/samples/bidirectional/remote-2.sql');
$res = $r2_dbh->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
[
[1, 'abc', 1, '2010-02-01 05:45:30'],
[3, 'ghi', 5, '2010-02-01 09:17:52'],
[4, 'jkl', 6, '2010-02-01 10:11:33'],
[5, 'mno', 3, '2010-02-01 10:17:40'],
[6, 'p', 4, '2010-01-31 10:17:00'],
[7, 'qrs', 5, '2010-02-01 10:11:11'],
[8, 'TUV', 6, '2010-01-31 10:17:20'],
[9, 'wxy', 7, '2010-02-01 10:17:00'],
[10, 'rdy', 5, '2010-02-02 09:09:10'],
[11, '?', 0, '2010-01-29 12:19:48'],
[12, '', 0, '2010-02-01 13:17:19'],
[13, undef, 0, '2010-02-01 12:17:31'],
[14, undef, 0, '2010-02-02 13:00:00'],
[15, 'NA', 0, '2010-01-31 07:00:01'],
[16, undef, 0, '2010-02-02 13:20:00'],
],
'r2 data before sync'
);
{
$err = '';
local *STDERR;
open STDERR, '>', \$err;
$output = output(
sub { pt_table_sync::main(@args, 'h=127.1,P=2901',
qw(--print --execute --chunk-size 2),
qw(--conflict-column ts --conflict-comparison newest)) }
);
}
like(
$err,
qr/Cannot resolve conflict WHERE `id`='8': `ts` values are the same/,
'Warned that id=8 differs but has same ts'
);
$res = $c1_dbh->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
[
['1','abc','1','2010-02-01 05:45:30'],
['2','def','2','2010-01-31 06:11:11'],
['3','ghi','5','2010-02-01 09:17:52'],
['4','jkl','6','2010-02-01 10:11:33'],
['5',undef,'0','2010-02-02 05:10:00'],
['6','p','4','2010-01-31 10:17:00'],
['7','qrs','5','2010-02-01 10:11:11'],
['8','tuv','6','2010-01-31 10:17:20'],
['9','wxy','7','2010-02-01 10:17:00'],
['10','rdy','5','2010-02-02 09:09:10'],
['11','?','0','2010-01-29 12:19:48'],
['12','','0','2010-02-01 13:17:19'],
['13','hmm','1','2010-02-02 12:17:31'],
['14',undef,'0','2010-02-02 13:00:00'],
['15','gtg','7','2010-02-02 06:01:08'],
['16',undef,'0','2010-02-02 13:20:00'],
['17','good','1','2010-02-02 21:38:03'],
['20','new','100','2010-02-01 04:15:36']
],
'c1 data synced 1st pass'
);
$res = $r1_dbh->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
[
['1','abc','1','2010-02-01 05:45:30'],
['2','def','2','2010-01-31 06:11:11'],
['3','ghi','5','2010-02-01 09:17:52'],
['4','jkl','6','2010-02-01 10:11:33'],
['5',undef,'0','2010-02-02 05:10:00'],
['6','p','4','2010-01-31 10:17:00'],
['7','qrs','5','2010-02-01 10:11:11'],
['8','tuv','6','2010-01-31 10:17:20'],
['9','wxy','7','2010-02-01 10:17:00'],
['10','z','8','2010-01-31 10:17:08'],
['11','?','0','2010-01-29 11:17:12'],
['12','','0','2010-02-01 11:17:00'],
['13','hmm','1','2010-02-02 12:17:31'],
['14',undef,'0','2010-01-31 10:17:00'],
['15','gtg','7','2010-02-02 06:01:08'],
['17','good','1','2010-02-02 21:38:03'],
['20','new','100','2010-02-01 04:15:36']
],
'r1 data synced 1st pass'
);
$res = $r2_dbh->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
[
['1','abc','1','2010-02-01 05:45:30'],
['2','def','2','2010-01-31 06:11:11'],
['3','ghi','5','2010-02-01 09:17:52'],
['4','jkl','6','2010-02-01 10:11:33'],
['5',undef,'0','2010-02-02 05:10:00'],
['6','p','4','2010-01-31 10:17:00'],
['7','qrs','5','2010-02-01 10:11:11'],
['8','TUV','6','2010-01-31 10:17:20'],
['9','wxy','7','2010-02-01 10:17:00'],
['10','rdy','5','2010-02-02 09:09:10'],
['11','?','0','2010-01-29 12:19:48'],
['12','','0','2010-02-01 13:17:19'],
['13','hmm','1','2010-02-02 12:17:31'],
['14',undef,'0','2010-02-02 13:00:00'],
['15','gtg','7','2010-02-02 06:01:08'],
['16',undef,'0','2010-02-02 13:20:00'],
['17','good','1','2010-02-02 21:38:03'],
['20','new','100','2010-02-01 04:15:36']
],
'r2 data synced first pass'
);
# 2nd pass to sync r2 changes to r1
{
$err = '';
local *STDERR;
open STDERR, '>', \$err;
$output = output(
sub { pt_table_sync::main(@args, 'h=127.1,P=2901',
qw(--print --execute --chunk-size 2),
qw(--conflict-column ts --conflict-comparison newest)) }
);
}
like(
$err,
qr/Cannot resolve conflict WHERE `id`='8': `ts` values are the same/,
'Warned again that id=8 differs but has same ts'
);
my $all_synced = [
['1','abc','1','2010-02-01 05:45:30'],
['2','def','2','2010-01-31 06:11:11'],
['3','ghi','5','2010-02-01 09:17:52'],
['4','jkl','6','2010-02-01 10:11:33'],
['5',undef,'0','2010-02-02 05:10:00'],
['6','p','4','2010-01-31 10:17:00'],
['7','qrs','5','2010-02-01 10:11:11'],
['8','tuv','6','2010-01-31 10:17:20'],
['9','wxy','7','2010-02-01 10:17:00'],
['10','rdy','5','2010-02-02 09:09:10'],
['11','?','0','2010-01-29 12:19:48'],
['12','','0','2010-02-01 13:17:19'],
['13','hmm','1','2010-02-02 12:17:31'],
['14',undef,'0','2010-02-02 13:00:00'],
['15','gtg','7','2010-02-02 06:01:08'],
['16',undef,'0','2010-02-02 13:20:00'],
['17','good','1','2010-02-02 21:38:03'],
['20','new','100','2010-02-01 04:15:36']
];
$res = $c1_dbh->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
$all_synced,
'c1 data synced 2nd pass'
);
$res = $r1_dbh->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
$all_synced,
'r1 data synced 2nd pass'
);
$res = $r2_dbh->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
[
['1','abc','1','2010-02-01 05:45:30'],
['2','def','2','2010-01-31 06:11:11'],
['3','ghi','5','2010-02-01 09:17:52'],
['4','jkl','6','2010-02-01 10:11:33'],
['5',undef,'0','2010-02-02 05:10:00'],
['6','p','4','2010-01-31 10:17:00'],
['7','qrs','5','2010-02-01 10:11:11'],
# Identical to $all_synced except this conflicted row:
['8','TUV','6','2010-01-31 10:17:20'],
['9','wxy','7','2010-02-01 10:17:00'],
['10','rdy','5','2010-02-02 09:09:10'],
['11','?','0','2010-01-29 12:19:48'],
['12','','0','2010-02-01 13:17:19'],
['13','hmm','1','2010-02-02 12:17:31'],
['14',undef,'0','2010-02-02 13:00:00'],
['15','gtg','7','2010-02-02 06:01:08'],
['16',undef,'0','2010-02-02 13:20:00'],
['17','good','1','2010-02-02 21:38:03'],
['20','new','100','2010-02-01 04:15:36'],
],
'r2 data synced 2nd pass'
);
}
# #############################################################################
# Done.
# #############################################################################
diag(`$trunk/sandbox/stop-sandbox 2901 >/dev/null`);
diag(`$trunk/sandbox/stop-sandbox 2900 >/dev/null`);
$sb->wipe_clean($c1_dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
exit;
Reference in New Issue View Git Blame Copy Permalink