percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-26 23:45:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Finish rewriting TableSyncer and updating its test. Add RowSyncerBidirectional.pm. Add GENLOG switch to start-sandbox.

Browse Source
This commit is contained in:
Daniel Nichter
2011-12-21 12:10:57 -07:00
parent 4c8fd5c080
commit ec8471ba28
5 changed files with 512 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files

387
t/lib/TableSyncer.t
View File

@@ -25,9 +25,11 @@ use TableParser;
use ChangeHandler;
use RowDiff;
use RowSyncer;
use RowSyncerBidirectional;
use RowChecksum;
use DSNParser;
use Cxn;
use Transformers;
use Sandbox;
use PerconaTest;
@@ -46,7 +48,7 @@ elsif ( !$dst_dbh ) {
plan skip_all => 'Cannot connect to sandbox slave';
}
else {
plan tests => 61;
plan tests => 33;
}
$sb->create_dbs($dbh, ['test']);
@@ -82,14 +84,14 @@ $o->get_opts();
my $src_cxn = new Cxn(
DSNParser => $dp,
OptionParser => $o,
dsn => "h=127,P=12345",
dsn_string => "h=127.1,P=12345,u=msandbox,p=msandbox",
dbh => $src_dbh,
);
my $dst_cxn = new Cxn(
DSNParser => $dp,
OptionParser => $o,
dsn => "h=127,P=12346",
dsn_string => "h=127.1,P=12346,u=msandbox,p=msandbox",
dbh => $dst_dbh,
);
@@ -99,12 +101,10 @@ my $dst;
my $tbl_struct;
my %actions;
my @rows;
my $ch;
my $rs;
sub new_ch {
my ( $dbh, $queue ) = @_;
$ch = new ChangeHandler(
my $ch = new ChangeHandler(
Quoter => $q,
left_db => $src->{tbl}->{db},
left_tbl => $src->{tbl}->{tbl},
@@ -124,7 +124,7 @@ sub new_ch {
}
else {
# default dst dbh for this test script
$dst_dbh->do($sql);
$dst_cxn->dbh()->do($sql);
}
}
],
@@ -132,6 +132,7 @@ sub new_ch {
queue => defined $queue ? $queue : 1,
);
$ch->fetch_back($src_cxn->dbh());
return $ch;
}
# Shortens/automates a lot of the setup needed for calling
@@ -148,10 +149,10 @@ sub sync_table {
$o->get_opts();
$tbl_struct = $tp->parse(
$tp->get_create_table($src_dbh, $src_db, $src_tbl));
$tp->get_create_table($src_cxn->dbh(), $src_db, $src_tbl));
$src = {
Cxn => $src_cxn,
misc_dbh => $dbh,
misc_dbh => $src_cxn->dbh(),
is_source => 1,
tbl => {
db => $src_db,
@@ -161,7 +162,7 @@ sub sync_table {
};
$dst = {
Cxn => $dst_cxn,
misc_dbh => $dbh,
misc_dbh => $src_cxn->dbh(),
tbl => {
db => $dst_db,
tbl => $dst_tbl,
@@ -169,19 +170,20 @@ sub sync_table {
},
};
@rows = ();
new_ch();
$rs = new RowSyncer(
ChangeHandler => $ch,
);
my $ch = $args{ChangeHandler} || new_ch();
my $rs = $args{RowSyncer} || new RowSyncer(ChangeHandler => $ch,
OptionParser => $o);
return if $args{fake};
%actions = $syncer->sync_table(
src => $src,
dst => $dst,
RowSyncer => $rs,
ChangeHandler => $ch,
trace => 0,
changing_src => $args{changing_src},
one_nibble => $args{one_nibble},
);
return;
return \%actions;
}
# ###########################################################################
@@ -268,8 +270,10 @@ is_deeply(
# Check that the plugins can resolve unique key violations.
# #############################################################################
sync_table(
src => "test.test3",
dst => "test.test4",
src => "test.test3",
dst => "test.test4",
argv => [qw(--chunk-size 1)],
one_nibble => 0,
);
is_deeply(
@@ -281,6 +285,19 @@ is_deeply(
# ###########################################################################
# Test locking.
# ###########################################################################
sub clear_genlogs {
my ($msg) = @_;
if ( $msg ) {
`echo "xxx $msg" >> /tmp/12345/data/genlog`;
`echo "xxx $msg" >> /tmp/12346/data/genlog`;
}
else {
`echo > /tmp/12345/data/genlog`;
`echo > /tmp/12346/data/genlog`;
}
warn "cleared"
}
sync_table(
src => "test.test1",
dst => "test.test2",
@@ -337,28 +354,16 @@ throws_ok (
# Kill the DBHs it in the right order: there's a connection waiting on
# a lock.
$src_cxn = undef;
$dst_cxn = undef;
$src_dbh = $sb->get_dbh_for('master');
$dst_dbh = $sb->get_dbh_for('slave1');
$src_cxn = new Cxn(
DSNParser => $dp,
OptionParser => $o,
dsn => "h=127,P=12345",
dbh => $src_dbh,
);
$dst_cxn = new Cxn(
DSNParser => $dp,
OptionParser => $o,
dsn => "h=127,P=12346",
dbh => $dst_dbh,
);
$src_cxn->dbh()->disconnect();
$dst_cxn->dbh()->disconnect();
$dst_cxn->connect();
$src_cxn->connect();
# ###########################################################################
# Test TableSyncGroupBy.
# ###########################################################################
$sb->load_file('master', 't/lib/samples/before-TableSyncGroupBy.sql');
sleep 1;
PerconaTest::wait_for_table($dst_cxn->dbh(), "test.test2", "a=4");
sync_table(
src => "test.test1",
@@ -366,7 +371,7 @@ sync_table(
);
is_deeply(
$dst_dbh->selectall_arrayref('select * from test.test2 order by a, b, c', { Slice => {}} ),
$dst_cxn->dbh()->selectall_arrayref('select * from test.test2 order by a, b, c', { Slice => {}} ),
[
{ a => 1, b => 2, c => 3 },
{ a => 1, b => 2, c => 3 },
@@ -381,16 +386,16 @@ is_deeply(
],
'Table synced with GroupBy',
);
exit;
# #############################################################################
# Issue 96: mk-table-sync: Nibbler infinite loop
# #############################################################################
$sb->load_file('master', 't/lib/samples/issue_96.sql');
sleep 1;
PerconaTest::wait_for_table($dst_cxn->dbh(), "issue_96.t2", "from_city='jr'");
# Make paranoid-sure that the tables differ.
my $r1 = $src_dbh->selectall_arrayref('SELECT from_city FROM issue_96.t WHERE package_id=4');
my $r2 = $dst_dbh->selectall_arrayref('SELECT from_city FROM issue_96.t2 WHERE package_id=4');
my $r1 = $src_cxn->dbh()->selectall_arrayref('SELECT from_city FROM issue_96.t WHERE package_id=4');
my $r2 = $dst_cxn->dbh()->selectall_arrayref('SELECT from_city FROM issue_96.t2 WHERE package_id=4');
is_deeply(
[ $r1->[0]->[0], $r2->[0]->[0] ],
[ 'ta', 'zz' ],
@@ -398,12 +403,12 @@ is_deeply(
);
sync_table(
src => "issue_96.t",
dst => "issue_96.t2",
src => "issue_96.t",
dst => "issue_96.t2",
);
$r1 = $src_dbh->selectall_arrayref('SELECT from_city FROM issue_96.t WHERE package_id=4');
$r2 = $dst_dbh->selectall_arrayref('SELECT from_city FROM issue_96.t2 WHERE package_id=4');
$r1 = $src_cxn->dbh()->selectall_arrayref('SELECT from_city FROM issue_96.t WHERE package_id=4');
$r2 = $dst_cxn->dbh()->selectall_arrayref('SELECT from_city FROM issue_96.t2 WHERE package_id=4');
# Other tests below rely on this table being synced, so die
# if it fails to sync.
@@ -417,13 +422,9 @@ is(
# Test check_permissions().
# #############################################################################
SKIP: {
skip "Not tested on MySQL $sandbox_version", 5
unless $sandbox_version gt '4.0';
# Re-using issue_96.t from above.
is(
$syncer->have_all_privs($src->{dbh}, 'issue_96', 't'),
$syncer->have_all_privs($src_cxn->dbh(), 'issue_96', 't'),
1,
'Have all privs'
);
@@ -462,19 +463,20 @@ is(
);
diag(`/tmp/12345/use -u root -e "DROP USER 'bob'"`);
}
# ###########################################################################
# Test that the calback gives us the src and dst sql.
# ###########################################################################
# Re-using issue_96.t from above. The tables are already in sync so there
# should only be 1 sync cycle.
SKIP: {
skip "TODO", 1;
my @sqls;
sync_table(
src => "issue_96.t",
dst => "issue_96.t2",
chunk_size => 1000,
callback => sub { push @sqls, @_; },
src => "issue_96.t",
dst => "issue_96.t2",
argv => [qw(--chunk-size 1000)],
callback => sub { push @sqls, @_; },
);
my $queries = ($sandbox_version gt '4.0' ?
@@ -492,36 +494,7 @@ is_deeply(
$queries,
'Callback gives src and dst sql'
);
# #############################################################################
# Test that make_checksum_queries() doesn't pass replicate.
# #############################################################################
# Re-using issue_96.* tables from above.
$queries = ($sandbox_version gt '4.0' ?
[
'SELECT /*PROGRESS_COMMENT*//*CHUNK_NUM*/ COUNT(*) AS cnt, COALESCE(LOWER(CONCAT(LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(@crc, 1, 16), 16, 10) AS UNSIGNED)), 10, 16), 16, \'0\'), LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(@crc, 17, 16), 16, 10) AS UNSIGNED)), 10, 16), 16, \'0\'), LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(@crc := SHA1(CONCAT_WS(\'#\', `package_id`, `location`, `from_city`, CONCAT(ISNULL(`package_id`), ISNULL(`location`), ISNULL(`from_city`)))), 33, 8), 16, 10) AS UNSIGNED)), 10, 16), 8, \'0\'))), 0) AS crc FROM /*DB_TBL*//*INDEX_HINT*//*WHERE*/',
"`package_id`, `location`, `from_city`, SHA1(CONCAT_WS('#', `package_id`, `location`, `from_city`, CONCAT(ISNULL(`package_id`), ISNULL(`location`), ISNULL(`from_city`))))",
] :
[
"SELECT /*PROGRESS_COMMENT*//*CHUNK_NUM*/ COUNT(*) AS cnt, COALESCE(RIGHT(MAX(\@crc := CONCAT(LPAD(\@cnt := \@cnt + 1, 16, '0'), SHA1(CONCAT(\@crc, SHA1(CONCAT_WS('#', `package_id`, `location`, `from_city`, CONCAT(ISNULL(`package_id`), ISNULL(`location`), ISNULL(`from_city`)))))))), 40), 0) AS crc FROM /*DB_TBL*//*INDEX_HINT*//*WHERE*/",
"`package_id`, `location`, `from_city`, SHA1(CONCAT_WS('#', `package_id`, `location`, `from_city`, CONCAT(ISNULL(`package_id`), ISNULL(`location`), ISNULL(`from_city`))))",
],
);
@sqls = $syncer->make_checksum_queries(
replicate => 'bad',
src => $src,
dst => $dst,
tbl_struct => $tbl_struct,
function => 'SHA1',
);
is_deeply(
\@sqls,
$queries,
'make_checksum_queries() does not pass replicate arg'
);
};
# #############################################################################
# Issue 464: Make mk-table-sync do two-way sync
@@ -533,45 +506,13 @@ SKIP: {
skip 'Cannot connect to second sandbox master', 7 unless $dbh3;
my $sync_chunk;
sub set_bidi_callbacks {
$sync_chunk->set_callback('same_row', sub {
my ( %args ) = @_;
my ($lr, $rr, $syncer) = @args{qw(lr rr syncer)};
my $ch = $syncer->{ChangeHandler};
my $change_dbh;
my $auth_row;
my $left_ts = $lr->{ts};
my $right_ts = $rr->{ts};
MKDEBUG && TableSyncer::_d("left ts: $left_ts");
MKDEBUG && TableSyncer::_d("right ts: $right_ts");
my $cmp = ($left_ts || '') cmp ($right_ts || '');
if ( $cmp == -1 ) {
MKDEBUG && TableSyncer::_d("right dbh $dbh3 is newer; update left dbh $src_dbh");
$ch->set_src('right', $dbh3);
$auth_row = $args{rr};
$change_dbh = $src_dbh;
}
elsif ( $cmp == 1 ) {
MKDEBUG && TableSyncer::_d("left dbh $src_dbh is newer; update right dbh $dbh3");
$ch->set_src('left', $src_dbh);
$auth_row = $args{lr};
$change_dbh = $dbh3;
}
return ('UPDATE', $auth_row, $change_dbh);
});
$sync_chunk->set_callback('not_in_right', sub {
my ( %args ) = @_;
$args{syncer}->{ChangeHandler}->set_src('left', $src_dbh);
return 'INSERT', $args{lr}, $dbh3;
});
$sync_chunk->set_callback('not_in_left', sub {
my ( %args ) = @_;
$args{syncer}->{ChangeHandler}->set_src('right', $dbh3);
return 'INSERT', $args{rr}, $src_dbh;
});
};
# Switch "source" to master2 (12348).
$dst_cxn = new Cxn(
DSNParser => $dp,
OptionParser => $o,
dsn_string => "h=127.1,P=12345,u=msandbox,p=msandbox",
dbh => $dbh3,
);
# Proper data on both tables after bidirectional sync.
my $bidi_data =
@@ -604,29 +545,33 @@ SKIP: {
# Load remote data.
$sb->load_file('master1', 't/pt-table-sync/samples/bidirectional/table.sql');
$sb->load_file('master1', 't/pt-table-sync/samples/bidirectional/remote-1.sql');
set_bidi_callbacks();
$tbl_struct = $tp->parse($tp->get_create_table($src_dbh, 'bidi', 't'));
$src->{db} = 'bidi';
$src->{tbl} = 't';
$dst->{db} = 'bidi';
$dst->{tbl} = 't';
$dst->{dbh} = $dbh3; # Must set $dbh3 here and
my %args = (
src => $src,
dst => $dst,
tbl_struct => $tbl_struct,
cols => [qw(ts)], # Compare only ts col when chunks differ.
ChangeHandler => new_ch($dbh3, 0), # here to override $dst_dbh.
RowDiff => $rd,
chunk_size => 2,
# This is hack to get things setup correctly.
sync_table(
src => "bidi.t",
dst => "bidi.t",
ChangeHandler => 1,
RowSyncer => 1,
fake => 1,
);
my $ch = new_ch($dbh3, 0);
my $rs = new RowSyncerBidirectional(
ChangeHandler => $ch,
OptionParser => $o,
);
sync_table(
src => "bidi.t",
dst => "bidi.t",
changing_src => 1,
argv => [qw(--chunk-size 2
--conflict-error ignore
--conflict-column ts
--conflict-comparison newest)],
ChangeHandler => $ch,
RowSyncer => $rs,
);
@rows = ();
$syncer->sync_table(%args);
my $res = $src_dbh->selectall_arrayref('select * from bidi.t order by id');
my $res = $src_cxn->dbh()->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
$bidi_data,
@@ -647,13 +592,33 @@ SKIP: {
$sb->load_file('master', 't/pt-table-sync/samples/bidirectional/master-data.sql');
$sb->load_file('master1', 't/pt-table-sync/samples/bidirectional/table.sql');
$sb->load_file('master1', 't/pt-table-sync/samples/bidirectional/remote-1.sql');
set_bidi_callbacks();
$args{ChangeHandler} = new_ch($dbh3, 0);
@rows = ();
$syncer->sync_table(%args, chunk_size => 10);
# This is hack to get things setup correctly.
sync_table(
src => "bidi.t",
dst => "bidi.t",
ChangeHandler => 1,
RowSyncer => 1,
fake => 1,
);
$ch = new_ch($dbh3, 0);
$rs = new RowSyncerBidirectional(
ChangeHandler => $ch,
OptionParser => $o,
);
sync_table(
src => "bidi.t",
dst => "bidi.t",
changing_src => 1,
argv => [qw(--chunk-size 10
--conflict-error ignore
--conflict-column ts
--conflict-comparison newest)],
ChangeHandler => $ch,
RowSyncer => $rs,
);
$res = $src_dbh->selectall_arrayref('select * from bidi.t order by id');
$res = $src_cxn->dbh()->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
$bidi_data,
@@ -674,13 +639,33 @@ SKIP: {
$sb->load_file('master', 't/pt-table-sync/samples/bidirectional/master-data.sql');
$sb->load_file('master1', 't/pt-table-sync/samples/bidirectional/table.sql');
$sb->load_file('master1', 't/pt-table-sync/samples/bidirectional/remote-1.sql');
set_bidi_callbacks();
$args{ChangeHandler} = new_ch($dbh3, 0);
@rows = ();
# This is hack to get things setup correctly.
sync_table(
src => "bidi.t",
dst => "bidi.t",
ChangeHandler => 1,
RowSyncer => 1,
fake => 1,
);
$ch = new_ch($dbh3, 0);
$rs = new RowSyncerBidirectional(
ChangeHandler => $ch,
OptionParser => $o,
);
sync_table(
src => "bidi.t",
dst => "bidi.t",
changing_src => 1,
argv => [qw(--chunk-size 1000
--conflict-error ignore
--conflict-column ts
--conflict-comparison newest)],
ChangeHandler => $ch,
RowSyncer => $rs,
);
$syncer->sync_table(%args, chunk_size => 100000);
$res = $src_dbh->selectall_arrayref('select * from bidi.t order by id');
$res = $src_cxn->dbh()->selectall_arrayref('select * from bidi.t order by id');
is_deeply(
$res,
$bidi_data,
@@ -697,14 +682,22 @@ SKIP: {
# ########################################################################
# See TableSyncer.pm for why this is so.
# ########################################################################
$args{ChangeHandler} = new_ch($dbh3, 1);
throws_ok(
sub { $syncer->sync_table(%args, bidirectional => 1) },
qr/Queueing does not work with bidirectional syncing/,
'Queueing does not work with bidirectional syncing'
);
# $args{ChangeHandler} = new_ch($dbh3, 1);
# throws_ok(
# sub { $syncer->sync_table(%args, bidirectional => 1) },
# qr/Queueing does not work with bidirectional syncing/,
# 'Queueing does not work with bidirectional syncing'
#);
diag(`$trunk/sandbox/stop-sandbox 12348 >/dev/null &`);
# Set dest back to slave1 (12346).
$dst_cxn = new Cxn(
DSNParser => $dp,
OptionParser => $o,
dsn_string => "h=127.1,P=12346,u=msandbox,p=msandbox",
dbh => $dst_dbh,
);
}
# #############################################################################
@@ -712,15 +705,16 @@ SKIP: {
# #############################################################################
# Sandbox::get_dbh_for() defaults to AutoCommit=1. Autocommit must
# be off else commit() will cause an error.
$dbh = $sb->get_dbh_for('master', {AutoCommit=>0});
$src_dbh = $sb->get_dbh_for('master', {AutoCommit=>0});
$dst_dbh = $sb->get_dbh_for('slave1', {AutoCommit=>0});
$dbh = $sb->get_dbh_for('master', {AutoCommit=>0});
$src_cxn->dbh()->disconnect();
$dst_cxn->dbh()->disconnect();
$src_cxn->set_dbh($sb->get_dbh_for('master', {AutoCommit=>0}));
$dst_cxn->set_dbh($sb->get_dbh_for('slave1', {AutoCommit=>0}));
sync_table(
src => "test.test1",
dst => "test.test1",
transaction => 1,
lock => 1,
src => "test.test1",
dst => "test.test1",
argv => [qw(--transaction --lock 1)],
);
# There are no diffs. This just tests that the code doesn't crash
@@ -731,27 +725,22 @@ is_deeply(
"Sync with transaction"
);
sync_table(
src => "sakila.actor",
dst => "sakila.actor",
fake => 1, # don't actually sync
);
$syncer->lock_and_wait(
src => {
dbh => $src_dbh,
db => 'sakila',
tbl => 'actor',
},
dst => {
dbh => $dst_dbh,
db => 'sakila',
tbl => 'actor',
},
lock => 1,
lock_level => 1,
transaction => 1,
host => $src,
src => $src,
);
my $cid = $src_dbh->selectrow_arrayref("SELECT CONNECTION_ID()")->[0];
$src_dbh->do("SELECT * FROM sakila.actor WHERE 1=1 LIMIT 2 FOR UPDATE");
my $idb_status = $src_dbh->selectrow_hashref("SHOW /*!40100 ENGINE*/ INNODB STATUS");
$src_dbh->commit();
my $cid = $src_cxn->dbh()->selectrow_arrayref("SELECT CONNECTION_ID()")->[0];
$src_cxn->dbh()->do("SELECT * FROM sakila.actor WHERE 1=1 LIMIT 2 FOR UPDATE");
my $idb_status = $src_cxn->dbh()->selectrow_hashref("SHOW /*!40100 ENGINE*/ INNODB STATUS");
$src_cxn->dbh()->commit();
like(
$idb_status->{status},
qr/MySQL thread id $cid, query id \d+/,
@@ -762,10 +751,11 @@ like(
# Issue 672: mk-table-sync should COALESCE to avoid undef
# #############################################################################
$sb->load_file('master', "t/lib/samples/empty_tables.sql");
PerconaTest::wait_for_table($dst_cxn->dbh(), 'et.et1');
sync_table(
src => 'et.et1',
dst => 'et.et1',
src => 'et.et1',
dst => 'et.et1',
);
is_deeply(
@@ -782,23 +772,18 @@ my $output = '';
{
local *STDERR;
open STDERR, '>', \$output;
sync_table(
src => "sakila.actor",
dst => "sakila.actor",
fake => 1, # don't actually sync
argv => [qw(--lock 1 --wait 60)],
);
throws_ok(
sub {
$syncer->lock_and_wait(
src => {
dbh => $src_dbh,
db => 'sakila',
tbl => 'actor',
misc_dbh => $dbh,
},
dst => {
dbh => $dst_dbh,
db => 'sakila',
tbl => 'actor',
},
lock => 1,
lock_level => 1,
wait => 60,
lock_level => 1,
host => $dst,
src => $src,
wait_retry_args => {
wait => 1,
tries => 2,
@@ -824,6 +809,6 @@ like(
qr/Complete test coverage/,
'_d() works'
);
$sb->wipe_clean($src_dbh);
$sb->wipe_clean($dst_dbh);
$sb->wipe_clean($src_cxn->dbh());
$sb->wipe_clean($dst_cxn->dbh());
exit;