percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-07 21:09:14 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
e1548d953f8aa4fef895fed9b59885982fb99842
percona-toolkit/t/pt-table-checksum/pxc.t
Frank Cizmich 1ccd6e528a changed version, added changelog, fixed pxc.t
2015-01-22 10:51:26 -02:00

738 lines
22 KiB
Perl
Raw Blame History

#!/usr/bin/env perl
BEGIN {
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
};
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Test::More;
use Data::Dumper;
# Hostnames make testing less accurate. Tests need to see
# that such-and-such happened on specific slave hosts, but
# the sandbox servers are all on one host so all slaves have
# the same hostname.
$ENV{PERCONA_TOOLKIT_TEST_USE_DSN_NAMES} = 1;
use PerconaTest;
use Sandbox;
require "$trunk/bin/pt-table-checksum";
my $ip = qr/\Q127.1\E|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/;
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $node1 = $sb->get_dbh_for('node1');
my $node2 = $sb->get_dbh_for('node2');
my $node3 = $sb->get_dbh_for('node3');
if ( !$node1 ) {
plan skip_all => 'Cannot connect to cluster node1';
}
elsif ( !$node2 ) {
plan skip_all => 'Cannot connect to cluster node2';
}
elsif ( !$node3 ) {
plan skip_all => 'Cannot connect to cluster node3';
}
elsif ( !$sb->is_cluster_mode ) {
plan skip_all => "PXC tests";
}
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
# so we need to specify --set-vars innodb_lock_wait_timeout=3 else the tool will die.
my $node1_dsn = $sb->dsn_for('node1');
my @args = ($node1_dsn, qw(--set-vars innodb_lock_wait_timeout=3));
my $output;
my $exit_status;
my $sample = "t/pt-table-checksum/samples/";
# #############################################################################
# pt-table-checksum v2.1.4 doesn't detect diffs on Percona XtraDB Cluster nodes
# https://bugs.launchpad.net/percona-toolkit/+bug/1062563
# #############################################################################
# #############################################################################
# Check just a cluster
# #############################################################################
# This DSN table has node2 and node3 (12346 and 12347) but not node1 (12345)
# because it was originally created for traditional setups which require only
# slave DSNs, but the DSN table for a PXC setup can/should contain DSNs for
# all nodes so the user can run pxc on any node and find all the others.
$sb->load_file('node1', "$sample/dsn-table.sql");
$node1->do(qq/INSERT INTO dsns.dsns VALUES (1, 1, '$node1_dsn')/);
# First a little test to make sure the tool detects and bails out
# if no other cluster nodes are detected, in which case the user
# probably didn't specifying --recursion-method dsn.
$output = output(
sub { pt_table_checksum::main(@args) },
stderr => 1,
);
like(
$output,
qr/h=127(?:\Q.0.0\E)?.1,P=12345 is a cluster node but no other nodes/,
"Dies if no other nodes are found"
);
($output, $exit_status) = full_output(
sub { pt_table_checksum::main(@args, '--recursion-method', 'none') },
stderr => 1,
);
ok (
$output !~ qr/no other nodes or regular replicas were found/i && !$exit_status,
"checksums even if --recursion-method=none - issue 1373937"
);
for my $args (
["using recusion-method=dsn", '--recursion-method', "dsn=$node1_dsn,D=dsns,t=dsns"],
["using recursion-method=cluster", '--recursion-method', 'cluster']
)
{
my $test = shift @$args;
$output = output(
sub { pt_table_checksum::main(@args,
@$args)
},
stderr => 1,
);
is(
PerconaTest::count_checksum_results($output, 'errors'),
0,
"No diffs: no errors ($test)"
);
is(
PerconaTest::count_checksum_results($output, 'skipped'),
0,
"No diffs: no skips ($test)"
);
is(
PerconaTest::count_checksum_results($output, 'diffs'),
0,
"No diffs: no diffs ($test)"
);
}
# Now really test checksumming a cluster. To create a diff we have to disable
# the binlog. Although PXC doesn't need or use the binlog to communicate
# (it has its own broadcast-based protocol implemented via the Galera lib)
# it still respects sql_log_bin, so we can make a change on one node without
# affecting the others.
$sb->load_file('node1', "$sample/a-z.sql");
$node2->do("set sql_log_bin=0");
$node2->do("update test.t set c='zebra' where c='z'");
$node2->do("set sql_log_bin=1");
my ($row) = $node2->selectrow_array("select c from test.t order by c desc limit 1");
is(
$row,
"zebra",
"Node2 is changed"
);
($row) = $node1->selectrow_array("select c from test.t order by c desc limit 1");
is(
$row,
"z",
"Node1 not changed"
);
($row) = $node3->selectrow_array("select c from test.t order by c desc limit 1");
is(
$row,
"z",
"Node3 not changed"
);
sub test_recursion_methods {
my $same_ids = shift;
my ($orig_id_1, $orig_id_2, $orig_id_3);
my ($orig_ia_1, $orig_ia_2, $orig_ia_3);
if ($same_ids) {
# save original values
my $sql = 'SELECT @@server_id';
($orig_id_1) = $node1->selectrow_array($sql);
($orig_id_2) = $node2->selectrow_array($sql);
($orig_id_3) = $node3->selectrow_array($sql);
# set server_id value to 1 on all nodes
$sql = 'SET GLOBAL server_id = 1';
$node1->do($sql);
$node2->do($sql);
$node3->do($sql);
# since we're testing server id issues, set wsrep_node_incoming_address=AUTO ( https://bugs.launchpad.net/percona-toolkit/+bug/1399789 )
# save original values
$sql = 'SELECT @@wsrep_node_incoming_address';
($orig_ia_1) = $node1->selectrow_array($sql);
($orig_ia_2) = $node2->selectrow_array($sql);
($orig_ia_3) = $node3->selectrow_array($sql);
# set wsrep_node_incoming_address value to AUTO on all nodes
$sql = 'SET GLOBAL wsrep_node_incoming_address = AUTO';
$node1->do($sql);
$node2->do($sql);
$node3->do($sql);
}
for my $args (
["using recusion-method=dsn", '--recursion-method', "dsn=$node1_dsn,D=dsns,t=dsns"],
["using recursion-method=cluster", '--recursion-method', 'cluster']
)
{
my $test = shift @$args;
$test = $same_ids ? $test.' - Nodes with different ids' : $test.' - Nodes with same ids';
$output = output(
sub { pt_table_checksum::main(@args,
@$args)
},
stderr => 1,
);
is(
PerconaTest::count_checksum_results($output, 'errors'),
0,
"1 diff: no errors ($test)"
);
is(
PerconaTest::count_checksum_results($output, 'skipped'),
0,
"1 diff: no skips ($test)"
);
is(
PerconaTest::count_checksum_results($output, 'diffs'),
1,
"1 diff: 1 diff ($test)"
) or diag($output);
# 11-17T13:02:54 0 1 26 1 0 0.021 test.t
like(
$output,
qr/^\S+\s+ # ts
0\s+ # errors
1\s+ # diffs
26\s+ # rows
\d+\s+ # chunks
0\s+ # skipped
\S+\s+ # time
test.t$ # table
/xm,
"1 diff: it's in test.t ($test)"
);
}
if ($same_ids) {
# reset server_id's to original values
$node1->do("SET GLOBAL server_id = $orig_id_1");
$node2->do("SET GLOBAL server_id = $orig_id_2");
$node3->do("SET GLOBAL server_id = $orig_id_3");
# reset node wsrep_node_incoming_address to original values
$node1->do("SET GLOBAL wsrep_node_incoming_address = '$orig_ia_1'");
$node2->do("SET GLOBAL wsrep_node_incoming_address = '$orig_ia_2'");
$node3->do("SET GLOBAL wsrep_node_incoming_address = '$orig_ia_3'");
}
}
# test recursion methods
test_recursion_methods(0);
# test recursion methods when all nodes have the same id
test_recursion_methods(1);
# #############################################################################
# cluster, node1 -> slave, run on node1
# #############################################################################
my ($slave_dbh, $slave_dsn) = $sb->start_sandbox(
server => 'cslave1',
type => 'slave',
master => 'node1',
env => q/FORK="pxc" BINLOG_FORMAT="ROW"/,
);
# Add the slave to the DSN table.
$node1->do(qq/INSERT INTO dsns.dsns VALUES (4, 3, '$slave_dsn')/);
# Fix what we changed earlier on node2 so the cluster is consistent.
$node2->do("set sql_log_bin=0");
$node2->do("update test.t set c='z' where c='zebra'");
$node2->do("set sql_log_bin=1");
# Wait for the slave to apply the binlogs from node1 (its master).
# Then change it so it's not consistent.
PerconaTest::wait_for_table($slave_dbh, 'test.t');
$sb->wait_for_slaves(master => 'node1', slave => 'cslave1');
$slave_dbh->do("update test.t set c='zebra' where c='z'");
# Another quick test first: the tool should complain about the slave's
# binlog format but only the slave's, not the cluster nodes:
# https://bugs.launchpad.net/percona-toolkit/+bug/1080385
# Cluster nodes default to ROW format because that's what Galeara
# works best with, even though it doesn't really use binlogs.
for my $args (
["using recusion-method=dsn", '--recursion-method', "dsn=$node1_dsn,D=dsns,t=dsns"],
["using recursion-method=cluster,hosts", '--recursion-method', 'cluster,hosts']
)
{
my $test = shift @$args;
# Wait for the slave to apply the binlogs from node1 (its master).
# Then change it so it's not consistent.
PerconaTest::wait_for_table($slave_dbh, 'test.t');
$sb->wait_for_slaves(master => 'node1', slave => 'cslave1');
$slave_dbh->do("update test.t set c='zebra' where c='z'");
$output = output(
sub { pt_table_checksum::main(@args,
@$args)
},
stderr => 1,
);
like(
$output,
qr/replica h=127(?:\Q.0.0\E)?\.1,P=12348 has binlog_format ROW/i,
"--check-binlog-format warns about slave's binlog format ($test)"
);
# Now really test that diffs on the slave are detected.
$output = output(
sub { pt_table_checksum::main(@args,
@$args,
qw(--no-check-binlog-format)),
},
stderr => 1,
);
is(
PerconaTest::count_checksum_results($output, 'diffs'),
1,
"Detects diffs on slave of cluster node1 ($test)"
) or diag($output);
}
$slave_dbh->disconnect;
$sb->stop_sandbox('cslave1');
# #############################################################################
# cluster, node2 -> slave, run on node1
#
# Does not work because we only set binglog_format=STATEMENT on node1 which
# does not affect other nodes, so node2 gets checksum queries in STATEMENT
# format, executes them, but then logs the results in ROW format (since ROW
# format is the default for cluster nodes) which doesn't work on the slave
# (i.e. the slave doesn't execute the query). So any diffs on the slave are
# not detected.
# #############################################################################
($slave_dbh, $slave_dsn) = $sb->start_sandbox(
server => 'cslave1',
type => 'slave',
master => 'node2',
env => q/FORK="pxc" BINLOG_FORMAT="ROW"/,
);
# Wait for the slave to apply the binlogs from node2 (its master).
# Then change it so it's not consistent.
PerconaTest::wait_for_table($slave_dbh, 'test.t');
$sb->wait_for_slaves(master => 'node1', slave => 'cslave1');
$slave_dbh->do("update test.t set c='zebra' where c='z'");
($row) = $slave_dbh->selectrow_array("select c from test.t order by c desc limit 1");
is(
$row,
"zebra",
"Slave is changed"
);
for my $args (
["using recusion-method=dsn", '--recursion-method', "dsn=$node1_dsn,D=dsns,t=dsns"],
["using recursion-method=cluster,hosts", '--recursion-method', 'cluster,hosts']
)
{
my $test = shift @$args;
$output = output(
sub { pt_table_checksum::main(@args,
@$args,
qw(--no-check-binlog-format -d test)),
},
stderr => 1,
);
is(
PerconaTest::count_checksum_results($output, 'diffs'),
0,
"Limitation: does not detect diffs on slave of cluster node2 ($test)"
) or diag($output);
}
$slave_dbh->disconnect;
$sb->stop_sandbox('cslave1');
# Restore the original DSN table.
$node1->do(qq/DELETE FROM dsns.dsns WHERE id=4/);
# #############################################################################
# master -> node1 in cluster, run on master
# #############################################################################
# CAREFUL: The master and the cluster are different, so don't do stuff
# on the master that will conflict with stuff already done on the cluster.
# And since we're using RBR, we have to do a lot of stuff on the master
# again, manually, because REPLACE and INSERT IGNORE don't work in RBR
# like they do SBR.
my ($master_dbh, $master_dsn) = $sb->start_sandbox(
server => 'cmaster',
type => 'master',
env => q/FORK="pxc" BINLOG_FORMAT="ROW"/,
);
# Since master is new, node1 shouldn't have binlog to replay.
$sb->set_as_slave('node1', 'cmaster');
# We have to load a-z-cluster.sql else the pk id won'ts match because nodes use
# auto-inc offsets but the master doesn't.
$sb->load_file('cmaster', "$sample/a-z-cluster.sql", undef, no_wait => 1);
# Do this stuff manually and only on the master because node1/the cluster
# already has it, and due to RBR, we can't do it other ways.
$master_dbh->do("SET sql_log_bin=0");
# This DSN table does not include 12345 (node1/slave) intentionally,
# so a later test can auto-find 12345 then warn "Diffs will only be
# detected if the cluster is consistent with h=127.1,P=12345...".
$master_dbh->do("CREATE DATABASE dsns");
$master_dbh->do("CREATE TABLE dsns.dsns (
id int auto_increment primary key,
parent_id int default null,
dsn varchar(255) not null
)");
$master_dbh->do("INSERT INTO dsns.dsns VALUES
(2, 1, 'h=127.1,P=12346,u=msandbox,p=msandbox'),
(3, 2, 'h=127.1,P=12347,u=msandbox,p=msandbox')");
$master_dbh->do("INSERT INTO percona_test.sentinel (id, ping) VALUES (1, '')");
$master_dbh->do("SET sql_log_bin=1");
$sb->wait_for_slaves(master => 'cmaster', slave => 'node1');
# Notice: no --recursion-method=dsn yet. Since node1 is a traditional slave
# of the master, ptc should auto-detect it, which we'll test later by making
# the slave differ.
$output = output(
sub { pt_table_checksum::main($master_dsn,
qw(-d test))
},
stderr => 1,
);
is(
PerconaTest::count_checksum_results($output, 'errors'),
0,
"master->cluster no diffs: no errors"
);
is(
PerconaTest::count_checksum_results($output, 'skipped'),
0,
"master->cluster no diffs: no skips"
);
is(
PerconaTest::count_checksum_results($output, 'diffs'),
0,
"master->cluster no diffs: no diffs"
) or diag($output);
# Make a diff on node1. If ptc is really auto-detecting node1, then it
# should report this diff.
$node1->do("set sql_log_bin=0");
$node1->do("update test.t set c='zebra' where c='z'");
$node1->do("set sql_log_bin=1");
$output = output(
sub { pt_table_checksum::main($master_dsn,
qw(-d test))
},
stderr => 1,
);
is(
PerconaTest::count_checksum_results($output, 'errors'),
0,
"master->cluster 1 diff: no errors"
);
is(
PerconaTest::count_checksum_results($output, 'skipped'),
0,
"master->cluster 1 diff: no skips"
);
is(
PerconaTest::count_checksum_results($output, 'diffs'),
1,
"master->cluster 1 diff: 1 diff"
) or diag($output);
# 11-17T13:02:54 0 1 26 1 0 0.021 test.t
like(
$output,
qr/^\S+\s+ # ts
0\s+ # errors
1\s+ # diffs
26\s+ # rows
\d+\s+ # chunks
0\s+ # skipped
\S+\s+ # time
test.t$ # table
/xm,
"master->cluster 1 diff: it's in test.t"
);
# Use the DSN table to check for diffs on node2 and node3. This works
# because the diff is on node1 and node1 is the direct slave of the master,
# so the checksum query will replicate from the master in STATEMENT format,
# node1 will execute it, find the diff, then broadcast that result to all
# other nodes. -- Remember: the DSN table on the master has node2 and node3.
$output = output(
sub { pt_table_checksum::main($master_dsn,
'--recursion-method', "dsn=$master_dsn,D=dsns,t=dsns",
qw(-d test))
},
stderr => 1,
);
is(
PerconaTest::count_checksum_results($output, 'errors'),
0,
"...check other nodes: no errors"
);
is(
PerconaTest::count_checksum_results($output, 'skipped'),
0,
"...check other nodes: no skips"
);
is(
PerconaTest::count_checksum_results($output, 'diffs'),
1,
"...check other nodes: 1 diff"
) or diag($output);
# 11-17T13:02:54 0 1 26 1 0 0.021 test.t
like(
$output,
qr/^\S+\s+ # ts
0\s+ # errors
1\s+ # diffs
26\s+ # rows
\d+\s+ # chunks
0\s+ # skipped
\S+\s+ # time
test.t$ # table
/xm,
"...check other nodes: it's in test.t"
);
like(
$output,
qr/the direct replica of h=$ip,P=12349 was not found or specified/,
"Warns that direct replica of the master isn't found or specified",
);
# Use the other DSN table with all three nodes. Now the tool should
# give a more specific warning than that ^.
# Originally, these tested a dsn table with all nodes; now we hijack
# those tests to also try the autodetection
for my $args (
["using recusion-method=dsn", '--recursion-method', "dsn=$node1_dsn,D=dsns,t=dsns"],
["using recursion-method=cluster,hosts", '--recursion-method', 'cluster,hosts']
)
{
my $test = shift @$args;
# Make a diff on node1. If ptc is really auto-detecting node1, then it
# should report this diff.
$node1->do("set sql_log_bin=0");
$node1->do("update test.t set c='zebra' where c='z'");
$node1->do("set sql_log_bin=1");
$output = output(
sub { pt_table_checksum::main($master_dsn,
@$args,
qw(-d test))
},
stderr => 1,
);
is(
PerconaTest::count_checksum_results($output, 'diffs'),
1,
"...check all nodes: 1 diff ($test)"
) or diag($output);
# 11-17T13:02:54 0 1 26 1 0 0.021 test.t
like(
$output,
qr/^\S+\s+ # ts
0\s+ # errors
1\s+ # diffs
26\s+ # rows
\d+\s+ # chunks
0\s+ # skipped
\S+\s+ # time
test.t$ # table
/xm,
"...check all nodes: it's in test.t ($test)"
);
like(
$output,
qr/Diffs will only be detected if the cluster is consistent with h=$ip,P=12345 because h=$ip,P=12349/,
"Warns that diffs only detected if cluster consistent with direct replica ($test)",
);
# Restore node1 so the cluster is consistent, but then make node2 differ.
# ptc should NOT detect this diff because the checksum query will replicate
# to node1, node1 isn't different, so it broadcasts the result in ROW format
# that all is ok, which node2 gets and thus false reports. This is why
# those ^ warnings exist.
$node1->do("set sql_log_bin=0");
$node1->do("update test.t set c='z' where c='zebra'");
$node1->do("set sql_log_bin=1");
$node2->do("set sql_log_bin=0");
$node2->do("update test.t set c='zebra' where c='z'");
$node2->do("set sql_log_bin=1");
($row) = $node2->selectrow_array("select c from test.t order by c desc limit 1");
is(
$row,
"zebra",
"Node2 is changed again ($test)"
);
($row) = $node1->selectrow_array("select c from test.t order by c desc limit 1");
is(
$row,
"z",
"Node1 not changed again ($test)"
);
($row) = $node3->selectrow_array("select c from test.t order by c desc limit 1");
is(
$row,
"z",
"Node3 not changed again ($test)"
);
# the other DSN table with all three nodes, but it won't matter because
# node1 is going to broadcast the false-positive that there are no diffs.
$output = output(
sub { pt_table_checksum::main($master_dsn,
@$args,
qw(-d test))
},
stderr => 1,
);
is(
PerconaTest::count_checksum_results($output, 'diffs'),
0,
"Limitation: diff not on direct replica not detected ($test)"
) or diag($output);
}
# ###########################################################################
# Be sure to stop the slave on node1, else further test will die with:
# Failed to execute -e "change master to master_host='127.0.0.1',
# master_user='msandbox', master_password='msandbox', master_port=12349"
# on node1: ERROR 1198 (HY000) at line 1: This operation cannot be performed
# with a running slave; run STOP SLAVE first
# ###########################################################################
$master_dbh->disconnect;
$sb->stop_sandbox('cmaster');
$node1->do("STOP SLAVE");
$node1->do("RESET SLAVE");
# #############################################################################
# cluster -> cluster
#
# This is not supported. The link between the two clusters is probably
# a traditional MySQL replication setup in ROW format, so any checksum
# results will be lost across it.
# #############################################################################
my $c = $sb->start_cluster(
nodes => [qw(node4 node5 node6)],
env => q/CLUSTER_NAME="cluster2"/,
);
# Load the same db just in case this does work (it shouldn't), then there
# will be normal results instead of an error because the db is missing.
$sb->load_file('node4', "$sample/a-z.sql");
# Add node4 in the cluster2 to the DSN table.
$node1->do(qq/INSERT INTO dsns.dsns VALUES (5, null, '$c->{node4}->{dsn}')/);
$output = output(
sub { pt_table_checksum::main(@args,
'--recursion-method', "dsn=$node1_dsn,D=dsns,t=dsns",
qw(-d test))
},
stderr => 1,
);
like(
$output,
qr/h=127(?:\Q.0.0\E)?.1,P=12345 is in cluster pt_sandbox_cluster/,
"Detects that node1 is in pt_sandbox_cluster"
);
like(
$output,
qr/h=127(?:\Q.0.0\E)?.1,P=2900 is in cluster cluster2/,
"Detects that node4 is in cluster2"
);
unlike(
$output,
qr/test/,
"Different clusters, no results"
);
$sb->stop_sandbox(qw(node4 node5 node6));
# Restore the DSN table in case there are more tests.
$node1->do(qq/DELETE FROM dsns.dsns WHERE id=5/);
# #############################################################################
# Done.
# #############################################################################
$sb->wipe_clean($node1);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
done_testing;
Reference in New Issue View Git Blame Copy Permalink