percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-01 18:25:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #329 from percona/PT-118

Browse Source 
PT-118 Modify pt-checksum such that it can report the number of rows of difference between master and slave
This commit is contained in:
Carlos Salguero
2018-05-18 14:27:20 -03:00
committed by GitHub
parent 966a1e8c87 e8e8d010cb
commit 68d97edbed
19 changed files with 66 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
bin/pt-table-checksum
View File

@@ -11194,6 +11194,7 @@ sub main {
# Update chunk-size based on rows/s checksum rate. # Update chunk-size based on rows/s checksum rate.
$nibble_iter->set_chunk_size($tbl->{chunk_size}); $nibble_iter->set_chunk_size($tbl->{chunk_size});
PTDEBUG && _d('Updated chunk size: '.$tbl->{chunk_size});
} }
# Every table should have a Progress obj; update it. # Every table should have a Progress obj; update it.
@@ -11267,6 +11268,12 @@ sub main {
map { $diff_chunks{ $_->{chunk} }++ } @$diffs; map { $diff_chunks{ $_->{chunk} }++ } @$diffs;
$exit_status |= $PTC_EXIT_STATUS{TABLE_DIFF}; $exit_status |= $PTC_EXIT_STATUS{TABLE_DIFF};
} }
my $max_cnt_diff=0;
for my $diff (@$diffs) {
if ( $diff->{cnt_diff} > $max_cnt_diff ) {
$tbl->{checksum_results}->{max_rows_cnt_diff} = $diff->{cnt_diff};
}
}
}; };
if ($EVAL_ERROR) { if ($EVAL_ERROR) {
if ( $o->get('quiet') < 2 ) { if ( $o->get('quiet') < 2 ) {
@@ -11701,8 +11708,8 @@ sub exec_nibble {
} }
{ {
my $line_fmt = "%14s %6s %6s %8s %7s %7s %7s %-s\n"; my $line_fmt = "%14s %6s %6s %8s % 10s %7s %7s %7s %-s\n";
my @headers = qw(TS ERRORS DIFFS ROWS CHUNKS SKIPPED TIME TABLE); my @headers = qw(TS ERRORS DIFFS ROWS DIFF_ROWS CHUNKS SKIPPED TIME TABLE);
sub print_checksum_results { sub print_checksum_results {
my (%args) = @_; my (%args) = @_;
@@ -11722,7 +11729,8 @@ sub print_checksum_results {
ts(), ts(),
$res->{errors} || 0, $res->{errors} || 0,
$res->{diffs} || 0, $res->{diffs} || 0,
$res->{n_rows} || 0, $res->{n_rows} || 0,
$tbl->{checksum_results}->{max_rows_cnt_diff} || 0,
$res->{n_chunks} || 0, $res->{n_chunks} || 0,
$res->{skipped} || 0, $res->{skipped} || 0,
sprintf('%.3f', $res->{start_time} ? time - $res->{start_time} : 0), sprintf('%.3f', $res->{start_time} ? time - $res->{start_time} : 0),

19
lib/Percona/Test.pm
View File

@@ -656,14 +656,15 @@ sub test_bash_tool {
} }
my %checksum_result_col = ( my %checksum_result_col = (
ts => 0, ts => 0,
errors => 1, errors => 1,
diffs => 2, diffs => 2,
rows => 3, rows => 3,
chunks => 4, diff_rows => 4,
skipped => 5, chunks => 5,
time => 6, skipped => 5,
table => 7, time => 6,
table => 7,
); );
sub count_checksum_results { sub count_checksum_results {
my ($output, $column, $table) = @_; my ($output, $column, $table) = @_;
@@ -696,7 +697,7 @@ sub normalize_checksum_results {
open my $fh, ">", $tmp_file or die "Cannot open $tmp_file: $OS_ERROR"; open my $fh, ">", $tmp_file or die "Cannot open $tmp_file: $OS_ERROR";
printf $fh $output; printf $fh $output;
close $fh; close $fh;
my $normal_output = `cat $tmp_file | awk '/^[0-9 ]/ {print \$2 " " \$3 " " \$4 " " \$5 " " \$6 " " \$8} /^[A-Z]/ {print \$0}'`; my $normal_output = `cat $tmp_file | awk '/^[0-9 ]/ {print \$2 " " \$3 " " \$4 " " \$5 " " \$6 " " \$7 " " \$9} /^[A-Z]/ {print \$0}'`;
`rm $tmp_file >/dev/null`; `rm $tmp_file >/dev/null`;
return $normal_output; return $normal_output;
} }

19
lib/PerconaTest.pm
View File

@@ -692,14 +692,15 @@ sub test_bash_tool {
} }
my %checksum_result_col = ( my %checksum_result_col = (
ts => 0, ts => 0,
errors => 1, errors => 1,
diffs => 2, diffs => 2,
rows => 3, rows => 3,
chunks => 4, diff_rows => 4,
skipped => 5, chunks => 5,
time => 6, skipped => 6,
table => 7, time => 7,
table => 7,
); );
sub count_checksum_results { sub count_checksum_results {
my ($output, $column, $table) = @_; my ($output, $column, $table) = @_;
@@ -732,7 +733,7 @@ sub normalize_checksum_results {
open my $fh, ">", $tmp_file or die "Cannot open $tmp_file: $OS_ERROR"; open my $fh, ">", $tmp_file or die "Cannot open $tmp_file: $OS_ERROR";
printf $fh $output; printf $fh $output;
close $fh; close $fh;
my $normal_output = `cat $tmp_file | awk '/^[0-9 ]/ {print \$2 " " \$3 " " \$4 " " \$5 " " \$6 " " \$8} /^[A-Z]/ {print \$0}'`; my $normal_output = `cat $tmp_file | awk '/^[0-9 ]/ {print \$2 " " \$3 " " \$4 " " \$5 " " \$6 " " \$7 " " \$9} /^[A-Z]/ {print \$0}'`;
if ( wantarray ) { if ( wantarray ) {
my $original_output = `cat $tmp_file`; my $original_output = `cat $tmp_file`;
return $normal_output, $original_output; return $normal_output, $original_output;

BIN
sandbox/servers/5.5/data.tar.gz
View File

Binary file not shown.

3
sandbox/servers/5.5/my.sandbox.cnf
View File

@@ -24,5 +24,4 @@ report-host = 127.0.0.1
report-port = PORT report-port = PORT
log-error = /tmp/PORT/data/mysqld.log log-error = /tmp/PORT/data/mysqld.log
innodb_lock_wait_timeout = 3 innodb_lock_wait_timeout = 3
general_log secure-file-priv =
general_log_file = genlog

5
t/pt-table-checksum/basics.t
View File

@@ -73,7 +73,7 @@ ok(
no_diff( no_diff(
sub { pt_table_checksum::main(@args) }, sub { pt_table_checksum::main(@args) },
"$sample/default-results-$sandbox_version.txt", "$sample/default-results-$sandbox_version.txt",
post_pipe => 'awk \'{print $2 " " $3 " " $4 " " $6 " " $8}\'', post_pipe => 'awk \'{print $2 " " $3 " " $4 " " $7 " " $9}\'',
), ),
"Default checksum" "Default checksum"
); );
@@ -85,6 +85,7 @@ ok(
# 2 # 2
$row = $master_dbh->selectrow_arrayref("select count(*) from percona.checksums"); $row = $master_dbh->selectrow_arrayref("select count(*) from percona.checksums");
my $max_chunks = $sandbox_version < '5.7' ? 60 : 100; my $max_chunks = $sandbox_version < '5.7' ? 60 : 100;
ok( ok(
$row->[0] > 25 && $row->[0] < $max_chunks, $row->[0] > 25 && $row->[0] < $max_chunks,
'Between 25 and 60 chunks' 'Between 25 and 60 chunks'
@@ -98,7 +99,7 @@ ok(
no_diff( no_diff(
sub { pt_table_checksum::main(@args, qw(--chunk-time 0)) }, sub { pt_table_checksum::main(@args, qw(--chunk-time 0)) },
"$sample/static-chunk-size-results-$sandbox_version.txt", "$sample/static-chunk-size-results-$sandbox_version.txt",
post_pipe => 'awk \'{print $2 " " $3 " " $4 " " $5 " " $6 " " $8}\'', post_pipe => 'awk \'{print $2 " " $3 " " $4 " " $6 " " $7 " " $9}\'',
), ),
"Static chunk size (--chunk-time 0)" "Static chunk size (--chunk-time 0)"
); );

6
t/pt-table-checksum/char_chunking.t
View File

@@ -34,7 +34,7 @@ my @args = ($master_dsn, qw(--set-vars innodb_lock_wait_timeout=3), '--max
$sb->create_dbs($master_dbh, ['test']); $sb->create_dbs($master_dbh, ['test']);
$sb->load_file('master', "t/lib/samples/char-chunking/ascii.sql", 'test'); $sb->load_file('master', "t/lib/samples/char-chunking/ascii.sql", 'test');
#1
ok( ok(
no_diff( no_diff(
sub { pt_table_checksum::main(@args, sub { pt_table_checksum::main(@args,
@@ -44,14 +44,14 @@ ok(
), ),
"Char chunk ascii, explain" "Char chunk ascii, explain"
); );
#2
ok( ok(
no_diff( no_diff(
sub { pt_table_checksum::main(@args, sub { pt_table_checksum::main(@args,
qw(--tables test.ascii --chunk-index c --chunk-size 20), qw(--tables test.ascii --chunk-index c --chunk-size 20),
qw(--chunk-time 0)) }, qw(--chunk-time 0)) },
"t/pt-table-checksum/samples/char-chunk-ascii.txt", "t/pt-table-checksum/samples/char-chunk-ascii.txt",
post_pipe => 'awk \'{print $2 " " $3 " " $4 " " $5 " " $6 " " $8}\'', post_pipe => 'awk \'{print $2 " " $3 " " $4 " " $6 " " $7 " " $9}\'',
), ),
"Char chunk ascii, chunk size 20" "Char chunk ascii, chunk size 20"
); );

7
t/pt-table-checksum/create_replicate_table.t
View File

@@ -50,6 +50,7 @@ $sb->wipe_clean($master_dbh);
eval { eval {
pt_table_checksum::main(@args, '--no-create-replicate-table'); pt_table_checksum::main(@args, '--no-create-replicate-table');
}; };
#1
like( like(
$EVAL_ERROR, $EVAL_ERROR,
qr/--replicate database percona does not exist/, qr/--replicate database percona does not exist/,
@@ -61,6 +62,7 @@ $master_dbh->do('use percona');
eval { eval {
pt_table_checksum::main(@args, '--no-create-replicate-table'); pt_table_checksum::main(@args, '--no-create-replicate-table');
}; };
#2
like( like(
$EVAL_ERROR, $EVAL_ERROR,
qr/--replicate table `percona`.`checksums` does not exist/, qr/--replicate table `percona`.`checksums` does not exist/,
@@ -90,11 +92,12 @@ $output = output(
sub { pt_table_checksum::main(@args, '--no-create-replicate-table', sub { pt_table_checksum::main(@args, '--no-create-replicate-table',
qw(-t sakila.country)) }, qw(-t sakila.country)) },
); );
#3
like( like(
$output, $output,
qr/^\S+\s+0\s+0\s+109\s+1\s+0\s+\S+\s+sakila.country$/m, qr/^\S+\s+0\s+0\s+109\s+0\s+1\s+0\s+\S+\s+sakila.country$/m,
"Uses pre-created replicate table" "Uses pre-created replicate table"
); ) or diag($output);
# ############################################################################ # ############################################################################
# Issue 1318: mk-tabke-checksum --create-replicate-table doesn't replicate # Issue 1318: mk-tabke-checksum --create-replicate-table doesn't replicate

12
t/pt-table-checksum/error_handling.t
View File

@@ -61,13 +61,13 @@ $output = output(
qw(--chunk-time 0 --chunk-size 100) ) }, qw(--chunk-time 0 --chunk-size 100) ) },
stderr => 1, stderr => 1,
); );
#1
like( like(
$output, $output,
qr/MySQL error 1265: Data truncated/, qr/MySQL error 1265: Data truncated/,
"MySQL error 1265: Data truncated for column" "MySQL error 1265: Data truncated for column"
); );
#2
my (@errors) = $output =~ m/error/; my (@errors) = $output =~ m/error/;
is( is(
scalar @errors, scalar @errors,
@@ -91,16 +91,16 @@ $output = output(
my $original_output; my $original_output;
($output, $original_output) = PerconaTest::normalize_checksum_results($output); ($output, $original_output) = PerconaTest::normalize_checksum_results($output);
#3
like( like(
$original_output, $original_output,
qr/Lock wait timeout exceeded/, qr/Lock wait timeout exceeded/,
"Warns about lock wait timeout" "Warns about lock wait timeout"
); );
#4
like( like(
$output, $output,
qr/^0 0 0 1 1 sakila.city/m, qr/^0 0 0 0 1 1 sakila.city/m,
"Skips chunk that times out" "Skips chunk that times out"
); );
@@ -129,7 +129,7 @@ unlike(
like( like(
$output, $output,
qr/^0 0 600 1 0 sakila.city/m, qr/^0 0 600 0 1 0 sakila.city/m,
"Checksum retried after lock wait timeout" "Checksum retried after lock wait timeout"
); );

2
t/pt-table-checksum/issue_388.t
View File

@@ -56,7 +56,7 @@ unlike(
like( like(
$output, $output,
qr/^\S+\s+0\s+0\s+1\s+1\s+/m, qr/^\S+\s+0\s+0\s+1\s+0\s+1\s+/m,
'Checksums the table (issue 388)' 'Checksums the table (issue 388)'
); );

2
t/pt-table-checksum/pt-226.t
View File

@@ -63,7 +63,7 @@ isnt(
like( like(
$output, $output,
qr/1\s+100\s+1\s+0\s+.*test.joinit/, qr/1\s+100\s+0\s+1\s+0\s+.*test.joinit/,
"PT-226 table joinit has differences", "PT-226 table joinit has differences",
); );

12
t/pt-table-checksum/resume.t
View File

@@ -441,10 +441,10 @@ is(
"Checking if all tables can be checksummed ... "Checking if all tables can be checksummed ...
Starting checksum ... Starting checksum ...
Resuming from sakila.rental chunk 11, timestamp 2011-10-15 13:00:49 Resuming from sakila.rental chunk 11, timestamp 2011-10-15 13:00:49
ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE ERRORS DIFFS ROWS DIFF_ROWS CHUNKS SKIPPED TABLE
0 0 5044 8 0 sakila.rental 0 0 5044 0 8 0 sakila.rental
0 0 2 1 0 sakila.staff 0 0 2 0 1 0 sakila.staff
0 0 2 1 0 sakila.store 0 0 2 0 1 0 sakila.store
", ",
"Resumed from last updated chunk" "Resumed from last updated chunk"
); );
@@ -497,8 +497,8 @@ is(
$output, $output,
"Checking if all tables can be checksummed ... "Checking if all tables can be checksummed ...
Starting checksum ... Starting checksum ...
ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE ERRORS DIFFS ROWS DIFF_ROWS CHUNKS SKIPPED TABLE
0 0 26 8 0 test.t3 0 0 26 0 8 0 test.t3
", ",
"Resumed from t3" "Resumed from t3"
); );

2
t/pt-table-checksum/samples/char-chunk-ascii.txt
View File

@@ -1,4 +1,4 @@
if all tables can be ... if all tables be checksummed
checksum ... checksum ...
ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE
0 0 142 10 0 test.ascii 0 0 142 10 0 test.ascii

2
t/pt-table-checksum/samples/default-results-5.5.txt
View File

@@ -1,3 +1,5 @@
if all tables checksummed
checksum ...
ERRORS DIFFS ROWS SKIPPED TABLE ERRORS DIFFS ROWS SKIPPED TABLE
0 0 0 0 mysql.columns_priv 0 0 0 0 mysql.columns_priv
0 0 0 0 mysql.db 0 0 0 0 mysql.db

2
t/pt-table-checksum/samples/default-results-5.6.txt
View File

@@ -1,4 +1,4 @@
if all tables be ... if all tables checksummed
checksum ... checksum ...
ERRORS DIFFS ROWS SKIPPED TABLE ERRORS DIFFS ROWS SKIPPED TABLE
0 0 0 0 mysql.columns_priv 0 0 0 0 mysql.columns_priv

2
t/pt-table-checksum/samples/default-results-5.7.txt
View File

@@ -1,4 +1,4 @@
if all tables be ... if all tables checksummed
checksum ... checksum ...
ERRORS DIFFS ROWS SKIPPED TABLE ERRORS DIFFS ROWS SKIPPED TABLE
0 0 0 0 mysql.columns_priv 0 0 0 0 mysql.columns_priv

2
t/pt-table-checksum/samples/static-chunk-size-results-5.5.txt
View File

@@ -1,3 +1,5 @@
if all tables be checksummed
checksum ...
ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE
0 0 0 1 0 mysql.columns_priv 0 0 0 1 0 mysql.columns_priv
0 0 0 1 0 mysql.db 0 0 0 1 0 mysql.db

2
t/pt-table-checksum/samples/static-chunk-size-results-5.6.txt
View File

@@ -1,4 +1,4 @@
if all tables can be ... if all tables be checksummed
checksum ... checksum ...
ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE
0 0 0 1 0 mysql.columns_priv 0 0 0 1 0 mysql.columns_priv

2
t/pt-table-checksum/samples/static-chunk-size-results-5.7.txt
View File

@@ -1,4 +1,4 @@
if all tables can be ... if all tables be checksummed
checksum ... checksum ...
ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE
0 0 0 1 0 mysql.columns_priv 0 0 0 1 0 mysql.columns_priv