Merge pull request #329 from percona/PT-118

PT-118 Modify pt-checksum such that it can report the number of rows of difference between master and slave
2025-09-01 18:25:59 +00:00 · 2018-05-18 14:27:20 -03:00
parent 966a1e8c87 e8e8d010cb
commit 68d97edbed
19 changed files with 66 additions and 49 deletions
--- a/bin/pt-table-checksum
+++ b/bin/pt-table-checksum
@@ -11194,6 +11194,7 @@ sub main {

               # Update chunk-size based on rows/s checksum rate.
               $nibble_iter->set_chunk_size($tbl->{chunk_size});
+               PTDEBUG && _d('Updated chunk size: '.$tbl->{chunk_size});
            }

            # Every table should have a Progress obj; update it.
@@ -11267,6 +11268,12 @@ sub main {
                     map { $diff_chunks{ $_->{chunk} }++ } @$diffs;
                     $exit_status |= $PTC_EXIT_STATUS{TABLE_DIFF};
                  }
+                  my $max_cnt_diff=0;
+                  for my $diff (@$diffs) {
+                     if ( $diff->{cnt_diff} > $max_cnt_diff ) {
+                         $tbl->{checksum_results}->{max_rows_cnt_diff} = $diff->{cnt_diff};
+                     }
+                  }
               };
               if ($EVAL_ERROR) {
                  if ( $o->get('quiet') < 2 ) {
@@ -11701,8 +11708,8 @@ sub exec_nibble {
 }

 {
-my $line_fmt = "%14s %6s %6s %8s %7s %7s %7s %-s\n";
-my @headers  =  qw(TS ERRORS DIFFS ROWS CHUNKS SKIPPED TIME TABLE);
+my $line_fmt = "%14s %6s %6s %8s % 10s %7s %7s %7s %-s\n";
+my @headers  =  qw(TS ERRORS DIFFS ROWS DIFF_ROWS CHUNKS SKIPPED TIME TABLE);

 sub print_checksum_results {
   my (%args) = @_;
@@ -11722,7 +11729,8 @@ sub print_checksum_results {
      ts(),
      $res->{errors}   || 0,
      $res->{diffs}    || 0,
-      $res->{n_rows}   || 0,
+      $res->{n_rows}   || 0,            
+      $tbl->{checksum_results}->{max_rows_cnt_diff} || 0,
      $res->{n_chunks} || 0,
      $res->{skipped}  || 0,
      sprintf('%.3f', $res->{start_time} ? time - $res->{start_time} : 0),
--- a/lib/Percona/Test.pm
+++ b/lib/Percona/Test.pm
@@ -656,14 +656,15 @@ sub test_bash_tool {
 }

 my %checksum_result_col = (
-   ts      => 0,
-   errors  => 1,
-   diffs   => 2,
-   rows    => 3,
-   chunks  => 4,
-   skipped => 5,
-   time    => 6,
-   table   => 7,
+   ts        => 0,
+   errors    => 1,
+   diffs     => 2,
+   rows      => 3,
+   diff_rows => 4,
+   chunks    => 5,
+   skipped   => 5,
+   time      => 6,
+   table     => 7,
 );
 sub count_checksum_results {
   my ($output, $column, $table) = @_;
@@ -696,7 +697,7 @@ sub normalize_checksum_results {
   open my $fh, ">", $tmp_file or die "Cannot open $tmp_file: $OS_ERROR";
   printf $fh $output;
   close $fh;
-   my $normal_output = `cat $tmp_file | awk '/^[0-9 ]/ {print \$2 " " \$3 " " \$4 " " \$5 " " \$6 " " \$8} /^[A-Z]/ {print \$0}'`;
+   my $normal_output = `cat $tmp_file | awk '/^[0-9 ]/ {print \$2 " " \$3 " " \$4 " " \$5 " " \$6 " " \$7 " " \$9} /^[A-Z]/ {print \$0}'`;
   `rm $tmp_file >/dev/null`;
   return $normal_output;
 }
--- a/lib/PerconaTest.pm
+++ b/lib/PerconaTest.pm
@@ -692,14 +692,15 @@ sub test_bash_tool {
 }

 my %checksum_result_col = (
-   ts      => 0,
-   errors  => 1,
-   diffs   => 2,
-   rows    => 3,
-   chunks  => 4,
-   skipped => 5,
-   time    => 6,
-   table   => 7,
+   ts        => 0,
+   errors    => 1,
+   diffs     => 2,
+   rows      => 3,
+   diff_rows => 4,
+   chunks    => 5,
+   skipped   => 6,
+   time      => 7,
+   table     => 7,
 );
 sub count_checksum_results {
   my ($output, $column, $table) = @_;
@@ -732,7 +733,7 @@ sub normalize_checksum_results {
   open my $fh, ">", $tmp_file or die "Cannot open $tmp_file: $OS_ERROR";
   printf $fh $output;
   close $fh;
-   my $normal_output = `cat $tmp_file | awk '/^[0-9 ]/ {print \$2 " " \$3 " " \$4 " " \$5 " " \$6 " " \$8} /^[A-Z]/ {print \$0}'`;
+   my $normal_output = `cat $tmp_file | awk '/^[0-9 ]/ {print \$2 " " \$3 " " \$4 " " \$5 " " \$6 " " \$7 " " \$9} /^[A-Z]/ {print \$0}'`;
   if ( wantarray ) {
      my $original_output = `cat $tmp_file`;
      return $normal_output, $original_output;
--- a/sandbox/servers/5.5/data.tar.gz
+++ b/sandbox/servers/5.5/data.tar.gz
--- a/sandbox/servers/5.5/my.sandbox.cnf
+++ b/sandbox/servers/5.5/my.sandbox.cnf
@@ -24,5 +24,4 @@ report-host                = 127.0.0.1
 report-port                = PORT
 log-error                  = /tmp/PORT/data/mysqld.log
 innodb_lock_wait_timeout   = 3
-general_log
-general_log_file           = genlog
+secure-file-priv           = 
--- a/t/pt-table-checksum/basics.t
+++ b/t/pt-table-checksum/basics.t
@@ -73,7 +73,7 @@ ok(
   no_diff(
      sub { pt_table_checksum::main(@args) },
      "$sample/default-results-$sandbox_version.txt",
-      post_pipe => 'awk \'{print $2 " " $3 " " $4 " " $6 " " $8}\'',
+      post_pipe => 'awk \'{print $2 " " $3 " " $4 " " $7 " " $9}\'',
   ),
   "Default checksum"
 );
@@ -85,6 +85,7 @@ ok(
 # 2
 $row = $master_dbh->selectrow_arrayref("select count(*) from percona.checksums");
 my $max_chunks = $sandbox_version < '5.7' ? 60 : 100;
+
 ok(
   $row->[0] > 25 && $row->[0] < $max_chunks,
   'Between 25 and 60 chunks'
@@ -98,7 +99,7 @@ ok(
   no_diff(
      sub { pt_table_checksum::main(@args, qw(--chunk-time 0)) },
      "$sample/static-chunk-size-results-$sandbox_version.txt",
-      post_pipe => 'awk \'{print $2 " " $3 " " $4 " " $5 " " $6 " " $8}\'',
+      post_pipe => 'awk \'{print $2 " " $3 " " $4 " " $6 " " $7 " " $9}\'',
   ),
   "Static chunk size (--chunk-time 0)"
 );
--- a/t/pt-table-checksum/char_chunking.t
+++ b/t/pt-table-checksum/char_chunking.t
@@ -34,7 +34,7 @@ my @args       = ($master_dsn, qw(--set-vars innodb_lock_wait_timeout=3), '--max

 $sb->create_dbs($master_dbh, ['test']);
 $sb->load_file('master', "t/lib/samples/char-chunking/ascii.sql", 'test');
-
+#1
 ok(
   no_diff(
      sub { pt_table_checksum::main(@args,
@@ -44,14 +44,14 @@ ok(
   ),
   "Char chunk ascii, explain"
 );
-
+#2
 ok(
   no_diff(
      sub { pt_table_checksum::main(@args,
         qw(--tables test.ascii --chunk-index c --chunk-size 20),
         qw(--chunk-time 0)) },
      "t/pt-table-checksum/samples/char-chunk-ascii.txt",
-      post_pipe => 'awk \'{print $2 " " $3 " " $4 " " $5 " " $6 " " $8}\'',
+      post_pipe => 'awk \'{print $2 " " $3 " " $4 " " $6 " " $7 " " $9}\'',
   ),
   "Char chunk ascii, chunk size 20"
 );
--- a/t/pt-table-checksum/create_replicate_table.t
+++ b/t/pt-table-checksum/create_replicate_table.t
@@ -50,6 +50,7 @@ $sb->wipe_clean($master_dbh);
 eval {
   pt_table_checksum::main(@args, '--no-create-replicate-table');
 };
+#1
 like(
   $EVAL_ERROR,
   qr/--replicate database percona does not exist/,
@@ -61,6 +62,7 @@ $master_dbh->do('use percona');
 eval {
   pt_table_checksum::main(@args, '--no-create-replicate-table');
 };
+#2
 like(
   $EVAL_ERROR,
   qr/--replicate table `percona`.`checksums` does not exist/,
@@ -90,11 +92,12 @@ $output = output(
   sub { pt_table_checksum::main(@args, '--no-create-replicate-table',
      qw(-t sakila.country)) },
 );
+#3
 like(
   $output,
-   qr/^\S+\s+0\s+0\s+109\s+1\s+0\s+\S+\s+sakila.country$/m,
+   qr/^\S+\s+0\s+0\s+109\s+0\s+1\s+0\s+\S+\s+sakila.country$/m,
   "Uses pre-created replicate table"
-);
+) or diag($output);

 # ############################################################################
 # Issue 1318: mk-tabke-checksum --create-replicate-table doesn't replicate
--- a/t/pt-table-checksum/error_handling.t
+++ b/t/pt-table-checksum/error_handling.t
@@ -61,13 +61,13 @@ $output = output(
      qw(--chunk-time 0 --chunk-size 100) ) },
   stderr => 1,
 );
-
+#1
 like(
   $output,
   qr/MySQL error 1265: Data truncated/,
   "MySQL error 1265: Data truncated for column"
 );
-
+#2
 my (@errors) = $output =~ m/error/;
 is(
   scalar @errors,
@@ -91,16 +91,16 @@ $output = output(

 my $original_output;
 ($output, $original_output) = PerconaTest::normalize_checksum_results($output);
-
+#3
 like(
   $original_output,
   qr/Lock wait timeout exceeded/,
   "Warns about lock wait timeout"
 );
-
+#4
 like(
   $output,
-   qr/^0 0 0 1 1 sakila.city/m,
+   qr/^0 0 0 0 1 1 sakila.city/m,
   "Skips chunk that times out"
 );

@@ -129,7 +129,7 @@ unlike(

 like(
   $output,
-   qr/^0 0 600 1 0 sakila.city/m,
+   qr/^0 0 600 0 1 0 sakila.city/m,
   "Checksum retried after lock wait timeout"
 );

--- a/t/pt-table-checksum/issue_388.t
+++ b/t/pt-table-checksum/issue_388.t
@@ -56,7 +56,7 @@ unlike(

 like(
   $output,
-   qr/^\S+\s+0\s+0\s+1\s+1\s+/m,
+   qr/^\S+\s+0\s+0\s+1\s+0\s+1\s+/m,
   'Checksums the table (issue 388)'
 );

--- a/t/pt-table-checksum/pt-226.t
+++ b/t/pt-table-checksum/pt-226.t
@@ -63,7 +63,7 @@ isnt(

 like(
    $output,
-    qr/1\s+100\s+1\s+0\s+.*test.joinit/,
+    qr/1\s+100\s+0\s+1\s+0\s+.*test.joinit/,
    "PT-226 table joinit has differences",
 );

--- a/t/pt-table-checksum/resume.t
+++ b/t/pt-table-checksum/resume.t
@@ -441,10 +441,10 @@ is(
 "Checking if all tables can be checksummed ...
 Starting checksum ...
 Resuming from sakila.rental chunk 11, timestamp 2011-10-15 13:00:49
-ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE
-0 0 5044 8 0 sakila.rental
-0 0 2 1 0 sakila.staff
-0 0 2 1 0 sakila.store
+ERRORS DIFFS ROWS DIFF_ROWS CHUNKS SKIPPED TABLE
+0 0 5044 0 8 0 sakila.rental
+0 0 2 0 1 0 sakila.staff
+0 0 2 0 1 0 sakila.store
 ",
   "Resumed from last updated chunk"
 );
@@ -497,8 +497,8 @@ is(
   $output,
 "Checking if all tables can be checksummed ...
 Starting checksum ...
-ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE
-0 0 26 8 0 test.t3
+ERRORS DIFFS ROWS DIFF_ROWS CHUNKS SKIPPED TABLE
+0 0 26 0 8 0 test.t3
 ",
   "Resumed from t3"
 );
--- a/t/pt-table-checksum/samples/char-chunk-ascii.txt
+++ b/t/pt-table-checksum/samples/char-chunk-ascii.txt
@@ -1,4 +1,4 @@
-if all tables can be ...
+if all tables be checksummed 
 checksum ...    
 ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE
 0 0 142 10 0 test.ascii
--- a/t/pt-table-checksum/samples/default-results-5.5.txt
+++ b/t/pt-table-checksum/samples/default-results-5.5.txt
@@ -1,3 +1,5 @@
+if all tables checksummed 
+checksum ...   
 ERRORS DIFFS ROWS SKIPPED TABLE
 0 0 0 0 mysql.columns_priv
 0 0 0 0 mysql.db
--- a/t/pt-table-checksum/samples/default-results-5.6.txt
+++ b/t/pt-table-checksum/samples/default-results-5.6.txt
@@ -1,4 +1,4 @@
-if all tables be ...
+if all tables checksummed 
 checksum ...   
 ERRORS DIFFS ROWS SKIPPED TABLE
 0 0 0 0 mysql.columns_priv
--- a/t/pt-table-checksum/samples/default-results-5.7.txt
+++ b/t/pt-table-checksum/samples/default-results-5.7.txt
@@ -1,4 +1,4 @@
-if all tables be ...
+if all tables checksummed 
 checksum ...   
 ERRORS DIFFS ROWS SKIPPED TABLE
 0 0 0 0 mysql.columns_priv
--- a/t/pt-table-checksum/samples/static-chunk-size-results-5.5.txt
+++ b/t/pt-table-checksum/samples/static-chunk-size-results-5.5.txt
@@ -1,3 +1,5 @@
+if all tables be checksummed 
+checksum ...    
 ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE
 0 0 0 1 0 mysql.columns_priv
 0 0 0 1 0 mysql.db
--- a/t/pt-table-checksum/samples/static-chunk-size-results-5.6.txt
+++ b/t/pt-table-checksum/samples/static-chunk-size-results-5.6.txt
@@ -1,4 +1,4 @@
-if all tables can be ...
+if all tables be checksummed 
 checksum ...    
 ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE
 0 0 0 1 0 mysql.columns_priv
--- a/t/pt-table-checksum/samples/static-chunk-size-results-5.7.txt
+++ b/t/pt-table-checksum/samples/static-chunk-size-results-5.7.txt
@@ -1,4 +1,4 @@
-if all tables can be ...
+if all tables be checksummed 
 checksum ...    
 ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE
 0 0 0 1 0 mysql.columns_priv