Merge pull request #106 from percona/bug-1592608

Bug-1592608 Large BLOB/TEXT/BINARY Produces NULL Checksum
2025-09-07 12:58:45 +00:00 · 2016-07-05 12:56:19 -03:00
parent 3dfffccffd 38979b2096
commit f602f0456d
11 changed files with 92 additions and 7 deletions
--- a/bin/pt-table-checksum
+++ b/bin/pt-table-checksum
@@ -5885,6 +5885,9 @@ sub get_checksum_columns {
         elsif ( $trim && $type =~ m/varchar/ ) {
            $result = "TRIM($result)";
         }
+         elsif ( $type =~ m/blob|text|binary/ ) {
+            $result = "CRC32($result)";
+         }
         $result;
      }
      grep {
--- a/bin/pt-table-sync
+++ b/bin/pt-table-sync
@@ -4862,6 +4862,9 @@ sub make_row_checksum {
         elsif ( $args{trim} && $type =~ m/varchar/ ) {
            $result = "TRIM($result)";
         }
+         elsif ( $type =~ m/binary|text|blob/ ) {
+            $result = "CRC32($result)";
+         }
         $result;
      }
      grep {
--- a/lib/RowChecksum.pm
+++ b/lib/RowChecksum.pm
@@ -224,6 +224,9 @@ sub get_checksum_columns {
         elsif ( $trim && $type =~ m/varchar/ ) {
            $result = "TRIM($result)";
         }
+         elsif ( $type =~ m/blob|text|binary/ ) {
+            $result = "CRC32($result)";
+         }
         $result;
      }
      grep {
--- a/sandbox/servers/5.6/my.sandbox.cnf
+++ b/sandbox/servers/5.6/my.sandbox.cnf
@@ -15,7 +15,7 @@ innodb_buffer_pool_size    = 16M
 innodb_data_home_dir       = /tmp/PORT/data
 innodb_log_group_home_dir  = /tmp/PORT/data
 innodb_data_file_path      = ibdata1:10M:autoextend
-innodb_log_file_size       = 5M
+innodb_log_file_size       = 64M
 log-bin                    = mysql-bin
 relay_log                  = mysql-relay-bin
 log_slave_updates
--- a/t/pt-table-checksum/bugs.t
+++ b/t/pt-table-checksum/bugs.t
@@ -105,7 +105,7 @@ is_deeply(
   $rows,
   [
      ['test', 'test_empty', '1', '0',        '0'],  # empty
-      ['test', 'test_full',  '1', 'ac967054', '1'],  # row
+      ['test', 'test_full',  '1', '4d284606', '1'],  # row
   ],
   "Bug 987393 (empty table): checksums"
 ) or print STDERR Dumper($rows);
--- a/t/pt-table-checksum/chunk_index.t
+++ b/t/pt-table-checksum/chunk_index.t
@@ -36,7 +36,6 @@ my $out        = "t/pt-table-checksum/samples/";

 $sb->load_file('master', "t/pt-table-checksum/samples/issue_519.sql");

-
 ok(
   no_diff(
      sub { pt_table_checksum::main(@args, qw(-t issue_519.t --explain)) },
@@ -58,7 +57,7 @@ ok(
   no_diff(
      sub { pt_table_checksum::main(@args, qw(--chunk-index myidx),
         qw(-t issue_519.t --explain)) },
-      "$out/chunkidx002.txt",
+      "$out/chunkidx002.txt", 
   ),
   "Use --chunk-index"
 );
@@ -118,6 +117,7 @@ ok(
 # #############################################################################
 $sb->load_file('master', "t/pt-table-checksum/samples/all-uc-table.sql");
 my $exit_status = 0;
+
 $output = output(sub {
   $exit_status = pt_table_checksum::main(
      $master_dsn, '--max-load', '',
@@ -260,4 +260,5 @@ cmp_ok(
 # #############################################################################
 $sb->wipe_clean($dbh);
 ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+
 exit;
--- a/t/pt-table-checksum/issue_1592608.t
+++ b/t/pt-table-checksum/issue_1592608.t
@@ -0,0 +1,64 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+
+use PerconaTest;
+use Sandbox;
+use SqlModes;
+require "$trunk/bin/pt-table-checksum";
+
+my $dp  = new DSNParser(opts=>$dsn_opts);
+my $sb  = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $dbh = $sb->get_dbh_for('master');
+
+if ( !$dbh ) {
+   plan skip_all => 'Cannot connect to sandbox master';
+}
+else {
+   plan tests => 2;
+}
+
+# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
+# so we need to specify --set-vars innodb_lock_wait_timeout=3 else the tool will die.
+# And --max-load "" prevents waiting for status variables.
+my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox,D=bug_1592608';
+my @args       = ($master_dsn, qw(--set-vars innodb_lock_wait_timeout=3), '--max-load', ''); 
+my $output;
+
+# We test that checksum works with invalid dates, 
+# but for that we need to turn off MySQL's NO_ZERO_IN_DATE mode 
+my $modes = new SqlModes($dbh, global=>1);
+$modes->del('NO_ZERO_IN_DATE');
+$sb->load_file('master', 't/pt-table-checksum/samples/issue_1592608.sql');
+# #############################################################################
+# Issue 602: mk-table-checksum issue with invalid dates
+# #############################################################################
+
+#sub { pt_table_checksum::main(@args, qw(-t issue_1592608.t --tables t )) },
+$output = output(
+   sub { pt_table_checksum::main(@args, qw(-t t)) },
+   stderr => 1,
+);
+
+is(
+   PerconaTest::count_checksum_results($output, 'rows'),
+   1,
+   "Large BLOB/TEXT/BINARY Checksum"
+);
+
+$modes->restore_original_modes();
+# #############################################################################
+# Done.
+# #############################################################################
+$sb->wipe_clean($dbh);
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+exit;
--- a/t/pt-table-checksum/samples/chunkidx001.txt
+++ b/t/pt-table-checksum/samples/chunkidx001.txt
@@ -2,7 +2,7 @@
 -- issue_519.t
 --

-REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, `t`, CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`PRIMARY`) WHERE ((`i` >= ?)) AND ((`i` <= ?)) /*checksum chunk*/
+REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, CRC32(`t`), CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`PRIMARY`) WHERE ((`i` >= ?)) AND ((`i` <= ?)) /*checksum chunk*/

 REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*), '0' FROM `issue_519`.`t` FORCE INDEX(`PRIMARY`) WHERE ((`i` < ?)) ORDER BY `i` /*past lower chunk*/

--- a/t/pt-table-checksum/samples/chunkidx002.txt
+++ b/t/pt-table-checksum/samples/chunkidx002.txt
@@ -2,7 +2,7 @@
 -- issue_519.t
 --

-REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, `t`, CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`myidx`) WHERE ((`i` > ?) OR (`i` = ? AND `y` >= ?)) AND ((`i` < ?) OR (`i` = ? AND `y` <= ?)) /*checksum chunk*/
+REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, CRC32(`t`), CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`myidx`) WHERE ((`i` > ?) OR (`i` = ? AND `y` >= ?)) AND ((`i` < ?) OR (`i` = ? AND `y` <= ?)) /*checksum chunk*/

 REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*), '0' FROM `issue_519`.`t` FORCE INDEX(`myidx`) WHERE ((`i` < ?) OR (`i` = ? AND `y` < ?)) ORDER BY `i`, `y` /*past lower chunk*/

--- a/t/pt-table-checksum/samples/chunkidx003.txt
+++ b/t/pt-table-checksum/samples/chunkidx003.txt
@@ -2,7 +2,7 @@
 -- issue_519.t
 --

-REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, `t`, CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`y`) WHERE ((`y` >= ?)) AND ((`y` <= ?)) /*checksum chunk*/
+REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, CRC32(`t`), CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`y`) WHERE ((`y` >= ?)) AND ((`y` <= ?)) /*checksum chunk*/

 REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*), '0' FROM `issue_519`.`t` FORCE INDEX(`y`) WHERE ((`y` < ?)) ORDER BY `y` /*past lower chunk*/

--- a/t/pt-table-checksum/samples/issue_1592608.sql
+++ b/t/pt-table-checksum/samples/issue_1592608.sql
@@ -0,0 +1,11 @@
+CREATE SCHEMA IF NOT EXISTS bug_1592608;
+USE bug_1592608;
+DROP TABLE IF EXISTS t;
+CREATE TABLE t (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, 
+  a  MEDIUMTEXT, 
+  b  MEDIUMBLOB
+)engine=innodb;
+
+INSERT INTO t (a, b) VALUES (REPEAT('a', 2097152*2), CAST(REPEAT('a', 2097152*2) AS BINARY));
+