diff --git a/bin/pt-archiver b/bin/pt-archiver index 4f9fca80..445e44bc 100755 --- a/bin/pt-archiver +++ b/bin/pt-archiver @@ -3201,7 +3201,7 @@ sub generate_del_stmt { my $index = $tp->find_best_index($tbl, $args{index}); die "Cannot find an ascendable index in table" unless $index; - if ( $index ) { + if ( $index && $tbl->{keys}->{$index}->{is_unique}) { @del_cols = @{$tbl->{keys}->{$index}->{cols}}; } else { diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index 52dc42b9..e13e3ab9 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -3053,7 +3053,7 @@ sub generate_del_stmt { my $index = $tp->find_best_index($tbl, $args{index}); die "Cannot find an ascendable index in table" unless $index; - if ( $index ) { + if ( $index && $tbl->{keys}->{$index}->{is_unique}) { @del_cols = @{$tbl->{keys}->{$index}->{cols}}; } else { diff --git a/bin/pt-table-checksum b/bin/pt-table-checksum index 95456179..b5892ec0 100755 --- a/bin/pt-table-checksum +++ b/bin/pt-table-checksum @@ -4901,7 +4901,7 @@ sub generate_del_stmt { my $index = $tp->find_best_index($tbl, $args{index}); die "Cannot find an ascendable index in table" unless $index; - if ( $index ) { + if ( $index && $tbl->{keys}->{$index}->{is_unique}) { @del_cols = @{$tbl->{keys}->{$index}->{cols}}; } else { @@ -5885,6 +5885,9 @@ sub get_checksum_columns { elsif ( $trim && $type =~ m/varchar/ ) { $result = "TRIM($result)"; } + elsif ( $type =~ m/blob|text|binary/ ) { + $result = "CRC32($result)"; + } $result; } grep { diff --git a/bin/pt-table-sync b/bin/pt-table-sync index 75fa81ba..9f2d1196 100755 --- a/bin/pt-table-sync +++ b/bin/pt-table-sync @@ -4862,6 +4862,9 @@ sub make_row_checksum { elsif ( $args{trim} && $type =~ m/varchar/ ) { $result = "TRIM($result)"; } + elsif ( $type =~ m/binary|text|blob/ ) { + $result = "CRC32($result)"; + } $result; } grep { @@ -6480,7 +6483,7 @@ sub generate_del_stmt { my $index = $tp->find_best_index($tbl, $args{index}); die "Cannot find an ascendable index in table" unless $index; - if ( $index ) { + if ( $index && $tbl->{keys}->{$index}->{is_unique}) { @del_cols = @{$tbl->{keys}->{$index}->{cols}}; } else { diff --git a/lib/RowChecksum.pm b/lib/RowChecksum.pm index 3d6e814a..1833437a 100644 --- a/lib/RowChecksum.pm +++ b/lib/RowChecksum.pm @@ -224,6 +224,9 @@ sub get_checksum_columns { elsif ( $trim && $type =~ m/varchar/ ) { $result = "TRIM($result)"; } + elsif ( $type =~ m/blob|text|binary/ ) { + $result = "CRC32($result)"; + } $result; } grep { diff --git a/lib/TableNibbler.pm b/lib/TableNibbler.pm index 0eed5177..945da85f 100644 --- a/lib/TableNibbler.pm +++ b/lib/TableNibbler.pm @@ -247,7 +247,7 @@ sub generate_del_stmt { die "Cannot find an ascendable index in table" unless $index; # These are the columns needed for the DELETE statement's WHERE clause. - if ( $index ) { + if ( $index && $tbl->{keys}->{$index}->{is_unique}) { @del_cols = @{$tbl->{keys}->{$index}->{cols}}; } else { diff --git a/sandbox/servers/5.6/my.sandbox.cnf b/sandbox/servers/5.6/my.sandbox.cnf index 0f4f33ef..aedf801c 100644 --- a/sandbox/servers/5.6/my.sandbox.cnf +++ b/sandbox/servers/5.6/my.sandbox.cnf @@ -15,7 +15,7 @@ innodb_buffer_pool_size = 64M innodb_data_home_dir = /tmp/PORT/data innodb_log_group_home_dir = /tmp/PORT/data innodb_data_file_path = ibdata1:10M:autoextend -innodb_log_file_size = 5M +innodb_log_file_size = 64M log-bin = mysql-bin relay_log = mysql-relay-bin log_slave_updates diff --git a/t/lib/TableNibbler.t b/t/lib/TableNibbler.t index 51fe17cf..4df9a4a5 100644 --- a/t/lib/TableNibbler.t +++ b/t/lib/TableNibbler.t @@ -134,11 +134,13 @@ is_deeply( cols => [qw(film_id)], ), { - cols => [qw(film_id title)], + cols => [qw(film_id title description release_year language_id original_language_id rental_duration rental_rate length replacement_cost rating special_features last_update)], index => 'idx_title', - where => '(`title` = ?)', - slice => [1], - scols => [qw(title)], + where => '(`film_id` = ? AND `title` = ? AND ((? IS NULL AND `description` IS NULL) OR (`description` = ?)) AND ((? IS NULL AND `release_year` IS NULL) OR (`release_year` = ?)) AND `language_id` = ? AND ((? IS NULL AND `original_language_id` IS NULL) OR (`original_language_id` = ?)) AND `rental_duration` = ? AND `rental_rate` = ? AND ((? IS NULL AND `length` IS NULL) OR (`length` = ?)) AND `replacement_cost` = ? AND ((? IS NULL AND `rating` IS NULL) OR (`rating` = ?)) AND ((? IS NULL AND `special_features` IS NULL) OR (`special_features` = ?)) AND `last_update` = ?)', + slice => [ 0, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 8, 8, 9, 10, 10, 11, 11, 12 ], + scols => [qw( film_id title description description release_year release_year language_id original_language_id original_language_id + rental_duration rental_rate length length replacement_cost rating rating special_features special_features last_update)], + }, 'del stmt on sakila.film with different index and extra column', ); diff --git a/t/pt-archiver/issue_1166.t b/t/pt-archiver/issue_1166.t index 749c90b6..fb511d3b 100644 --- a/t/pt-archiver/issue_1166.t +++ b/t/pt-archiver/issue_1166.t @@ -56,7 +56,7 @@ $output = output( ); like( $output, - qr/DELETE FROM `test`\.`issue_1166` WHERE \(`id` = \?\) LIMIT 1$/m, + qr/LIMIT 1$/m, "LIMIT 1 with non-unique index (issue 1166)" ); diff --git a/t/pt-archiver/issue_1593265.t b/t/pt-archiver/issue_1593265.t new file mode 100644 index 00000000..2ba5a340 --- /dev/null +++ b/t/pt-archiver/issue_1593265.t @@ -0,0 +1,64 @@ +#!/usr/bin/env perl + +BEGIN { + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; +}; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use Test::More; + +use PerconaTest; +use Sandbox; +require "$trunk/bin/pt-archiver"; + +my $dp = new DSNParser(opts=>$dsn_opts); +my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); +my $dbh = $sb->get_dbh_for('master'); + +if ( !$dbh ) { + plan skip_all => 'Cannot connect to sandbox master'; +} +else { + plan tests => 3; +} + +my $output; + +# ############################################################################# +# Issue 1152: mk-archiver columns option resulting in null archived table data +# ############################################################################# +$sb->load_file('master', 't/pt-archiver/samples/issue_1593265.sql'); + +$dbh->do('set names "utf8"'); + +$output = output( + sub { pt_archiver::main( + '--source', 'h=127.1,P=12345,D=test,t=t1,u=msandbox,p=msandbox', + '--dest', 't=t2', '--where', 'b in (1,2,3)') + }, +); + +my $untouched_rows = $dbh->selectall_arrayref('SELECT a, b FROM test.t1'); +is_deeply( + $untouched_rows, + [ ['10', '5'], ['10', '4'] ], + "Rows were left on the original table" +); + +my $new_rows = $dbh->selectall_arrayref('SELECT a, b FROM test.t2'); +is_deeply( + $new_rows, + [ ['10', '3'], ['10', '2'], ['10', '1'] ], + "Rows were archived into the new table" +); + +# ############################################################################# +# Done. +# ############################################################################# +$sb->wipe_clean($dbh); +ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox"); +exit; diff --git a/t/pt-archiver/samples/issue_1593265.sql b/t/pt-archiver/samples/issue_1593265.sql new file mode 100644 index 00000000..ac329ead --- /dev/null +++ b/t/pt-archiver/samples/issue_1593265.sql @@ -0,0 +1,13 @@ +CREATE DATABASE IF NOT EXISTS test; +USE test; +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; + +create table t1 (a int, b int, key (a),key(b)); +create table t2 like t1; + +insert into t1 (a,b) values (10,5); +insert into t1 (a,b) values (10,4); +insert into t1 (a,b) values (10,3); +insert into t1 (a,b) values (10,2); +insert into t1 (a,b) values (10,1); diff --git a/t/pt-table-checksum/bugs.t b/t/pt-table-checksum/bugs.t index 2d9fb00d..80701111 100644 --- a/t/pt-table-checksum/bugs.t +++ b/t/pt-table-checksum/bugs.t @@ -105,7 +105,7 @@ is_deeply( $rows, [ ['test', 'test_empty', '1', '0', '0'], # empty - ['test', 'test_full', '1', 'ac967054', '1'], # row + ['test', 'test_full', '1', '4d284606', '1'], # row ], "Bug 987393 (empty table): checksums" ) or print STDERR Dumper($rows); diff --git a/t/pt-table-checksum/chunk_index.t b/t/pt-table-checksum/chunk_index.t index 32fdfb1f..dc577fa4 100644 --- a/t/pt-table-checksum/chunk_index.t +++ b/t/pt-table-checksum/chunk_index.t @@ -36,7 +36,6 @@ my $out = "t/pt-table-checksum/samples/"; $sb->load_file('master', "t/pt-table-checksum/samples/issue_519.sql"); - ok( no_diff( sub { pt_table_checksum::main(@args, qw(-t issue_519.t --explain)) }, @@ -58,7 +57,7 @@ ok( no_diff( sub { pt_table_checksum::main(@args, qw(--chunk-index myidx), qw(-t issue_519.t --explain)) }, - "$out/chunkidx002.txt", + "$out/chunkidx002.txt", ), "Use --chunk-index" ); @@ -118,6 +117,7 @@ ok( # ############################################################################# $sb->load_file('master', "t/pt-table-checksum/samples/all-uc-table.sql"); my $exit_status = 0; + $output = output(sub { $exit_status = pt_table_checksum::main( $master_dsn, '--max-load', '', @@ -260,4 +260,5 @@ cmp_ok( # ############################################################################# $sb->wipe_clean($dbh); ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox"); + exit; diff --git a/t/pt-table-checksum/issue_1592608.t b/t/pt-table-checksum/issue_1592608.t new file mode 100644 index 00000000..788fac8e --- /dev/null +++ b/t/pt-table-checksum/issue_1592608.t @@ -0,0 +1,64 @@ +#!/usr/bin/env perl + +BEGIN { + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; +}; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use Test::More; + +use PerconaTest; +use Sandbox; +use SqlModes; +require "$trunk/bin/pt-table-checksum"; + +my $dp = new DSNParser(opts=>$dsn_opts); +my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); +my $dbh = $sb->get_dbh_for('master'); + +if ( !$dbh ) { + plan skip_all => 'Cannot connect to sandbox master'; +} +else { + plan tests => 2; +} + +# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic +# so we need to specify --set-vars innodb_lock_wait_timeout=3 else the tool will die. +# And --max-load "" prevents waiting for status variables. +my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox,D=bug_1592608'; +my @args = ($master_dsn, qw(--set-vars innodb_lock_wait_timeout=3), '--max-load', ''); +my $output; + +# We test that checksum works with invalid dates, +# but for that we need to turn off MySQL's NO_ZERO_IN_DATE mode +my $modes = new SqlModes($dbh, global=>1); +$modes->del('NO_ZERO_IN_DATE'); +$sb->load_file('master', 't/pt-table-checksum/samples/issue_1592608.sql'); +# ############################################################################# +# Issue 602: mk-table-checksum issue with invalid dates +# ############################################################################# + +#sub { pt_table_checksum::main(@args, qw(-t issue_1592608.t --tables t )) }, +$output = output( + sub { pt_table_checksum::main(@args, qw(-t t)) }, + stderr => 1, +); + +is( + PerconaTest::count_checksum_results($output, 'rows'), + 1, + "Large BLOB/TEXT/BINARY Checksum" +); + +$modes->restore_original_modes(); +# ############################################################################# +# Done. +# ############################################################################# +$sb->wipe_clean($dbh); +ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox"); +exit; diff --git a/t/pt-table-checksum/samples/chunkidx001.txt b/t/pt-table-checksum/samples/chunkidx001.txt index 62ca8d11..f12d5e29 100644 --- a/t/pt-table-checksum/samples/chunkidx001.txt +++ b/t/pt-table-checksum/samples/chunkidx001.txt @@ -2,7 +2,7 @@ -- issue_519.t -- -REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, `t`, CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`PRIMARY`) WHERE ((`i` >= ?)) AND ((`i` <= ?)) /*checksum chunk*/ +REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, CRC32(`t`), CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`PRIMARY`) WHERE ((`i` >= ?)) AND ((`i` <= ?)) /*checksum chunk*/ REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*), '0' FROM `issue_519`.`t` FORCE INDEX(`PRIMARY`) WHERE ((`i` < ?)) ORDER BY `i` /*past lower chunk*/ diff --git a/t/pt-table-checksum/samples/chunkidx002.txt b/t/pt-table-checksum/samples/chunkidx002.txt index 5b532454..9f462fe5 100644 --- a/t/pt-table-checksum/samples/chunkidx002.txt +++ b/t/pt-table-checksum/samples/chunkidx002.txt @@ -2,7 +2,7 @@ -- issue_519.t -- -REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, `t`, CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`myidx`) WHERE ((`i` > ?) OR (`i` = ? AND `y` >= ?)) AND ((`i` < ?) OR (`i` = ? AND `y` <= ?)) /*checksum chunk*/ +REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, CRC32(`t`), CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`myidx`) WHERE ((`i` > ?) OR (`i` = ? AND `y` >= ?)) AND ((`i` < ?) OR (`i` = ? AND `y` <= ?)) /*checksum chunk*/ REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*), '0' FROM `issue_519`.`t` FORCE INDEX(`myidx`) WHERE ((`i` < ?) OR (`i` = ? AND `y` < ?)) ORDER BY `i`, `y` /*past lower chunk*/ diff --git a/t/pt-table-checksum/samples/chunkidx003.txt b/t/pt-table-checksum/samples/chunkidx003.txt index ba0c7647..1fb3a2f5 100644 --- a/t/pt-table-checksum/samples/chunkidx003.txt +++ b/t/pt-table-checksum/samples/chunkidx003.txt @@ -2,7 +2,7 @@ -- issue_519.t -- -REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, `t`, CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`y`) WHERE ((`y` >= ?)) AND ((`y` <= ?)) /*checksum chunk*/ +REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, CRC32(`t`), CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`y`) WHERE ((`y` >= ?)) AND ((`y` <= ?)) /*checksum chunk*/ REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*), '0' FROM `issue_519`.`t` FORCE INDEX(`y`) WHERE ((`y` < ?)) ORDER BY `y` /*past lower chunk*/ diff --git a/t/pt-table-checksum/samples/issue_1592608.sql b/t/pt-table-checksum/samples/issue_1592608.sql new file mode 100644 index 00000000..4e5445a8 --- /dev/null +++ b/t/pt-table-checksum/samples/issue_1592608.sql @@ -0,0 +1,11 @@ +CREATE SCHEMA IF NOT EXISTS bug_1592608; +USE bug_1592608; +DROP TABLE IF EXISTS t; +CREATE TABLE t ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + a MEDIUMTEXT, + b MEDIUMBLOB +)engine=innodb; + +INSERT INTO t (a, b) VALUES (REPEAT('a', 2097152*2), CAST(REPEAT('a', 2097152*2) AS BINARY)); +