Merge branch '2.2' into bug-984057

bin/pt-archiver

@@ -3201,7 +3201,7 @@ sub generate_del_stmt {
    my $index = $tp->find_best_index($tbl, $args{index});
    die "Cannot find an ascendable index in table" unless $index;
 
-   if ( $index ) {
+   if ( $index && $tbl->{keys}->{$index}->{is_unique}) {
       @del_cols = @{$tbl->{keys}->{$index}->{cols}};
    }
    else {

bin/pt-online-schema-change

@@ -3053,7 +3053,7 @@ sub generate_del_stmt {
    my $index = $tp->find_best_index($tbl, $args{index});
    die "Cannot find an ascendable index in table" unless $index;
 
-   if ( $index ) {
+   if ( $index && $tbl->{keys}->{$index}->{is_unique}) {
       @del_cols = @{$tbl->{keys}->{$index}->{cols}};
    }
    else {

bin/pt-table-checksum

@@ -4901,7 +4901,7 @@ sub generate_del_stmt {
    my $index = $tp->find_best_index($tbl, $args{index});
    die "Cannot find an ascendable index in table" unless $index;
 
-   if ( $index ) {
+   if ( $index && $tbl->{keys}->{$index}->{is_unique}) {
       @del_cols = @{$tbl->{keys}->{$index}->{cols}};
    }
    else {
@@ -5885,6 +5885,9 @@ sub get_checksum_columns {
          elsif ( $trim && $type =~ m/varchar/ ) {
             $result = "TRIM($result)";
          }
+         elsif ( $type =~ m/blob|text|binary/ ) {
+            $result = "CRC32($result)";
+         }
          $result;
       }
       grep {

bin/pt-table-sync

@@ -4862,6 +4862,9 @@ sub make_row_checksum {
          elsif ( $args{trim} && $type =~ m/varchar/ ) {
             $result = "TRIM($result)";
          }
+         elsif ( $type =~ m/binary|text|blob/ ) {
+            $result = "CRC32($result)";
+         }
          $result;
       }
       grep {
@@ -6480,7 +6483,7 @@ sub generate_del_stmt {
    my $index = $tp->find_best_index($tbl, $args{index});
    die "Cannot find an ascendable index in table" unless $index;
 
-   if ( $index ) {
+   if ( $index && $tbl->{keys}->{$index}->{is_unique}) {
       @del_cols = @{$tbl->{keys}->{$index}->{cols}};
    }
    else {

lib/RowChecksum.pm

@@ -224,6 +224,9 @@ sub get_checksum_columns {
          elsif ( $trim && $type =~ m/varchar/ ) {
             $result = "TRIM($result)";
          }
+         elsif ( $type =~ m/blob|text|binary/ ) {
+            $result = "CRC32($result)";
+         }
          $result;
       }
       grep {

lib/TableNibbler.pm

@@ -247,7 +247,7 @@ sub generate_del_stmt {
    die "Cannot find an ascendable index in table" unless $index;
 
    # These are the columns needed for the DELETE statement's WHERE clause.
-   if ( $index ) {
+   if ( $index && $tbl->{keys}->{$index}->{is_unique}) {
       @del_cols = @{$tbl->{keys}->{$index}->{cols}};
    }
    else {

sandbox/servers/5.6/my.sandbox.cnf

@@ -15,7 +15,7 @@ innodb_buffer_pool_size    = 64M
 innodb_data_home_dir       = /tmp/PORT/data
 innodb_log_group_home_dir  = /tmp/PORT/data
 innodb_data_file_path      = ibdata1:10M:autoextend
-innodb_log_file_size       = 5M
+innodb_log_file_size       = 64M
 log-bin                    = mysql-bin
 relay_log                  = mysql-relay-bin
 log_slave_updates

t/lib/TableNibbler.t

@@ -134,11 +134,13 @@ is_deeply(
       cols   => [qw(film_id)],
    ),
    {
-      cols  => [qw(film_id title)],
+      cols  => [qw(film_id title description release_year language_id original_language_id rental_duration rental_rate length replacement_cost rating special_features last_update)],
       index => 'idx_title',
-      where => '(`title` = ?)',
+      where => '(`film_id` = ? AND `title` = ? AND ((? IS NULL AND `description` IS NULL) OR (`description` = ?)) AND ((? IS NULL AND `release_year` IS NULL) OR (`release_year` = ?)) AND `language_id` = ? AND ((? IS NULL AND `original_language_id` IS NULL) OR (`original_language_id` = ?)) AND `rental_duration` = ? AND `rental_rate` = ? AND ((? IS NULL AND `length` IS NULL) OR (`length` = ?)) AND `replacement_cost` = ? AND ((? IS NULL AND `rating` IS NULL) OR (`rating` = ?)) AND ((? IS NULL AND `special_features` IS NULL) OR (`special_features` = ?)) AND `last_update` = ?)',
-      slice => [1],
+      slice => [ 0, 1, 2, 2, 3, 3, 4, 5, 5, 6, 7, 8, 8, 9, 10, 10, 11, 11, 12 ],
-      scols => [qw(title)],
+      scols  => [qw( film_id title description description release_year release_year language_id original_language_id original_language_id
+                     rental_duration rental_rate length length replacement_cost rating rating special_features special_features last_update)],
+
    },
    'del stmt on sakila.film with different index and extra column',
 );

t/pt-archiver/issue_1166.t

@@ -56,7 +56,7 @@ $output = output(
 );
 like(
    $output,
-   qr/DELETE FROM `test`\.`issue_1166` WHERE \(`id` = \?\) LIMIT 1$/m,
+   qr/LIMIT 1$/m,
    "LIMIT 1 with non-unique index (issue 1166)"
 );
 

t/pt-archiver/issue_1593265.t

@@ -0,0 +1,64 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+
+use PerconaTest;
+use Sandbox;
+require "$trunk/bin/pt-archiver";
+
+my $dp  = new DSNParser(opts=>$dsn_opts);
+my $sb  = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $dbh = $sb->get_dbh_for('master');
+
+if ( !$dbh ) {
+   plan skip_all => 'Cannot connect to sandbox master';
+}
+else {
+   plan tests => 3;
+}
+
+my $output;
+
+# #############################################################################
+# Issue 1152: mk-archiver columns option resulting in null archived table data
+# #############################################################################
+$sb->load_file('master', 't/pt-archiver/samples/issue_1593265.sql');
+
+$dbh->do('set names "utf8"');
+
+$output = output(
+   sub { pt_archiver::main(
+      '--source',  'h=127.1,P=12345,D=test,t=t1,u=msandbox,p=msandbox',
+      '--dest', 't=t2', '--where', 'b in (1,2,3)')
+   },
+);
+
+my $untouched_rows = $dbh->selectall_arrayref('SELECT a, b FROM test.t1');
+is_deeply(
+   $untouched_rows,
+   [ ['10', '5'], ['10', '4'] ],
+   "Rows were left on the original table"
+);
+
+my $new_rows = $dbh->selectall_arrayref('SELECT a, b FROM test.t2');
+is_deeply(
+   $new_rows,
+   [ ['10', '3'], ['10', '2'], ['10', '1'] ],
+   "Rows were archived into the new table"
+);
+
+# #############################################################################
+# Done.
+# #############################################################################
+$sb->wipe_clean($dbh);
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+exit;

t/pt-archiver/samples/issue_1593265.sql

@@ -0,0 +1,13 @@
+CREATE DATABASE IF NOT EXISTS test;
+USE test;
+DROP TABLE IF EXISTS t1;
+DROP TABLE IF EXISTS t2;
+
+create table t1 (a int, b int, key (a),key(b));
+create table t2 like t1;
+
+insert into t1 (a,b) values (10,5);
+insert into t1 (a,b) values (10,4);
+insert into t1 (a,b) values (10,3);
+insert into t1 (a,b) values (10,2);
+insert into t1 (a,b) values (10,1);

t/pt-table-checksum/bugs.t

@@ -105,7 +105,7 @@ is_deeply(
    $rows,
    [
       ['test', 'test_empty', '1', '0',        '0'],  # empty
-      ['test', 'test_full',  '1', 'ac967054', '1'],  # row
+      ['test', 'test_full',  '1', '4d284606', '1'],  # row
    ],
    "Bug 987393 (empty table): checksums"
 ) or print STDERR Dumper($rows);

t/pt-table-checksum/chunk_index.t

@@ -36,7 +36,6 @@ my $out        = "t/pt-table-checksum/samples/";
 
 $sb->load_file('master', "t/pt-table-checksum/samples/issue_519.sql");
 
-
 ok(
    no_diff(
       sub { pt_table_checksum::main(@args, qw(-t issue_519.t --explain)) },
@@ -58,7 +57,7 @@ ok(
    no_diff(
       sub { pt_table_checksum::main(@args, qw(--chunk-index myidx),
          qw(-t issue_519.t --explain)) },
-      "$out/chunkidx002.txt",
+      "$out/chunkidx002.txt", 
    ),
    "Use --chunk-index"
 );
@@ -118,6 +117,7 @@ ok(
 # #############################################################################
 $sb->load_file('master', "t/pt-table-checksum/samples/all-uc-table.sql");
 my $exit_status = 0;
+
 $output = output(sub {
    $exit_status = pt_table_checksum::main(
       $master_dsn, '--max-load', '',
@@ -260,4 +260,5 @@ cmp_ok(
 # #############################################################################
 $sb->wipe_clean($dbh);
 ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+
 exit;

t/pt-table-checksum/issue_1592608.t

@@ -0,0 +1,64 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+
+use PerconaTest;
+use Sandbox;
+use SqlModes;
+require "$trunk/bin/pt-table-checksum";
+
+my $dp  = new DSNParser(opts=>$dsn_opts);
+my $sb  = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $dbh = $sb->get_dbh_for('master');
+
+if ( !$dbh ) {
+   plan skip_all => 'Cannot connect to sandbox master';
+}
+else {
+   plan tests => 2;
+}
+
+# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
+# so we need to specify --set-vars innodb_lock_wait_timeout=3 else the tool will die.
+# And --max-load "" prevents waiting for status variables.
+my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox,D=bug_1592608';
+my @args       = ($master_dsn, qw(--set-vars innodb_lock_wait_timeout=3), '--max-load', ''); 
+my $output;
+
+# We test that checksum works with invalid dates, 
+# but for that we need to turn off MySQL's NO_ZERO_IN_DATE mode 
+my $modes = new SqlModes($dbh, global=>1);
+$modes->del('NO_ZERO_IN_DATE');
+$sb->load_file('master', 't/pt-table-checksum/samples/issue_1592608.sql');
+# #############################################################################
+# Issue 602: mk-table-checksum issue with invalid dates
+# #############################################################################
+
+#sub { pt_table_checksum::main(@args, qw(-t issue_1592608.t --tables t )) },
+$output = output(
+   sub { pt_table_checksum::main(@args, qw(-t t)) },
+   stderr => 1,
+);
+
+is(
+   PerconaTest::count_checksum_results($output, 'rows'),
+   1,
+   "Large BLOB/TEXT/BINARY Checksum"
+);
+
+$modes->restore_original_modes();
+# #############################################################################
+# Done.
+# #############################################################################
+$sb->wipe_clean($dbh);
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+exit;

t/pt-table-checksum/samples/chunkidx001.txt

@@ -2,7 +2,7 @@
 -- issue_519.t
 --
 
-REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, `t`, CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`PRIMARY`) WHERE ((`i` >= ?)) AND ((`i` <= ?)) /*checksum chunk*/
+REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, CRC32(`t`), CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`PRIMARY`) WHERE ((`i` >= ?)) AND ((`i` <= ?)) /*checksum chunk*/
 
 REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*), '0' FROM `issue_519`.`t` FORCE INDEX(`PRIMARY`) WHERE ((`i` < ?)) ORDER BY `i` /*past lower chunk*/
 

t/pt-table-checksum/samples/chunkidx002.txt

@@ -2,7 +2,7 @@
 -- issue_519.t
 --
 
-REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, `t`, CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`myidx`) WHERE ((`i` > ?) OR (`i` = ? AND `y` >= ?)) AND ((`i` < ?) OR (`i` = ? AND `y` <= ?)) /*checksum chunk*/
+REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, CRC32(`t`), CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`myidx`) WHERE ((`i` > ?) OR (`i` = ? AND `y` >= ?)) AND ((`i` < ?) OR (`i` = ? AND `y` <= ?)) /*checksum chunk*/
 
 REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*), '0' FROM `issue_519`.`t` FORCE INDEX(`myidx`) WHERE ((`i` < ?) OR (`i` = ? AND `y` < ?)) ORDER BY `i`, `y` /*past lower chunk*/
 

t/pt-table-checksum/samples/chunkidx003.txt

@@ -2,7 +2,7 @@
 -- issue_519.t
 --
 
-REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, `t`, CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`y`) WHERE ((`y` >= ?)) AND ((`y` <= ?)) /*checksum chunk*/
+REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `y`, CRC32(`t`), CONCAT(ISNULL(`t`)))) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `issue_519`.`t` FORCE INDEX(`y`) WHERE ((`y` >= ?)) AND ((`y` <= ?)) /*checksum chunk*/
 
 REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*), '0' FROM `issue_519`.`t` FORCE INDEX(`y`) WHERE ((`y` < ?)) ORDER BY `y` /*past lower chunk*/
 

t/pt-table-checksum/samples/issue_1592608.sql

@@ -0,0 +1,11 @@
+CREATE SCHEMA IF NOT EXISTS bug_1592608;
+USE bug_1592608;
+DROP TABLE IF EXISTS t;
+CREATE TABLE t (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, 
+  a  MEDIUMTEXT, 
+  b  MEDIUMBLOB
+)engine=innodb;
+
+INSERT INTO t (a, b) VALUES (REPEAT('a', 2097152*2), CAST(REPEAT('a', 2097152*2) AS BINARY));
+