PT-2114 Incorrect casting of BIT columns by pt-archiver

- Improved fix for PT-2123, so it works with 5.7
- Fixed tests for PT-2114, so they work with 5.7

bin/pt-archiver

@@ -6097,6 +6097,9 @@ $Data::Dumper::Quotekeys = 0;
 use Percona::Toolkit;
 use constant PTDEBUG => $ENV{PTDEBUG} || 0;
 
+# We need SQL types in order to properly handle BIT columns in WHERE clause
+use DBI qw(:sql_types);
+
 # Global variables; as few as possible.
 my $oktorun   = 1;
 my $txn_cnt   = 0;
@@ -6600,7 +6603,9 @@ sub main {
                   . ($o->get('replace')    ? ' REPLACE'      : '')
                   . ($o->get('ignore')     ? ' IGNORE'       : '')
                   . " INTO TABLE $dst->{db_tbl}"
-                  . ($got_charset ? "CHARACTER SET $got_charset" : "")
+                  . ($got_charset ? "CHARACTER SET $got_charset" : 
+					 ( $src->{info}->{charset} && $src->{info}->{charset} =~ /utf/ ?
+				     "CHARACTER SET utf8mb4" : "" ))
                   . "("
                   . join(",", map { $q->quote($_) } @{$ins_stmt->{cols}} )
                   . ")";
@@ -6864,7 +6869,15 @@ sub main {
             }
             if ( !$o->get('no-delete') ) {
                my $success = do_with_retries($o, 'deleting', sub {
-                  $del_row->execute(@{$row}[@del_slice]);
+                  # We have to make exception for BIT column, see PT-2114
+                  for my $i (0 .. $#del_slice) {
+                     if ($src->{info}->{type_for}->{$del_stmt->{cols}[$del_slice[$i]]} eq 'bit') {
+	                    $del_row->bind_param($i + 1, oct('0b' . unpack('B*', @{$row}[$del_slice[$i]])), SQL_INTEGER);					 
+					 } else {
+					    $del_row->bind_param($i + 1, @{$row}[$del_slice[$i]]);
+				     }
+				  }
+				  $del_row->execute();
                   PTDEBUG && _d('Deleted', $del_row->rows, 'rows');
                   $statistics{DELETE} += $del_row->rows;
                });
@@ -6967,10 +6980,17 @@ sub main {
             }
             if ( !$o->get('no-delete') ) {
                my $success = do_with_retries($o, 'bulk_deleting', sub {
-                  $del_row->execute(
-                     @{$first_row}[@bulkdel_slice],
-                     @{$lastrow}[@bulkdel_slice],
-                  );
+                  # We have to make exception for BIT column, see PT-2114
+                  for my $i (0 .. $#bulkdel_slice) {
+                     if ($src->{info}->{type_for}->{$del_stmt->{cols}[$bulkdel_slice[$i]]} eq 'bit') {
+	                    $del_row->bind_param($i + 1, oct('0b' . unpack('B*', @{$first_row}[$bulkdel_slice[$i]])), SQL_INTEGER);					 
+	                    $del_row->bind_param($i + $#bulkdel_slice + 2, oct('0b' . unpack('B*', @{$lastrow}[$bulkdel_slice[$i]])), SQL_INTEGER);					 
+					 } else {
+					    $del_row->bind_param($i + 1, @{$first_row}[$bulkdel_slice[$i]]);
+					    $del_row->bind_param($i + $#bulkdel_slice + 2, @{$lastrow}[$bulkdel_slice[$i]]);
+				     }
+				  }
+                  $del_row->execute();
                   PTDEBUG && _d('Bulk deleted', $del_row->rows, 'rows');
                   $statistics{DELETE} += $del_row->rows;
                });

t/pt-archiver/pt-2114.t

@@ -0,0 +1,286 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+
+use PerconaTest;
+use Sandbox;
+require "$trunk/bin/pt-archiver";
+
+my $dp  = new DSNParser(opts=>$dsn_opts);
+my $sb  = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $dbh = $sb->get_dbh_for('master');
+
+if ( !$dbh ) {
+   plan skip_all => 'Cannot connect to sandbox master';
+}
+else {
+   plan tests => 23;
+}
+
+my $output;
+
+# #############################################################################
+# PT-2114: Incorrect casting of BIT columns by pt-archiver 
+# #############################################################################
+$sb->load_file('master', 't/pt-archiver/samples/pt-2114.sql');
+
+my $zero_rows = $dbh->selectall_arrayref('select id, hex(val) from pt_2114.t1 where val = 0');
+my $exit_status;
+
+$output = output(
+   sub { $exit_status = pt_archiver::main(
+      '--source',  'h=127.1,P=12345,D=pt_2114,t=t1,u=msandbox,p=msandbox,A=utf8mb4',
+      '--where', '(val) in (select a.val from pt_2114.t1_tmp a where id =2)', 
+	  '--purge')
+   },
+);
+
+is (
+    $exit_status,
+    0,
+    "PT-2114 exit status OK",
+);
+
+my $left_rows = $dbh->selectall_arrayref('select id, hex(val) from pt_2114.t1');
+
+is_deeply(
+   $zero_rows,
+   $left_rows,
+   "PT-2114 Only rows with val=0 left in the table"
+);
+
+my $count_rows = $dbh->selectrow_arrayref('select count(*) from pt_2114.t1');
+
+is (
+   @{$count_rows}[0],
+   4,
+   "PT-2114 Four rows left in the table"
+);
+
+# #############################################################################
+# Reloading dump to perform archiving
+# #############################################################################
+$sb->load_file('master', 't/pt-archiver/samples/pt-2114.sql');
+
+my $one_rows = $dbh->selectall_arrayref('select id, hex(val) from pt_2114.t1 where val = 1');
+
+$output = output(
+   sub { $exit_status = pt_archiver::main(
+      '--source',  'h=127.1,P=12345,D=pt_2114,t=t1,u=msandbox,p=msandbox,A=utf8mb4',
+      '--dest',  'h=127.1,P=12345,D=pt_2114,t=t2,u=msandbox,p=msandbox,A=utf8mb4',
+      '--where', '(val) in (select a.val from pt_2114.t1_tmp a where id =2)', 
+	  )
+   },
+);
+
+is (
+    $exit_status,
+    0,
+    "PT-2114 exit status OK",
+);
+
+$left_rows = $dbh->selectall_arrayref('select id, hex(val) from pt_2114.t1');
+
+is_deeply(
+   $zero_rows,
+   $left_rows,
+   "PT-2114 Only rows with val=0 left in the table"
+);
+
+$count_rows = $dbh->selectrow_arrayref('select count(*) from pt_2114.t1');
+
+is (
+   @{$count_rows}[0],
+   4,
+   "PT-2114 Four rows left in the table"
+);
+
+my $archived_rows = $dbh->selectall_arrayref('select id, hex(val) from pt_2114.t2');
+
+is_deeply(
+   $one_rows,
+   $archived_rows,
+   "PT-2114 Correct rows archived"
+);
+
+# #############################################################################
+# Reloading dump to perform archiving
+# #############################################################################
+$sb->load_file('master', 't/pt-archiver/samples/pt-2114.sql');
+
+$output = output(
+   sub { $exit_status = pt_archiver::main(
+      '--source',  'h=127.1,P=12345,D=pt_2114,t=t1,u=msandbox,p=msandbox,A=utf8mb4,L=yes',
+      '--dest',  'h=127.1,P=12345,D=pt_2114,t=t2,u=msandbox,p=msandbox,A=utf8mb4,L=yes',
+      '--where', '(val) in (select a.val from pt_2114.t1_tmp a where id =2)', 
+	  '--bulk-insert', '--limit', '10')
+   },
+);
+
+is (
+    $exit_status,
+    0,
+    "PT-2114 exit status OK",
+);
+
+$left_rows = $dbh->selectall_arrayref('select id, hex(val) from pt_2114.t1');
+
+is_deeply(
+   $zero_rows,
+   $left_rows,
+   "PT-2114 Only rows with val=0 left in the table with --bulk-insert"
+);
+
+$count_rows = $dbh->selectrow_arrayref('select count(*) from pt_2114.t1');
+
+is (
+   @{$count_rows}[0],
+   4,
+   "PT-2114 Four rows left in the table"
+);
+
+$archived_rows = $dbh->selectall_arrayref('select id, hex(val) from pt_2114.t2');
+
+is_deeply(
+   $one_rows,
+   $archived_rows,
+   "PT-2114 Correct rows archived with --bulk-insert"
+);
+
+# #############################################################################
+# Reloading dump to perform archiving
+# #############################################################################
+$sb->load_file('master', 't/pt-archiver/samples/pt-2114.sql');
+
+$output = output(
+   sub { $exit_status = pt_archiver::main(
+      '--source',  'h=127.1,P=12345,D=pt_2114,t=t1,u=msandbox,p=msandbox,A=utf8mb4,L=yes',
+      '--where', '(val) in (select a.val from pt_2114.t1_tmp a where id =2)', 
+	  '--bulk-delete', '--purge', '--limit', '10')
+   },
+);
+
+is (
+    $exit_status,
+    0,
+    "PT-2114 exit status OK",
+);
+
+$left_rows = $dbh->selectall_arrayref('select id, hex(val) from pt_2114.t1');
+
+is_deeply(
+   $zero_rows,
+   $left_rows,
+   "PT-2114 Only rows with val=0 left in the table with --bulk-delete"
+);
+
+$count_rows = $dbh->selectrow_arrayref('select count(*) from pt_2114.t1');
+
+is (
+   @{$count_rows}[0],
+   4,
+   "PT-2114 Four rows left in the table"
+);
+
+# #############################################################################
+# Reloading dump to perform archiving
+# #############################################################################
+$sb->load_file('master', 't/pt-archiver/samples/pt-2114.sql');
+
+# Archiving into a file
+$output = output(
+   sub { $exit_status = pt_archiver::main(
+      '--where', '(val) in (select a.val from pt_2114.t1_tmp a where id =2)', 
+      '--source',  'h=127.1,P=12345,D=pt_2114,t=t1,u=msandbox,p=msandbox,A=utf8mb4,L=yes',
+      '--file',  'archive.%D.%t', '-c', 'id'
+  )
+   },
+);
+
+is (
+    $exit_status,
+    0,
+    "PT-2114 exit status OK",
+);
+
+ok(-f 'archive.pt_2114.t1', 'PT-2114 Archive file written OK');
+
+$output = `cat archive.pt_2114.t1`;
+is($output, <<EOF
+123
+125
+128
+130
+EOF
+, 'PT-2114 Correct rows archived into the file');
+`rm -f archive.pt_2114.t1`;
+
+$left_rows = $dbh->selectall_arrayref('select id, hex(val) from pt_2114.t1');
+
+is_deeply(
+   $zero_rows,
+   $left_rows,
+   "PT-2114 Only rows with val=0 left in the table after archiving into the file"
+);
+
+$count_rows = $dbh->selectrow_arrayref('select count(*) from pt_2114.t1');
+
+is (
+   @{$count_rows}[0],
+   4,
+   "PT-2114 Four rows left in the table"
+);
+
+# #############################################################################
+# Longer BIT values
+# Loading dump to perform archiving
+# #############################################################################
+$sb->load_file('master', 't/pt-archiver/samples/pt-2114-2.sql');
+my $not_archived_rows = $dbh->selectall_arrayref("select id, hex(val) from pt_2114.t1 where val = b'1111000010'");
+
+$output = output(
+   sub { $exit_status = pt_archiver::main(
+      '--source',  'h=127.1,P=12345,D=pt_2114,t=t1,u=msandbox,p=msandbox,A=utf8mb4',
+      '--where', '(val) in (select a.val from pt_2114.t1_tmp a where id =2)', 
+	  '--purge')
+   },
+);
+
+is (
+    $exit_status,
+    0,
+    "PT-2114 exit status OK",
+);
+
+$left_rows = $dbh->selectall_arrayref('select id, hex(val) from pt_2114.t1');
+
+is_deeply(
+   $not_archived_rows,
+   $left_rows,
+   "PT-2114 Only rows with val=0 left in the table"
+);
+
+$count_rows = $dbh->selectrow_arrayref('select count(*) from pt_2114.t1');
+
+is (
+   @{$count_rows}[0],
+   4,
+   "PT-2114 Four rows left in the table"
+);
+
+
+# #############################################################################
+# Done.
+# #############################################################################
+$sb->wipe_clean($dbh);
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+exit;

t/pt-archiver/samples/pt-2114-2.sql

@@ -0,0 +1,19 @@
+DROP DATABASE IF EXISTS pt_2114;
+CREATE DATABASE pt_2114;
+USE pt_2114;
+CREATE TABLE `pt_2114`.`t1` (
+`id` int NOT NULL AUTO_INCREMENT,
+`val` bit(10) NOT NULL DEFAULT b'0',
+PRIMARY KEY (`id`,`val`)
+) ENGINE=InnoDB AUTO_INCREMENT=122 DEFAULT CHARSET=utf8mb4;
+CREATE TABLE pt_2114.t1_tmp LIKE t1;
+INSERT INTO pt_2114.t1 (val) VALUES (b'1111000010'),(b'1101001100');
+INSERT INTO pt_2114.t1 (val) SELECT val FROM pt_2114.t1 ;
+INSERT INTO pt_2114.t1 (val) SELECT val FROM pt_2114.t1 ;
+INSERT INTO pt_2114.t1_tmp SELECT NULL, val FROM pt_2114.t1;
+
+CREATE TABLE `pt_2114`.`t2` (
+`id` int(11) NOT NULL AUTO_INCREMENT,
+`val` bit(1) NOT NULL DEFAULT b'0',
+PRIMARY KEY (`id`,`val`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

t/pt-archiver/samples/pt-2114.sql

@@ -0,0 +1,19 @@
+DROP DATABASE IF EXISTS pt_2114;
+CREATE DATABASE pt_2114;
+USE pt_2114;
+CREATE TABLE `pt_2114`.`t1` (
+`id` int(11) NOT NULL AUTO_INCREMENT,
+`val` bit(1) NOT NULL DEFAULT b'0',
+PRIMARY KEY (`id`,`val`)
+) ENGINE=InnoDB AUTO_INCREMENT=122 DEFAULT CHARSET=utf8mb4;
+CREATE TABLE pt_2114.t1_tmp LIKE t1;
+INSERT INTO pt_2114.t1 (val) VALUES (0),(1);
+INSERT INTO pt_2114.t1 (val) SELECT val FROM pt_2114.t1 ;
+INSERT INTO pt_2114.t1 (val) SELECT val FROM pt_2114.t1 ;
+INSERT INTO pt_2114.t1_tmp SELECT NULL, val FROM pt_2114.t1;
+
+CREATE TABLE `pt_2114`.`t2` (
+`id` int(11) NOT NULL AUTO_INCREMENT,
+`val` bit(1) NOT NULL DEFAULT b'0',
+PRIMARY KEY (`id`,`val`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

t/pt-deadlock-logger/standard_options.t

@@ -82,8 +82,6 @@ $sb->load_file('master', 't/pt-deadlock-logger/samples/deadlocks_tbl.sql', 'test
 
 $output = `$trunk/bin/pt-deadlock-logger $dsn --dest D=test,t=deadlocks --daemonize --run-time 10 --interval 1 --pid $pid_file 1>/dev/null 2>/dev/null`;
 
-#REMOVEME
-`echo "test 3" >>/tmp/REMOVEME`;
 PerconaTest::wait_for_files($pid_file);
 
 $output = `ps x | grep 'pt-deadlock-logger $dsn' | grep -v grep`;
@@ -93,8 +91,6 @@ like(
    'It lives daemonized'
 ) or diag($output);
 
-#REMOVEME
-`echo "test 4" >>/tmp/REMOVEME`;
 my ($pid) = $output =~ /(\d+)/;
 
 ok(

t/pt-table-checksum/fnv_64.t

@@ -37,8 +37,6 @@ $sb->create_dbs($master_dbh, [qw(test)]);
 eval { $master_dbh->do('DROP FUNCTION IF EXISTS fnv_64'); };
 eval { $master_dbh->do("CREATE FUNCTION fnv_64 RETURNS INTEGER SONAME 'libfnv_udf.so';"); };
 if ( $EVAL_ERROR ) {
-#REMOVEME
-print  $EVAL_ERROR;
    chomp $EVAL_ERROR;
    plan skip_all => "No FNV_64 UDF lib"
 }

t/pt-table-checksum/pt-226.t

@@ -54,8 +54,6 @@ $output = output(
    stderr => 1,
 );
 
-#REMOVEME
-diag($exit_status);
 isnt(
    $exit_status,
    0,