Merge skip-load-data-file

bin/pt-archiver

@@ -3955,24 +3955,8 @@ sub main {
          $dp->get_cxn_params($table), { AutoCommit => $ac });
       PTDEBUG && _d('Inspecting table on', $dp->as_string($table));
 
-      if ( $o->get('bulk-insert') ) {
-         local $@;
-         my $sql = "LOAD DATA LOCAL INFILE '/dev/null' INTO TABLE "
-                 . "`test`.`pt_not_there`";
-         eval { $dbh->do($sql); 1 } or do {
-            my $e = $@;
-            my $error_re = qr/\QDBD::mysql::db do failed: The used command is not allowed with this MySQL version [for Statement "LOAD DATA LOCAL INFILE/;
-            if ($e =~ $error_re) {
-               $dbh->disconnect();
-               die("--bulk-insert cannot work as LOAD DATA LOCAL INFILE "
-                 . "is disabled. See http://kb.percona.com/troubleshoot-load-data-infile"
-               );
-            }
-         };
-      }
-      
-      # Set options that can enable removing data on the master and archiving it
-      # on the slaves.
+      # Set options that can enable removing data on the master
+      # and archiving it on the slaves.
       if ( $table->{a} ) {
          $dbh->do("USE $table->{a}");
       }

bin/pt-upgrade

@@ -10371,32 +10371,6 @@ sub main {
       $host->{name}  = $name || 'unknown host';
    }
 
-   # ########################################################################
-   # If we're comparing rows, check that LOAD DATA LOCAL INFILE works for,
-   # all hosts, or bail out early if it doesn't.
-   # ########################################################################
-   my $compare = $o->get('compare');
-   if ( $compare->{results}
-         && lc($o->get('compare-results-method')) eq 'rows' )
-   {
-      foreach my $host ( @$hosts ) {
-         local $@;
-         my $sql = "LOAD DATA LOCAL INFILE '/dev/null' INTO TABLE "
-                 . "`test`.`pt_not_there`";
-         eval { $host->{dbh}->do($sql); 1 } or do {
-            my $e = $@;
-            my $error_re = qr/\QDBD::mysql::db do failed: The used command is not allowed with this MySQL version [for Statement "LOAD DATA LOCAL INFILE/;
-            if ($e =~ $error_re) {
-               $_->{dbh}->disconnect() for @$hosts;
-               die("Cannot compare rows as LOAD DATA LOCAL INFILE "
-                 . "is disabled for $host->{name}. See "
-                 . "http://kb.percona.com/troubleshoot-load-data-infile"
-               );
-            }
-         };
-      }
-   }
-   
    # ########################################################################
    # Make some common modules.
    # ########################################################################
@@ -10430,6 +10404,7 @@ sub main {
    # ########################################################################
    # Make compare modules in order.
    # ########################################################################
+   my $compare = $o->get('compare');
    my @compare_modules;
    if ( $compare->{results} ) {
       my $method  = lc $o->get('compare-results-method');

lib/PerconaTest.pm

@@ -68,6 +68,7 @@ our @EXPORT      = qw(
    $trunk
    $dsn_opts
    $sandbox_version
+   $can_load_data
 );
 
 our $trunk = $ENV{PERCONA_TOOLKIT_BRANCH};
@@ -78,6 +79,8 @@ eval {
    $sandbox_version = $v if $v;
 };
 
+our $can_load_data = can_load_data();
+
 our $dsn_opts = [
    {
       key  => 'A',
@@ -777,17 +780,9 @@ sub tables_used {
    return [ sort keys %tables ];
 }
 
-sub load_data_is_disabled {
-    my ($dbh) = @_;
-    my $sql = "LOAD DATA LOCAL INFILE '/dev/null' INTO TABLE "
-            . "`t`.`pt_not_there`";
-    local $@;
-    if (!eval { $dbh->do($sql); 1 } ) {
-        my $e = $@;
-        return 1 if $e =~ qr/\QDBD::mysql::db do failed: The used command is not allowed with this MySQL version [for Statement "LOAD DATA LOCAL INFILE/;
-    }
-
-    return;
+sub can_load_data {
+    my $output = `/tmp/12345/use -e "SELECT * FROM percona_test.load_data" 2>/dev/null`;
+    return ($output || '') =~ /42/;
 }
 
 1;

sandbox/servers/start

@@ -41,18 +41,21 @@ $BASEDIR/bin/mysqld_safe --defaults-file=/tmp/PORT/my.sandbox.cnf > /dev/null 2>
 cd $PWD
 
 echo -n "Starting MySQL test server on port PORT... "
-for i in 1 2 3 4 5 6 7 8 9 10; do
-   sleep 1
-   if [ -f $PIDFILE ] && [ -S $SOCKETFILE ]; then
+for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do
+   if [ -f $PIDFILE -a  -S $SOCKETFILE -a -s "/tmp/PORT/data/ibdata1" ]; then
       break
    fi
+   sleep 1
 done
 
-sandbox_is_alive
-if [ $? -eq 1 ]; then
-   echo "OK"
-   exit 0
-else
-   echo "FAILED"
-   exit 1
-fi
+for i in 1 2 3 4 5; do
+   sandbox_is_alive
+   if [ $? -eq 1 ]; then
+      echo "OK"
+      exit 0
+   fi
+   sleep 1
+done
+
+echo "FAILED"
+exit 1

sandbox/start-sandbox

@@ -9,6 +9,23 @@ die() {
    exit 1
 }
 
+debug_sandbox() {
+   local port="$1"
+   echo
+   echo "MySQL processes:" >&2
+   ps x | grep mysql >&2
+   echo
+   if [ -d "/tmp/$port" ]; then
+      ls -lh /tmp/$port/* >&2
+      echo
+      cat /tmp/$port/data/mysqld.log >&2
+      echo
+      tail -n 100 /tmp/$port/data/genlog >&2
+   else
+      echo "/tmp/$port does not exist" >&2
+   fi
+}
+
 make_sandbox() {
    # Make the sandbox dir and extract the base files.
    rm -rf /tmp/$port || die "can't rm /tmp/$port"
@@ -67,14 +84,14 @@ make_sandbox() {
          /tmp/$port/use -e 'SHOW /*!40100 ENGINE*/ INNODB STATUS' | grep 'INNODB MONITOR OUTPUT' >/dev/null 2>&1
          # grep exits 0 if lines are found
          if [ $? -ne 0 ]; then
-            echo "****** WARNING sandbox doesn't have a working InnoDB! ******" >&2
-            cat /tmp/$port/data/mysqld.log >&2
+            echo "Sandbox $type $port doesn't have InnoDB" >&2
+            debug_sandbox $port
             exit 1
          fi
       fi
    else
       echo "Sandbox $type $port failed to start." >&2
-      cat /tmp/$port/data/mysqld.log >&2
+      debug_sandbox $port
       exit 1
    fi
 

sandbox/test-env

@@ -7,17 +7,29 @@
 # Exist 0 on success/no errors, or 1 on any warnings or errors.
 
 err() {
-   echo "MySQL processes:" >&2
-   ps x | grep mysql >&2
-
-   echo "Sandbox servers:" >&2
-   ls /tmp/1234? >&2
-   ls /tmp/290? >&2
-
-   echo
    for msg; do
       echo "$msg" >&2
    done
+
+   if [ "$DEBUG_SANDBOXES" ]; then 
+      echo
+      echo "MySQL processes:" >&2
+      ps x | grep mysql >&2
+
+      echo
+      for p in 12345 12346 12347; do
+         echo "Sandbox $p:" >&2
+         if [ -d "/tmp/$p" ]; then
+            ls -lh /tmp/$p/* >&2
+            echo
+            cat /tmp/$p/data/mysqld.log >&2
+            echo
+            tail -n 100 /tmp/$p/data/genlog >&2
+         else
+            echo "/tmp/$p does not exist" >&2
+         fi
+      done
+   fi
 }
 
 usage() {
@@ -289,7 +301,16 @@ case $opt in
             else
                echo "OK"
             fi
+
+            # Create percona_test db and checksum all the tables.
             ../util/checksum-test-dataset
+
+            # LOAD DATA is disabled or broken on some boxes.
+            # PerconaTest exports $can_load_data which is true
+            # if percona_test.load_data has the 42 row,
+            # signaling that LOAD DATA LOCAL INFILE worked.
+            ../util/check-load-data
+
             ping=$(/tmp/12345/use -ss -e "SELECT MD5(RAND())")
             /tmp/12345/use -e "create table percona_test.sentinel(id int primary key, ping varchar(64) not null default '')"
             /tmp/12345/use -e "insert into percona_test.sentinel(id, ping) values(1, '$ping')";
@@ -306,6 +327,7 @@ case $opt in
       if [ $exit_status -eq 0 ]; then
          echo "Percona Toolkit test environment started with MySQL v$MYSQL_VERSION."
       else
+         DEBUG_SANDBOXES=1
          err "There was an error starting the Percona Toolkit test environment."
       fi
       ;;
@@ -318,6 +340,7 @@ case $opt in
       if [ $exit_status -eq 0 ]; then
          echo "Percona Toolkit test environment stopped."
       else
+         DEBUG_SANDBOXES=1
          err "Error stopping the Percona Toolkit test environment."
       fi
       ;;

t/lib/CompareResults.t

@@ -42,9 +42,6 @@ if ( !$dbh1 ) {
 elsif ( !$dbh2 ) {
    plan skip_all => "Cannot connect to sandbox slave";
 }
-else {
-   plan tests => 57;
-}
 
 Transformers->import(qw(make_checksum));
 
@@ -306,8 +303,8 @@ is_deeply(
 # #############################################################################
 my $tmpdir = '/tmp/mk-upgrade-res';
 SKIP: {
-   skip "LOAD DATA LOCAL INFILE is disabled, can't test method => rows", 30
-      if PerconaTest::load_data_is_disabled($dbh1);
+   skip "LOAD DATA LOCAL INFILE is disabled", 30 unless $can_load_data;
+
 diag(`rm -rf $tmpdir 2>/dev/null; mkdir $tmpdir`);
 
 $sb->load_file('master', "t/lib/samples/compare-results.sql");
@@ -684,6 +681,7 @@ is(
    'rows: report, left with more rows'
 );
 }
+
 # #############################################################################
 # Try to compare without having done the actions.
 # #############################################################################
@@ -729,8 +727,8 @@ is_deeply(
 );
 
 SKIP: {
-   skip "LOAD DATA LOCAL INFILE is disabled, can't test method => rows", 2
-      if PerconaTest::load_data_is_disabled($dbh1);
+   skip "LOAD DATA LOCAL INFILE is disabled", 2 unless $can_load_data;
+
 $cr = new CompareResults(
    method     => 'rows',
    'base-dir' => $tmpdir,
@@ -781,4 +779,5 @@ diag(`rm -rf /tmp/*outfile.txt`);
 $sb->wipe_clean($dbh1);
 $sb->wipe_clean($dbh2);
 ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+done_testing;
 exit;

t/pt-archiver/bulk_insert.t

@@ -22,12 +22,8 @@ my $dbh = $sb->get_dbh_for('master');
 if ( !$dbh ) {
    plan skip_all => 'Cannot connect to sandbox master';
 }
-elsif ( PerconaTest::load_data_is_disabled($dbh) ) {
-   diag("LOAD DATA LOCAL INFILE is disabled, only going to test the error message");
-   plan tests => 2;
-}
-else {
-   plan tests => 11;
+elsif ( !$can_load_data ) {
+   plan skip_all => 'LOAD DATA LOCAL INFILE is disabled';
 }
 
 my $output;
@@ -35,11 +31,6 @@ my $rows;
 my $cnf = "/tmp/12345/my.sandbox.cnf";
 my $cmd = "$trunk/bin/pt-archiver";
 
-if ( PerconaTest::load_data_is_disabled($dbh) ) {
-   test_disabled_load_data($dbh, $sb, 'master', $cnf);
-}
-else {
-
 $sb->wipe_clean($dbh);
 $sb->create_dbs($dbh, ['test']);
 
@@ -67,7 +58,6 @@ $output = `/tmp/12345/use -N -e "checksum table test.table_5_dest, test.table_5_
 my ( $chks ) = $output =~ m/dest\s+(\d+)/;
 like($output, qr/copy\s+$chks/, 'copy checksum');
 
-
 # ############################################################################
 # Issue 1260: mk-archiver --bulk-insert data loss
 # ############################################################################
@@ -93,48 +83,10 @@ is_deeply(
    "--bulk-insert archived 7 rows (issue 1260)"
 );
 
-# Test that the tool bails out early if LOAD DATA LOCAL INFILE is disabled
-{
-   if ( -d "/tmp/2900" ) {
-      diag(`$trunk/sandbox/stop-sandbox 2900 >/dev/null 2>&1`);
-   }
-
-   local $ENV{LOCAL_INFILE} = 0;
-   diag(`$trunk/sandbox/start-sandbox master 2900 >/dev/null 2>&1`);
-
-   my $master3_dbh = $sb->get_dbh_for('master3');
-
-   test_disabled_load_data($master3_dbh, $sb, 'master3', "/tmp/2900/my.sandbox.cnf");
-
-   diag(`$trunk/sandbox/stop-sandbox 2900 >/dev/null 2>&1`);
-   $master3_dbh->disconnect() if $master3_dbh;
-}
-
-}
-
-sub test_disabled_load_data {
-   my ($dbh, $sb, $master, $cnf) = @_;
-   $sb->wipe_clean($dbh);
-   $sb->create_dbs($dbh, ['test']);
-   $sb->load_file($master, 't/pt-archiver/samples/table5.sql');
-   $dbh->do('INSERT INTO `test`.`table_5_copy` SELECT * FROM `test`.`table_5`');
-
-   my ($output, undef) = full_output(
-      sub { pt_archiver::main(qw(--no-ascend --limit 50 --bulk-insert),
-         qw(--bulk-delete --where 1=1 --statistics),
-         '--source', "D=test,t=table_5,F=$cnf",
-         '--dest',   "t=table_5_dest") },
-   );
-
-   like($output,
-      qr!\Q--bulk-insert cannot work as LOAD DATA LOCAL INFILE is disabled. See http://kb.percona.com/troubleshoot-load-data-infile!,
-      "--bulk-insert throws an error if LOCAL INFILE is disabled"
-   );
-}
-
 # #############################################################################
 # Done.
 # #############################################################################
 $sb->wipe_clean($dbh);
 ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+done_testing;
 exit;

t/pt-archiver/bulk_regular_insert.t

@@ -22,11 +22,8 @@ my $dbh = $sb->get_dbh_for('master');
 if ( !$dbh ) {
    plan skip_all => 'Cannot connect to sandbox master';
 }
-elsif ( PerconaTest::load_data_is_disabled($dbh) ) {
-    plan skip_all => 'Cannot use --bulk-insert with LOAD DATA LOCAL INFILE disabled';
-}
-else {
-   plan tests => 5;
+elsif ( !$can_load_data ) {
+    plan skip_all => 'LOAD DATA LOCAL INFILE is disabled';
 }
 
 my $output;
@@ -98,4 +95,5 @@ is_deeply(
 # #############################################################################
 $sb->wipe_clean($dbh);
 ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+done_testing;
 exit;

t/pt-upgrade/basics.t

@@ -29,11 +29,6 @@ if ( !$dbh1 ) {
 elsif ( !$dbh2 ) {
    plan skip_all => 'Cannot connect to second sandbox master';
 }
-else {
-   plan tests => 13;
-}
-
-my $load_data_is_disabled = PerconaTest::load_data_is_disabled($dbh1);
 
 my @host_args = ('h=127.1,P=12345', 'P=12348');
 my @op_args   = (qw(-u msandbox -p msandbox),
@@ -67,7 +62,7 @@ ok(
 );
 
 SKIP: {
-   skip "LOAD DATA LOCAL INFILE is disabled", 2 if $load_data_is_disabled;
+   skip "LOAD DATA LOCAL INFILE is disabled", 2 unless $can_load_data;
    
    ok(
       no_diff(
@@ -114,7 +109,8 @@ $sb->wipe_clean($dbh2);
 # compare-results-method=rows
 # #############################################################################
 SKIP: {
-   skip "LOAD DATA LOCAL INFILE is disabled", 4 if $load_data_is_disabled;
+   skip "LOAD DATA LOCAL INFILE is disabled", 4 unless $can_load_data;
+
    $sb->load_file('master',  "$sample/002/tables.sql");
    $sb->load_file('master1', "$sample/002/tables.sql");
 
@@ -159,7 +155,6 @@ SKIP: {
    
    $sb->wipe_clean($dbh1);
    $sb->wipe_clean($dbh2);
-
 }
 
 # #############################################################################
@@ -167,7 +162,7 @@ SKIP: {
 # precision (M) and scale (D) 
 # #############################################################################
 SKIP: {
-   skip "LOAD DATA LOCAL INFILE is disabled", 2 if $load_data_is_disabled;
+   skip "LOAD DATA LOCAL INFILE is disabled", 2 unless $can_load_data;
 
    $sb->load_file('master',  "$sample/003/tables.sql");
    $sb->load_file('master1', "$sample/003/tables.sql");
@@ -199,4 +194,5 @@ diag(`rm /tmp/left-outfile.txt /tmp/right-outfile.txt 2>/dev/null`);
 diag(`$trunk/sandbox/stop-sandbox 12348 >/dev/null`);
 $sb->wipe_clean($dbh1);
 ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+done_testing;
 exit;

t/pt-upgrade/warnings.t

@@ -15,6 +15,13 @@ use PerconaTest;
 use Sandbox;
 require "$trunk/bin/pt-upgrade";
 
+# This test calls pt-upgrade with --compare-results-method rows
+# which use LOAD DATA LOCAL INFILE.  If LOAD DATA is disabled,
+# then this this test can't run.
+if ( !$can_load_data ) {
+   plan skip_all => 'LOAD DATA LOCAL INFILE is disabled';
+}
+
 # This runs immediately if the server is already running, else it starts it.
 diag(`$trunk/sandbox/start-sandbox master 12348 >/dev/null`);
 
@@ -31,13 +38,6 @@ elsif ( !$dbh2 ) {
    diag(`$trunk/sandbox/stop-sandbox master 12348 >/dev/null`);
    plan skip_all => 'Cannot connect to second sandbox master';
 }
-elsif ( PerconaTest::load_data_is_disabled($dbh1) ) {
-   diag(`$trunk/sandbox/stop-sandbox master 12348 >/dev/null`);
-   plan skip_all => 'LOAD DATA LOCAL INFILE is disabled';
-}
-else {
-   plan tests => 6;
-}
 
 $sb->load_file('master', 't/pt-upgrade/samples/001/tables.sql');
 $sb->load_file('master1', 't/pt-upgrade/samples/001/tables.sql');
@@ -102,4 +102,5 @@ ok(
 $sb->wipe_clean($dbh1);
 diag(`$trunk/sandbox/stop-sandbox master 12348 >/dev/null`);
 ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+done_testing;
 exit;

util/check-load-data

@@ -0,0 +1,61 @@
+#!/usr/bin/env perl
+
+# This program is copyright 2009-2011 Percona Inc.
+# Feedback and improvements are welcome.
+#
+# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
+# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, version 2; OR the Perl Artistic License.  On UNIX and similar
+# systems, you can issue `man perlgpl' or `man perlartistic' to read these
+# licenses.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+# Place, Suite 330, Boston, MA  02111-1307  USA.
+
+# This program is intended to be run after loading Sakila into our test
+# database, when starting the "sandbox" MySQL instances. It will store the
+# checksums of all of the mysql and sakila tables into a magical
+# percona_test.checksums table on instance 12345. Afterwards, one can verify the
+# integrity of all of these tables by running
+# lib/Sandbox.pm::verify_test_data_integrity() which will checksum the master
+# and all of the slaves, and make sure all are OK.
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use DBI;
+
+my $dbh = DBI->connect(
+   'DBI:mysql:;host=127.0.0.1;port=12345;', 'msandbox', 'msandbox',
+   {
+      AutoCommit         => 1,
+      RaiseError         => 1,
+      PrintError         => 1,
+      ShowErrorStatement => 1,
+   });
+
+$dbh->do("CREATE TABLE IF NOT EXISTS percona_test.load_data (i int)");
+
+`echo 42 > /tmp/load_data_test.$$`;
+
+eval {
+   $dbh->do("LOAD DATA LOCAL INFILE '/tmp/load_data_test.$$' INTO TABLE percona_test.load_data");
+};
+
+unlink "/tmp/load_data_test.$$";
+
+my ($val) = $dbh->selectrow_array("SELECT i FROM percona_test.load_data");
+
+if ( ($val || 0) == 42 ) {
+   print "LOAD DATA LOCAL INFILE is enabled\n";
+}
+else {
+   print "LOAD DATA LOCAL INFILE is disabled\n"
+}
+
+exit;