Fix for 1047335: SchemaIterator fails when it encounters a crashed table

bin/pt-archiver

@@ -1584,13 +1584,11 @@ sub get_create_table {
    PTDEBUG && _d($show_sql);
    my $href;
    eval { $href = $dbh->selectrow_hashref($show_sql); };
-   if ( $EVAL_ERROR ) {
-      PTDEBUG && _d($EVAL_ERROR);
-
+   if ( my $e = $EVAL_ERROR ) {
       PTDEBUG && _d($old_sql_mode);
       $dbh->do($old_sql_mode);
 
-      return;
+      die $e;
    }
 
    PTDEBUG && _d($old_sql_mode);

bin/pt-config-diff

@@ -1576,7 +1576,9 @@ sub is_cluster_node {
    PTDEBUG && _d($sql);
    my $row = $self->{dbh}->selectrow_arrayref($sql);
    PTDEBUG && _d(defined $row ? @$row : 'undef');
-   $self->{is_cluster_node} = $row && $row->[0] ? 1 : 0;
+   $self->{is_cluster_node} = $row && $row->[1]
+                            ? ($row->[1] eq 'ON' || $row->[1] eq '1')
+                            : 0;
   
    return $self->{is_cluster_node};
 }

bin/pt-duplicate-key-checker

@@ -226,13 +226,11 @@ sub get_create_table {
    PTDEBUG && _d($show_sql);
    my $href;
    eval { $href = $dbh->selectrow_hashref($show_sql); };
-   if ( $EVAL_ERROR ) {
-      PTDEBUG && _d($EVAL_ERROR);
-
+   if ( my $e = $EVAL_ERROR ) {
       PTDEBUG && _d($old_sql_mode);
       $dbh->do($old_sql_mode);
 
-      return;
+      die $e;
    }
 
    PTDEBUG && _d($old_sql_mode);
@@ -3138,7 +3136,18 @@ sub _iterate_dbh {
    }
 
    while ( my $tbl = shift @{$self->{tbls}} ) {
-      my $ddl        = $tp->get_create_table($dbh, $self->{db}, $tbl);
+      local $EVAL_ERROR;
+      my $ddl        = eval { $tp->get_create_table($dbh, $self->{db}, $tbl) };
+      if ( my $e = $EVAL_ERROR ) {
+         my $table_name = "$self->{db}.$tbl";
+         if ( $e =~ /\QTable '$table_name' doesn't exist/ ) {
+            PTDEBUG && _d("Skipping $table_name because it no longer exists");
+         }
+         else {
+            warn "Skipping $table_name because SHOW CREATE TABLE failed: $e";
+         }
+         next;
+      }
       my $tbl_struct = $tp->parse($ddl);
       if ( $self->engine_is_allowed($tbl_struct->{engine}) ) {
          return {

bin/pt-find

@@ -1623,13 +1623,11 @@ sub get_create_table {
    PTDEBUG && _d($show_sql);
    my $href;
    eval { $href = $dbh->selectrow_hashref($show_sql); };
-   if ( $EVAL_ERROR ) {
-      PTDEBUG && _d($EVAL_ERROR);
-
+   if ( my $e = $EVAL_ERROR ) {
       PTDEBUG && _d($old_sql_mode);
       $dbh->do($old_sql_mode);
 
-      return;
+      die $e;
    }
 
    PTDEBUG && _d($old_sql_mode);

bin/pt-heartbeat

@@ -2549,13 +2549,11 @@ sub get_create_table {
    PTDEBUG && _d($show_sql);
    my $href;
    eval { $href = $dbh->selectrow_hashref($show_sql); };
-   if ( $EVAL_ERROR ) {
-      PTDEBUG && _d($EVAL_ERROR);
-
+   if ( my $e = $EVAL_ERROR ) {
       PTDEBUG && _d($old_sql_mode);
       $dbh->do($old_sql_mode);
 
-      return;
+      die $e;
    }
 
    PTDEBUG && _d($old_sql_mode);

bin/pt-index-usage

@@ -2821,13 +2821,11 @@ sub get_create_table {
    PTDEBUG && _d($show_sql);
    my $href;
    eval { $href = $dbh->selectrow_hashref($show_sql); };
-   if ( $EVAL_ERROR ) {
-      PTDEBUG && _d($EVAL_ERROR);
-
+   if ( my $e = $EVAL_ERROR ) {
       PTDEBUG && _d($old_sql_mode);
       $dbh->do($old_sql_mode);
 
-      return;
+      die $e;
    }
 
    PTDEBUG && _d($old_sql_mode);
@@ -3968,7 +3966,18 @@ sub _iterate_dbh {
    }
 
    while ( my $tbl = shift @{$self->{tbls}} ) {
-      my $ddl        = $tp->get_create_table($dbh, $self->{db}, $tbl);
+      local $EVAL_ERROR;
+      my $ddl        = eval { $tp->get_create_table($dbh, $self->{db}, $tbl) };
+      if ( my $e = $EVAL_ERROR ) {
+         my $table_name = "$self->{db}.$tbl";
+         if ( $e =~ /\QTable '$table_name' doesn't exist/ ) {
+            PTDEBUG && _d("Skipping $table_name because it no longer exists");
+         }
+         else {
+            warn "Skipping $table_name because SHOW CREATE TABLE failed: $e";
+         }
+         next;
+      }
       my $tbl_struct = $tp->parse($ddl);
       if ( $self->engine_is_allowed($tbl_struct->{engine}) ) {
          return {

bin/pt-kill

@@ -2499,13 +2499,11 @@ sub get_create_table {
    PTDEBUG && _d($show_sql);
    my $href;
    eval { $href = $dbh->selectrow_hashref($show_sql); };
-   if ( $EVAL_ERROR ) {
-      PTDEBUG && _d($EVAL_ERROR);
-
+   if ( my $e = $EVAL_ERROR ) {
       PTDEBUG && _d($old_sql_mode);
       $dbh->do($old_sql_mode);
 
-      return;
+      die $e;
    }
 
    PTDEBUG && _d($old_sql_mode);
@@ -4807,7 +4805,9 @@ sub is_cluster_node {
    PTDEBUG && _d($sql);
    my $row = $self->{dbh}->selectrow_arrayref($sql);
    PTDEBUG && _d(defined $row ? @$row : 'undef');
-   $self->{is_cluster_node} = $row && $row->[0] ? 1 : 0;
+   $self->{is_cluster_node} = $row && $row->[1]
+                            ? ($row->[1] eq 'ON' || $row->[1] eq '1')
+                            : 0;
   
    return $self->{is_cluster_node};
 }

bin/pt-online-schema-change

@@ -2733,13 +2733,11 @@ sub get_create_table {
    PTDEBUG && _d($show_sql);
    my $href;
    eval { $href = $dbh->selectrow_hashref($show_sql); };
-   if ( $EVAL_ERROR ) {
-      PTDEBUG && _d($EVAL_ERROR);
-
+   if ( my $e = $EVAL_ERROR ) {
       PTDEBUG && _d($old_sql_mode);
       $dbh->do($old_sql_mode);
 
-      return;
+      die $e;
    }
 
    PTDEBUG && _d($old_sql_mode);
@@ -3437,7 +3435,9 @@ sub is_cluster_node {
    PTDEBUG && _d($sql);
    my $row = $self->{dbh}->selectrow_arrayref($sql);
    PTDEBUG && _d(defined $row ? @$row : 'undef');
-   $self->{is_cluster_node} = $row && $row->[0] ? 1 : 0;
+   $self->{is_cluster_node} = $row && $row->[1]
+                            ? ($row->[1] eq 'ON' || $row->[1] eq '1')
+                            : 0;
   
    return $self->{is_cluster_node};
 }

bin/pt-query-advisor

@@ -5311,13 +5311,11 @@ sub get_create_table {
    PTDEBUG && _d($show_sql);
    my $href;
    eval { $href = $dbh->selectrow_hashref($show_sql); };
-   if ( $EVAL_ERROR ) {
-      PTDEBUG && _d($EVAL_ERROR);
-
+   if ( my $e = $EVAL_ERROR ) {
       PTDEBUG && _d($old_sql_mode);
       $dbh->do($old_sql_mode);
 
-      return;
+      die $e;
    }
 
    PTDEBUG && _d($old_sql_mode);

bin/pt-query-digest

@@ -8140,13 +8140,11 @@ sub get_create_table {
    PTDEBUG && _d($show_sql);
    my $href;
    eval { $href = $dbh->selectrow_hashref($show_sql); };
-   if ( $EVAL_ERROR ) {
-      PTDEBUG && _d($EVAL_ERROR);
-
+   if ( my $e = $EVAL_ERROR ) {
       PTDEBUG && _d($old_sql_mode);
       $dbh->do($old_sql_mode);
 
-      return;
+      die $e;
    }
 
    PTDEBUG && _d($old_sql_mode);

bin/pt-table-checksum

@@ -3725,13 +3725,11 @@ sub get_create_table {
    PTDEBUG && _d($show_sql);
    my $href;
    eval { $href = $dbh->selectrow_hashref($show_sql); };
-   if ( $EVAL_ERROR ) {
-      PTDEBUG && _d($EVAL_ERROR);
-
+   if ( my $e = $EVAL_ERROR ) {
       PTDEBUG && _d($old_sql_mode);
       $dbh->do($old_sql_mode);
 
-      return;
+      die $e;
    }
 
    PTDEBUG && _d($old_sql_mode);
@@ -6784,7 +6782,18 @@ sub _iterate_dbh {
    }
 
    while ( my $tbl = shift @{$self->{tbls}} ) {
-      my $ddl        = $tp->get_create_table($dbh, $self->{db}, $tbl);
+      local $EVAL_ERROR;
+      my $ddl        = eval { $tp->get_create_table($dbh, $self->{db}, $tbl) };
+      if ( my $e = $EVAL_ERROR ) {
+         my $table_name = "$self->{db}.$tbl";
+         if ( $e =~ /\QTable '$table_name' doesn't exist/ ) {
+            PTDEBUG && _d("Skipping $table_name because it no longer exists");
+         }
+         else {
+            warn "Skipping $table_name because SHOW CREATE TABLE failed: $e";
+         }
+         next;
+      }
       my $tbl_struct = $tp->parse($ddl);
       if ( $self->engine_is_allowed($tbl_struct->{engine}) ) {
          return {

bin/pt-table-sync

@@ -2990,7 +2990,7 @@ sub dump {
    my ( $self, $dbh, $quoter, $db, $tbl, $what ) = @_;
 
    if ( $what eq 'table' ) {
-      my $ddl = $self->get_create_table($dbh, $quoter, $db, $tbl);
+      my $ddl = $self->get_create_table($dbh, $dbh, $db, $tbl);
       return unless $ddl;
       if ( $ddl->[0] eq 'table' ) {
          return $before
@@ -3033,7 +3033,7 @@ sub dump {
       }
    }
    elsif ( $what eq 'view' ) {
-      my $ddl = $self->get_create_table($dbh, $quoter, $db, $tbl);
+      my $ddl = $self->get_create_table($dbh, $dbh, $db, $tbl);
       return '/*!50001 DROP TABLE IF EXISTS ' . $quoter->quote($tbl) . "*/;\n"
          . '/*!50001 DROP VIEW IF EXISTS ' . $quoter->quote($tbl) . "*/;\n"
          . '/*!50001 ' . $ddl->[1] . "*/;\n";

bin/pt-upgrade

@@ -498,13 +498,11 @@ sub get_create_table {
    PTDEBUG && _d($show_sql);
    my $href;
    eval { $href = $dbh->selectrow_hashref($show_sql); };
-   if ( $EVAL_ERROR ) {
-      PTDEBUG && _d($EVAL_ERROR);
-
+   if ( my $e = $EVAL_ERROR ) {
       PTDEBUG && _d($old_sql_mode);
       $dbh->do($old_sql_mode);
 
-      return;
+      die $e;
    }
 
    PTDEBUG && _d($old_sql_mode);

lib/SchemaIterator.pm

@@ -352,7 +352,22 @@ sub _iterate_dbh {
    }
 
    while ( my $tbl = shift @{$self->{tbls}} ) {
-      my $ddl        = $tp->get_create_table($dbh, $self->{db}, $tbl);
+      local $EVAL_ERROR;
+      my $ddl        = eval { $tp->get_create_table($dbh, $self->{db}, $tbl) };
+      if ( my $e = $EVAL_ERROR ) {
+         my $table_name = "$self->{db}.$tbl";
+         # SHOW CREATE TABLE failed. This is a bit puzzling;
+         # maybe the table got dropped, or crashed. Not much we can
+         # do about it; If the table is missing, just PTDEBUG it, but
+         # otherwise, warn with the error.
+         if ( $e =~ /\QTable '$table_name' doesn't exist/ ) {
+            PTDEBUG && _d("Skipping $table_name because it no longer exists");
+         }
+         else {
+            warn "Skipping $table_name because SHOW CREATE TABLE failed: $e";
+         }
+         next;
+      }
       my $tbl_struct = $tp->parse($ddl);
       if ( $self->engine_is_allowed($tbl_struct->{engine}) ) {
          return {

lib/TableParser.pm

@@ -82,18 +82,12 @@ sub get_create_table {
    PTDEBUG && _d($show_sql);
    my $href;
    eval { $href = $dbh->selectrow_hashref($show_sql); };
-   if ( $EVAL_ERROR ) {
-      # TODO: I think we fail silently for tools which may try to call
-      # this on temp tables, or don't care if the table goes away.  We
-      # should warn $EVAL_ERROR and require callers to eval us and do
-      # what they want with the warning.
-      PTDEBUG && _d($EVAL_ERROR);
-
+   if ( my $e = $EVAL_ERROR ) {
       # Restore old SQL mode.
       PTDEBUG && _d($old_sql_mode);
       $dbh->do($old_sql_mode);
 
-      return;
+      die $e;
    }
 
    # Restore old SQL mode.

t/lib/SchemaIterator.t

@@ -9,7 +9,7 @@ BEGIN {
 use strict;
 use warnings FATAL => 'all';
 use English qw(-no_match_vars);
-use Test::More tests => 30;
+use Test::More;
 
 use SchemaIterator;
 use FileIterator;
@@ -406,8 +406,76 @@ test_so(
    test_name => "Resume from ignored table"
 );
 
+# #############################################################################
+# Bug 1047335: pt-duplicate-key-checker fails when it encounters a crashed table
+# https://bugs.launchpad.net/percona-toolkit/+bug/1047335
+# #############################################################################
+
+use File::Spec;
+
+my $master3_port = 2900;
+diag(`$trunk/sandbox/stop-sandbox $master3_port >/dev/null`);
+diag(`$trunk/sandbox/start-sandbox master $master3_port >/dev/null`);
+my $dbh3 = $sb->get_dbh_for("master3");
+
+$sb->load_file('master3', File::Spec->catfile(qw(t lib samples bug_1047335_crashed_table.sql)));
+my $sth = $dbh3->prepare("INSERT INTO bug_1047335.crashed_table (trx_id, etc) VALUES(?, ?)");
+$sth->execute($_, $_ x 100) for 1..1000;
+$sth->finish();
+
+# Create the SI object before crashing the table
+my $tmp_si = new SchemaIterator(
+         dbh          => $dbh3,
+         OptionParser => $o,
+         Quoter       => $q,
+         TableParser  => $tp,
+         # This is needed because the way we corrupt tables
+         # accidentally removes the database from SHOW DATABASES
+         db           => 'bug_1047335',
+      );
+
+my $master_basedir = File::Spec->catdir(File::Spec->tmpdir(), $master3_port);
+my $db_dir         = File::Spec->catdir($master_basedir, "data", "bug_1047335");
+my $myi            = glob(File::Spec->catfile($db_dir, "crashed_table.[Mm][Yy][Iy]"));
+my $frm            = glob(File::Spec->catfile($db_dir, "crashed_table.[Ff][Rr][Mm]"));
+
+die "Cannot find .myi file for crashed_table" unless $myi && -f $myi;
+
+# Truncate the .myi file to corrupt it
+truncate($myi, 4096);
+
+use File::Slurp qw( prepend_file append_file write_file );
+
+# Corrupt the .frm file
+open my $urand_fh, q{<}, "/dev/urandom"
+   or die "Cannot open /dev/urandom";
+prepend_file($frm, scalar(<$urand_fh>));
+append_file($frm, scalar(<$urand_fh>));
+close $urand_fh;
+
+$dbh3->do("FLUSH TABLES");
+eval { $dbh3->do("SELECT etc FROM bug_1047335.crashed_table WHERE etc LIKE '10001' ORDER BY id ASC LIMIT 1") };
+
+my $w = '';
+{
+   local $SIG{__WARN__} = sub { $w .= shift };
+   1 while $tmp_si->next();
+}
+
+like(
+   $w,
+   qr/because SHOW CREATE TABLE failed:/,
+   "->next() gives a warning if ->get_create_table dies from a strange error",
+);
+
+# This might fail. Doesn't matter -- stop_sandbox will just rm -rf the folder
+eval { $dbh3->do("DROP DATABASE IF EXISTS bug_1047335") };
+
+diag(`$trunk/sandbox/stop-sandbox $master3_port >/dev/null`);
+
 # #############################################################################
 # Done.
 # #############################################################################
 ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
-exit;
+
+done_testing;

t/lib/TableParser.t

@@ -9,7 +9,7 @@ BEGIN {
 use strict;
 use warnings FATAL => 'all';
 use English qw(-no_match_vars);
-use Test::More tests => 40;
+use Test::More;
 
 use TableParser;
 use Quoter;
@@ -32,10 +32,10 @@ SKIP: {
    skip 'Sandbox master does not have the sakila database', 2
       unless @{$dbh->selectcol_arrayref("SHOW DATABASES LIKE 'sakila'")};
 
-   is(
-      $tp->get_create_table($dbh, 'sakila', 'FOO'),
-      undef,
-      "get_create_table(nonexistent table)"
+   eval { $tp->get_create_table($dbh, 'sakila', 'FOO') };
+   ok(
+      $EVAL_ERROR,
+      "get_create_table(nonexistent table) dies"
    );
 
    my $ddl = $tp->get_create_table($dbh, 'sakila', 'actor');
@@ -892,9 +892,68 @@ is_deeply(
    'Index with comma in its name (issue 388)'
 );
 
+# #############################################################################
+# Bug 1047335: pt-duplicate-key-checker fails when it encounters a crashed table
+# https://bugs.launchpad.net/percona-toolkit/+bug/1047335
+# #############################################################################
+
+# We need to create a new server here, otherwise the whole test suite might die
+# if the crashed table can't be dropped.
+
+use File::Spec;
+
+my $master3_port = 2900;
+diag(`$trunk/sandbox/stop-sandbox $master3_port >/dev/null`);
+diag(`$trunk/sandbox/start-sandbox master $master3_port >/dev/null`);
+my $dbh3 = $sb->get_dbh_for("master3");
+
+$sb->load_file('master3', File::Spec->catfile(qw(t lib samples bug_1047335_crashed_table.sql)));
+my $sth = $dbh3->prepare("INSERT INTO bug_1047335.crashed_table (trx_id, etc) VALUES(?, ?)");
+$sth->execute($_, $_ x 100) for 1..1000;
+$sth->finish();
+
+
+my $master_basedir = File::Spec->catdir(File::Spec->tmpdir(), $master3_port);
+my $db_dir         = File::Spec->catdir($master_basedir, "data", "bug_1047335");
+my $myi            = glob(File::Spec->catfile($db_dir, "crashed_table.[Mm][Yy][Iy]"));
+my $frm            = glob(File::Spec->catfile($db_dir, "crashed_table.[Ff][Rr][Mm]"));
+
+die "Cannot find .myi file for crashed_table" unless $myi && -f $myi;
+
+# Truncate the .myi file to corrupt it
+truncate($myi, 4096);
+
+# Corrupt the .frm file
+open my $urand_fh, q{<}, "/dev/urandom"
+   or die "Cannot open /dev/urandom: $OS_ERROR";
+
+open my $tmp_fh, q{>}, $frm
+   or die "Cannot open $frm: $OS_ERROR";
+print { $tmp_fh } scalar(<$urand_fh>), slurp_file($frm), scalar(<$urand_fh>);
+close $tmp_fh;
+   
+close $urand_fh;
+
+$dbh3->do("FLUSH TABLES");
+eval { $dbh3->do("SELECT etc FROM bug_1047335.crashed_table WHERE etc LIKE '10001' ORDER BY id ASC LIMIT 1") };
+
+eval { $tp->get_create_table($dbh3, 'bug_1047335', 'crashed_table') };
+ok(
+   $EVAL_ERROR,
+   "get_create_table dies if SHOW CREATE TABLE failed",
+);
+
+# This might fail. Doesn't matter -- stop_sandbox will just rm -rf the folder
+eval { $dbh3->do("DROP DATABASE IF EXISTS bug_1047335") };
+
+diag(`$trunk/sandbox/stop-sandbox $master3_port >/dev/null`);
+
 # #############################################################################
 # Done.
 # #############################################################################
 $sb->wipe_clean($dbh) if $dbh;
 ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+
+done_testing;
+
 exit;

t/lib/samples/bug_1047335_crashed_table.sql

@@ -0,0 +1,11 @@
+DROP DATABASE IF EXISTS bug_1047335;
+CREATE DATABASE bug_1047335;
+USE bug_1047335;
+CREATE TABLE bug_1047335.crashed_table (
+   `id` int(10) unsigned NOT NULL auto_increment,
+   `trx_id` int(10) unsigned default NULL,
+   `etc` text,
+   PRIMARY KEY (`id`),
+   KEY `idx1` (`trx_id`),
+   KEY `idx2` (`trx_id`, `etc`(128))
+) ENGINE=MyISAM;