Test and fix very small --chunk-time. Report immediately when a slave is stopped. Add short form -q for --quiet. Report very slow checksums once for each table. Use Cxn::name() instead of Cxn::dsn()->{n}; remove n from DSNParser; make cxn's name @@hostname by default, else stringified DSN parts.

lib/Cxn.pm

@@ -37,6 +37,12 @@ use warnings FATAL => 'all';
 use English qw(-no_match_vars);
 use constant MKDEBUG => $ENV{MKDEBUG} || 0;
 
+# Hostnames make testing less accurate.  Tests need to see
+# that such-and-such happened on specific slave hosts, but
+# the sandbox servers are all on one host so all slaves have
+# the same hostname.
+use constant PERCONA_TOOLKIT_TEST_USE_DSN_NAMES => $ENV{PERCONA_TOOLKIT_TEST_USE_DSN_NAMES} || 0;
+
 # Sub: new
 #
 # Required Arguments:
@@ -87,18 +93,19 @@ sub new {
       # copy values from it into this new DSN, resulting in a new DSN
       # with values from both sources.
       $dsn = $dp->copy($prev_dsn, $dsn);
-      $dsn->{n} = $dp->as_string($dsn, [qw(h P S F)]);
    }
 
    my $self = {
       dsn          => $dsn,
       dbh          => $args{dbh},
+      dsn_name     => $dp->as_string($dsn, [qw(h P S)]),
+      hostname     => '',
       set          => $args{set},
+      dbh_set      => 0,
       OptionParser => $o,
       DSNParser    => $dp,
    };
 
-   MKDEBUG && _d('New connection to', $dsn->{n});
    return bless $self, $class;
 }
 
@@ -115,10 +122,9 @@ sub connect {
          $dsn->{p} = OptionParser::prompt_noecho("Enter MySQL password: ");
          $self->{asked_for_pass} = 1;
       }
-
       $dbh = $dp->get_dbh($dp->get_cxn_params($dsn),  { AutoCommit => 1 });
-      MKDEBUG && _d('Connected dbh', $dbh, $dsn->{n});
    }
+   MKDEBUG && _d($dbh, 'Connected dbh to', $self->{name});
 
    return $self->set_dbh($dbh);
 }
@@ -126,34 +132,69 @@ sub connect {
 sub set_dbh {
    my ($self, $dbh) = @_;
 
-   # Don't set stuff twice on the same dbh.
-   return $dbh if $self->{dbh} && $self->{dbh} == $dbh;
+   # If we already have a dbh, and that dbh is the same as this dbh,
+   # and the dbh has already been set, then do not re-set the same
+   # dbh.  dbh_set is required so that if this obj was created with
+   # a dbh, we set that dbh when connect() is called because whoever
+   # created the dbh probably didn't set what we set here.  For example,
+   # MasterSlave makes dbhs when finding slaves, but it doesn't set
+   # anything.
+   if ( $self->{dbh} && $self->{dbh} == $dbh && $self->{dbh_set} ) {
+      MKDEBUG && _d($dbh, 'Already set dbh');
+      return $dbh;
+   }
+
+   MKDEBUG && _d($dbh, 'Setting dbh');
 
    # Set stuff for this dbh (i.e. initialize it).
    $dbh->{FetchHashKeyName} = 'NAME_lc';
+   
+   # Update the cxn's name.  Until we connect, the DSN parts
+   # h and P are used.  Once connected, use @@hostname.
+   my $sql = 'SELECT @@hostname, @@server_id';
+   MKDEBUG && _d($dbh, $sql);
+   my ($hostname, $server_id) = $dbh->selectrow_array($sql);
+   MKDEBUG && _d($dbh, 'hostname:', $hostname, $server_id);
+   if ( $hostname ) {
+      $self->{hostname} = $hostname;
+   }
 
+   # Call the set callback to let the caller SET any MySQL variables.
    if ( my $set = $self->{set}) {
       $set->($dbh);
    }
 
-   $self->{dbh} = $dbh;
+   $self->{dbh}     = $dbh;
+   $self->{dbh_set} = 1;
    return $dbh;
 }
 
+# Sub: dbh
+#   Return the cxn's dbh.
 sub dbh {
    my ($self) = @_;
    return $self->{dbh};
 }
 
+# Sub: dsn
+#   Return the cxn's dsn.
 sub dsn {
    my ($self) = @_;
    return $self->{dsn};
 }
 
+# Sub: name
+#   Return the cxn's name.
+sub name {
+   my ($self) = @_;
+   return $self->{dsn_name} if PERCONA_TOOLKIT_TEST_USE_DSN_NAMES;
+   return $self->{hostname} || $self->{dsn_name} || 'unknown host';
+}
+
 sub DESTROY {
    my ($self) = @_;
    if ( $self->{dbh} ) {
-      MKDEBUG && _d('Disconnecting dbh', $self->{dbh}, $self->{dsn}->{n});
+      MKDEBUG && _d('Disconnecting dbh', $self->{dbh}, $self->{name});
       $self->{dbh}->disconnect();
    }
    return;

lib/DSNParser.pm

@@ -170,10 +170,6 @@ sub parse {
       }
    }
 
-   if ( !$final_props{n} ) {  # name
-      $final_props{n} = $self->as_string(\%final_props, [qw(h P S F)]);
-   }
-
    return \%final_props;
 }
 

lib/PerconaTest.pm

@@ -467,12 +467,17 @@ sub no_diff {
    die "I need a cmd argument" unless $cmd;
    die "I need an expected_output argument" unless $expected_output;
 
+   die "$expected_output does not exist" unless -f "$trunk/$expected_output";
    $expected_output = "$trunk/$expected_output";
-   die "$expected_output does not exist" unless -f $expected_output;
 
    my $tmp_file      = '/tmp/percona-toolkit-test-output.txt';
    my $tmp_file_orig = '/tmp/percona-toolkit-test-output-original.txt';
 
+   if ( my $sed_args = $args{sed_out} ) {
+      `cat $expected_output | sed $sed_args > /tmp/pt-test-outfile-trf`;
+      $expected_output = "/tmp/pt-test-outfile-trf";
+   }
+
    # Determine cmd type and run it.
    if ( ref $cmd eq 'CODE' ) {
       output($cmd, file => $tmp_file);
@@ -522,7 +527,7 @@ sub no_diff {
    }
 
    # Remove our tmp files.
-   `rm -f $tmp_file $tmp_file_orig`
+   `rm -f $tmp_file $tmp_file_orig /tmp/pt-test-outfile-trf >/dev/null 2>&1`
       unless $ENV{KEEP_OUTPUT} || $args{keep_output};
 
    return !$retval;

lib/Progress.pm

@@ -125,11 +125,21 @@ sub start {
 # many lines we're done processing -- a number between 0 and 800.  You can also
 # optionally pass in the current time, but this is only for testing.
 sub update {
-   my ( $self, $callback, $now ) = @_;
+   my ( $self, $callback, %args ) = @_;
    my $jobsize   = $self->{jobsize};
-   $now        ||= time();
+   my $now    ||= $args{now} || time;
+
    $self->{iterations}++; # How many updates have happened;
 
+   # The caller may want to report something special before the actual
+   # first report ($callback) if, for example, they know that the wait
+   # could be long.  This is called only once; subsequent reports will
+   # come from $callback after 30s, or whatever the interval is.
+   if ( !$self->{first_report} && $args{first_report} ) {
+      $args{first_report}->();
+      $self->{first_report} = 1;
+   }
+
    # Determine whether to just quit and return...
    if ( $self->{report} eq 'time'
          && $self->{interval} > $now - $self->{last_reported}

lib/ReplicaLagWaiter.pm

@@ -79,12 +79,13 @@ sub wait {
 
    my $worst;  # most lagging slave
    my $pr_callback;
+   my $pr_first_report;
    if ( $pr ) {
       # If you use the default Progress report callback, you'll need to
       # to add Transformers.pm to this tool.
       $pr_callback = sub {
          my ($fraction, $elapsed, $remaining, $eta, $completed) = @_;
-         my $dsn_name = $worst->{cxn}->dsn()->{n} || '?';
+         my $dsn_name = $worst->{cxn}->name();
          if ( defined $worst->{lag} ) {
             print STDERR "Replica lag is " . ($worst->{lag} || '?')
                . " seconds on $dsn_name.  Waiting.\n";
@@ -95,6 +96,17 @@ sub wait {
          return;
       };
       $pr->set_callback($pr_callback);
+
+      # If a replic is stopped, don't wait 30s (or whatever interval)
+      # to report this.  Instead, report it once, immediately, then
+      # keep reporting it every interval.
+      $pr_first_report = sub {
+         my $dsn_name = $worst->{cxn}->name();
+         if ( !defined $worst->{lag} ) {
+            print STDERR "Replica $dsn_name is stopped.  Waiting.\n";
+         }
+         return;
+      };
    }
 
    # First check all slaves.
@@ -103,7 +115,7 @@ sub wait {
       MKDEBUG && _d('Checking slave lag');
       for my $i ( 0..$#lagged_slaves ) {
          my $lag = $get_lag->($lagged_slaves[$i]->{cxn});
-         MKDEBUG && _d($lagged_slaves[$i]->{cxn}->dsn()->{n},
+         MKDEBUG && _d($lagged_slaves[$i]->{cxn}->name(),
             'slave lag:', $lag);
          if ( !defined $lag || $lag > $max_lag ) {
             $lagged_slaves[$i]->{lag} = $lag;
@@ -131,7 +143,10 @@ sub wait {
             # it will take all slaves to catch up.  The progress reports
             # are just to inform the user every 30s which slave is still
             # lagging this most.
-            $pr->update(sub { return 0; });
+            $pr->update(
+               sub { return 0; },
+               first_report => $pr_first_report,
+            );
          }
 
          MKDEBUG && _d('Calling sleep callback');