Merge branch '3.0' into PMM-4278-official-mongo-driver

bin/pt-archiver

@@ -3745,6 +3745,7 @@ sub _find_slaves_by_processlist {
    grep { $_ }
    map  {
       my ( $host ) = $_->{host} =~ m/^([^:]+):/;
+      $host ||= $_->{host};
       if ( $host eq 'localhost' ) {
          $host = '127.0.0.1'; # Replication never uses sockets.
       }

bin/pt-diskstats

@@ -2240,8 +2240,11 @@ sub design_print_formats {
 sub parse_diskstats_line {
    my ( $self, $line, $block_size ) = @_;
 
+   # linux kernel source => Documentation/iostats.txt
+   # 2.6+   => 14 fields
+   # 4.18+  => 18 fields
    my @dev_stats = split ' ', $line;
-   return unless @dev_stats == 14;
+   return unless @dev_stats == 14 or @dev_stats == 18;
 
    my $read_bytes    = $dev_stats[READ_SECTORS]    * $block_size;
    my $written_bytes = $dev_stats[WRITTEN_SECTORS] * $block_size;

bin/pt-heartbeat

@@ -321,6 +321,7 @@ sub _find_slaves_by_processlist {
    grep { $_ }
    map  {
       my ( $host ) = $_->{host} =~ m/^([^:]+):/;
+      $host ||= $_->{host};
       if ( $host eq 'localhost' ) {
          $host = '127.0.0.1'; # Replication never uses sockets.
       }

bin/pt-kill

@@ -4027,6 +4027,7 @@ sub _find_slaves_by_processlist {
    grep { $_ }
    map  {
       my ( $host ) = $_->{host} =~ m/^([^:]+):/;
+      $host ||= $_->{host};
       if ( $host eq 'localhost' ) {
          $host = '127.0.0.1'; # Replication never uses sockets.
       }

bin/pt-online-schema-change

@@ -4281,8 +4281,7 @@ sub recurse_to_slaves {
       PTDEBUG && _d('Connected to', $dp->as_string($slave_dsn));
    };
    if ( $EVAL_ERROR ) {
-      #TODO REMOVE DEBUG
+      print STDERR "Cannot connect to ", $dp->as_string($slave_dsn), "\n"
-      print STDERR "1> Cannot connect to ", $dp->as_string($slave_dsn), "\n"
          or die "Cannot print: $OS_ERROR";
       return;
    }
@@ -4350,6 +4349,7 @@ sub _find_slaves_by_processlist {
    grep { $_ }
    map  {
       my ( $host ) = $_->{host} =~ m/^([^:]+):/;
+      $host ||= $_->{host};
       if ( $host eq 'localhost' ) {
          $host = '127.0.0.1'; # Replication never uses sockets.
       }

bin/pt-pmp

@@ -447,7 +447,6 @@ _parse_command_line() {
          else
             spec=$(grep "^short form:-$opt\$" "$PT_TMPDIR"/po/* | cut -d ':' -f 1)
             if [ -z "$spec"  ]; then
-               option_error "Unknown option: $real_opt"
                continue
             fi
          fi

bin/pt-query-digest

@@ -10675,6 +10675,7 @@ sub _find_slaves_by_processlist {
    grep { $_ }
    map  {
       my ( $host ) = $_->{host} =~ m/^([^:]+):/;
+      $host ||= $_->{host};
       if ( $host eq 'localhost' ) {
          $host = '127.0.0.1'; # Replication never uses sockets.
       }

bin/pt-slave-find

@@ -2435,6 +2435,7 @@ sub _find_slaves_by_processlist {
    grep { $_ }
    map  {
       my ( $host ) = $_->{host} =~ m/^([^:]+):/;
+      $host ||= $_->{host};
       if ( $host eq 'localhost' ) {
          $host = '127.0.0.1'; # Replication never uses sockets.
       }

bin/pt-slave-restart

@@ -2846,6 +2846,7 @@ sub _find_slaves_by_processlist {
    grep { $_ }
    map  {
       my ( $host ) = $_->{host} =~ m/^([^:]+):/;
+      $host ||= $_->{host};
       if ( $host eq 'localhost' ) {
          $host = '127.0.0.1'; # Replication never uses sockets.
       }

bin/pt-stalk

@@ -1352,6 +1352,8 @@ sleep_ok() {
 purge_samples() {
    local dir="$1"
    local retention_time="$2"
+   local retention_count="$3"
+   local retention_size="$4"
 
    # Delete collect files which more than --retention-time days old.
    find "$dir" -type f -mtime +$retention_time -exec rm -f '{}' \;
@@ -1363,6 +1365,30 @@ purge_samples() {
       find "$oprofile_dir" -depth -type d -name 'pt_collect_*' \
          -mtime +$retention_time -exec rm -rf '{}' \;
    fi
+
+   targetCnt=$(($retention_count + 0))
+   if [ $targetCnt -gt 0 ]; then
+      targetCnt=$(($retention_count + 1))
+      files_to_delete=$(find $dir -type f -exec basename {} \; | cut -f1 -d- | sort -r | uniq | tail -n +${targetCnt})
+      for prefix in $files_to_delete; do
+          echo "deleting files ${dir}${prefix}* according to the --retention-count param"
+          rm -f ${dir}${prefix}* 2>/dev/null
+      done
+   fi
+
+   targetSize=$(($retention_size + 0))
+   if [ $targetSize -gt 0 ]; then
+      files_to_delete=$(find $dir -type f -exec basename {} \; | cut -f1 -d- | sort -r | uniq | tail -n +1)
+      for prefix in $files_to_delete; do
+          current_size=$(du -BM $dir | cut -f1 -d"M")
+          if [ $current_size -gt $targetSize ]; then
+             echo "deleting files ${dir}${prefix}* according to the --retention-size param"
+             rm -f ${dir}${prefix}* 2>/dev/null
+          else
+             break
+          fi
+      done
+   fi
 }
 
 sigtrap() {
@@ -1471,7 +1497,7 @@ stalk() {
 
             # Purge old collect files.
             if [ -d "$OPT_DEST" ]; then
-               purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME"
+               purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME" "$OPT_RETENTION_COUNT" "$OPT_RETENTION_SIZE"
             fi
          fi
 
@@ -1496,7 +1522,7 @@ stalk() {
 
    # One final purge of old collect files, but only if in collect mode.
    if [ "$OPT_COLLECT" -a -d "$OPT_DEST" ]; then
-      purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME"
+      purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME" "$OPT_RETENTION_COUNT" "$OPT_RETENTION_SIZE"
    fi
 
    # Before exiting, the last collector may still be running.
@@ -2120,6 +2146,20 @@ The filename prefix for diagnostic samples.  By default, all files created
 by the same L<"--collect"> instance have a timestamp prefix based on the current
 local time, like C<2011_12_06_14_02_02>, which is December 6, 2011 at 14:02:02.
 
+=item --retention-count
+
+type: int; default: 0
+
+Keep the data for the last N runs. If N > 0, the program will keep the data for the last
+N runs and will delete the older data.
+
+=item --retention-size
+
+type: int; default: 0
+
+Keep up to --retention-size MB of data. It will keep at least 1 run even if the size is bigger
+than the specified in this parameter
+
 =item --retention-time
 
 type: int; default: 30

bin/pt-table-checksum

@@ -5299,6 +5299,7 @@ sub _find_slaves_by_processlist {
    grep { $_ }
    map  {
       my ( $host ) = $_->{host} =~ m/^([^:]+):/;
+      $host ||= $_->{host};
       if ( $host eq 'localhost' ) {
          $host = '127.0.0.1'; # Replication never uses sockets.
       }
@@ -6726,8 +6727,6 @@ sub can_nibble {
        if ( !$key || !$key_len || lc($key) ne lc($mysql_index)) {
            $one_nibble = 1;
        }
-   } else {
-       $one_nibble = 1;
    } 
 
    PTDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no');
@@ -9660,9 +9659,7 @@ sub _get_first_values {
       die "I need a $arg argument" unless $args{$arg};
    }
    my ($cxn, $tbl, $index, $n_index_cols) = @args{@required_args};
-
+   my $q  = new Quoter();
-   my $q = $self->{Quoter};
-
    my $index_struct  = $tbl->{tbl_struct}->{keys}->{$index};
    my $index_cols    = $index_struct->{cols};
    my $index_columns = join (', ',
@@ -9690,9 +9687,7 @@ sub _make_range_query {
       die "I need a $arg argument" unless $args{$arg};
    }
    my ($tbl, $index, $n_index_cols, $vals) = @args{@required_args};
-
+   my $q  = new Quoter();
-   my $q = $self->{Quoter};
-
    my $index_struct = $tbl->{tbl_struct}->{keys}->{$index};
    my $index_cols   = $index_struct->{cols};
 
@@ -11595,6 +11590,9 @@ sub nibble_is_safe {
    # See https://bugs.launchpad.net/percona-toolkit/+bug/987393
    my $sth      = $nibble_iter->statements();
    my $boundary = $nibble_iter->boundaries();
+   if (!defined($boundary) || !$boundary || (!$boundary->{lower} || !$boundary->{upper})) {
+        return 0;
+   }
    my $expl     = explain_statement(
       tbl  => $tbl,
       sth  => $sth->{explain_nibble},

bin/pt-table-sync

@@ -6844,6 +6844,7 @@ sub _find_slaves_by_processlist {
    grep { $_ }
    map  {
       my ( $host ) = $_->{host} =~ m/^([^:]+):/;
+      $host ||= $_->{host};
       if ( $host eq 'localhost' ) {
          $host = '127.0.0.1'; # Replication never uses sockets.
       }

lib/MasterSlave.pm

@@ -272,6 +272,7 @@ sub _find_slaves_by_processlist {
    grep { $_ }
    map  {
       my ( $host ) = $_->{host} =~ m/^([^:]+):/;
+      $host ||= $_->{host};
       if ( $host eq 'localhost' ) {
          $host = '127.0.0.1'; # Replication never uses sockets.
       }

lib/bash/parse_options.sh

@@ -468,7 +468,10 @@ _parse_command_line() {
          else
             spec=$(grep "^short form:-$opt\$" "$PT_TMPDIR"/po/* | cut -d ':' -f 1)
             if [ -z "$spec"  ]; then
-               option_error "Unknown option: $real_opt"
+               # Not all programs uses the same options and since these options can be stored
+               # in a common config file, we need to skip general options not used by a particular
+               # program
+               # option_error "Unknown option: $real_opt"
                continue
             fi
          fi

t/lib/TableParser.t

@@ -1267,6 +1267,14 @@ is_deeply(
    'Column having the word "generated" as part of the comment is OK',
 ) or diag Data::Dumper::Dumper($tbl);
 
+$tbl = $tp->parse( load_file('t/lib/samples/generated_cols_comments.sql') );
+warn Data::Dumper::Dumper($tbl);
+
+is_deeply(
+   $tbl,
+   {},
+   'pt-1728',
+);
 # #############################################################################
 # Done.
 # #############################################################################

t/pt-table-checksum/pt-1728.t

@@ -0,0 +1,103 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+
+use PerconaTest;
+use Sandbox;
+use SqlModes;
+use threads;
+use Time::HiRes qw( usleep );
+use constant PTDEBUG => $ENV{PTDEBUG} || 0;
+
+require "$trunk/bin/pt-table-checksum";
+
+my $dp  = new DSNParser(opts=>$dsn_opts);
+my $sb  = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $dbh = $sb->get_dbh_for('master');
+
+if ( !$dbh ) {
+   plan skip_all => 'Cannot connect to sandbox master';
+}
+else {
+   plan tests => 2;
+}
+
+my $num_rows = 1_000_000;
+my $table = 't1';
+
+$dbh->do("DROP DATABASE IF EXISTS test");
+$dbh->do("CREATE DATABASE IF NOT EXISTS test");
+$dbh->do("CREATE TABLE `test`.`$table` (id INT PRIMARY KEY AUTO_INCREMENT, name VARCHAR(5)) Engine=InnoDB");
+
+diag(`util/mysql_random_data_load --host=127.0.0.1 --port=12345 --user=msandbox --password=msandbox test $table $num_rows`);
+
+sub start_thread {
+   my ($dsn_opts, $initial_sleep_time) = @_;
+   my $dp = new DSNParser(opts=>$dsn_opts);
+   my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+   my $master_dbh = $sb->get_dbh_for('master');
+   my $slave_dbh = $sb->get_dbh_for('slave1');
+   diag("Sleeping");
+   sleep($initial_sleep_time);
+   diag("Woke up");
+   $slave_dbh->do("STOP SLAVE IO_THREAD FOR CHANNEL ''");
+   $slave_dbh->do("STOP SLAVE");
+   $master_dbh->do("TRUNCATE TABLE test.$table");
+   # PTDEBUG && diag("Exit thread")
+   sleep(2);
+   $slave_dbh->do("START SLAVE");
+   diag("Exit thread")
+}
+# This is not a realiable sleep value. It works for a i7, hybrid HDD
+my $initial_sleep_time = 17;
+my $thr = threads->create('start_thread', $dsn_opts, $initial_sleep_time);
+threads->yield();
+
+diag("Starting checksum");
+# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
+# so we need to specify --set-vars innodb_lock_wait_timeout=3 else the tool will die.
+# And --max-load "" prevents waiting for status variables.
+my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox';
+my @args       = ($master_dsn, qw(--no-check-binlog-format --chunk-size 10)); 
+my $output;
+
+$output = output(
+   sub { pt_table_checksum::main(@args) },
+   stderr => 1,
+);
+
+diag($output);
+unlike(
+   $output,
+   qr/Can't use an undefined value as an ARRAY/,
+   "Truncating tables while checksum is running"
+);
+
+$sb->load_file('master', 't/pt-table-checksum/samples/pt-1728.sql');
+@args = ($master_dsn, qw(--no-check-binlog-format)); 
+my $new_rows_count = $num_rows * 5;
+diag(`util/mysql_random_data_load --host=127.0.0.1 --port=12345 --user=msandbox --password=msandbox test $table $new_rows_count`);
+$output = output(
+   sub { pt_table_checksum::main(@args) },
+   stderr => 1,
+);
+
+diag($output);
+
+$thr->join();
+
+# #############################################################################
+# Done.
+# #############################################################################
+$sb->wipe_clean($dbh);
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+exit;