PT-1585 Added --force-nibbling in pt-tc

Change the implementation of release notes

.travis.yml

@@ -39,6 +39,9 @@ before_script:
 script:
   - go test -timeout 1m ./src/...
 
+allow_failures:
+    - tip
+
 notifications:
   email: false
   slack:

Changelog

@@ -1,11 +1,28 @@
 Changelog for Percona Toolkit
 
+v3.0.11 released 2018-07-06
+
+ * Improvement PT-1571 : Improved hostname recognition in pt-secure-collect
+ * Fixed bug   PT-1570 : pt-archiver fails to detect columns with the word GENERATED as part of the comment
+ * Improvement PT-1569 : Disabled --alter-foreign-keys-method=drop_swap in pt-osc
+ * Fixed bug   PT-1563 : Fixed pt-show-grants for MySQL 5.6
+ * Improvement PT-1562 : pt-mysql-summary: Fix mysqld command for Travis
+ * Fixed bug   PT-1551 : pt-table-checksum fails on MySQL 8.0.11
+ * Improvement  PT-242 : (pt-stalk) Include SHOW SLAVE STATUS on 5.7 (Thanks Marcelo Altmann)
+ * Fixed bug    PT-241 : (pt-stalk) Slave queries doesn't run on 5.7 (Thanks Marcelo Altmann)
+
+v3.0.10 released 2018-05-21
+
+ * Fixed bug   PT-1556 : pt-table-checksum 3.0.9 doesn't change binlog_format to statement anymore
+ * Improvement PT-1546 : Improved support of MySQL 8 roles
+ * Improvement PT-1543 : Encrypted table status query causes high load over multiple minutes
+ * Improvement PT-1536 : Add info about encrypted tablespaces in pt-mysql-summary
+ * Feature      PT-131 : Make pt-table checksum to disable QRT plugin
+ * Feature      PT-118 : pt-table-checksum report the number of rows of difference between master and slave
+
 v3.0.9 released 2018-04-17
 
-v3.0.9 
-
  * Feature     PT-1530 : Add support for encryption status to mysql-summary
- * Fixed bug   PT-1527 : pt-table-checksum ignores --nocheck-binlog-format
  * Feature     PT-1526 : Add ndb status to pt-mysql-summary (Thanks Fernando Ipar)
  * Feature     PT-1525 : Added support for MySQL 8 roles into pt-mysql-summary
  * Feature     PT-1509 : Only set binlog_format when necessary (Thanks Moritz Lenz)

Makefile.PL

@@ -2,7 +2,7 @@ use ExtUtils::MakeMaker;
 
 WriteMakefile(
     NAME      => 'percona-toolkit',
-    VERSION   => '3.0.9',
+    VERSION   => '3.0.11',
     EXE_FILES => [ <bin/*> ],
     MAN1PODS  => {
       'docs/percona-toolkit.pod' => 'blib/man1/percona-toolkit.1p',

bin/pt-align

@@ -1359,6 +1359,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-align 3.0.9
+pt-align 3.0.11
 
 =cut

bin/pt-archiver

@@ -45,7 +45,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -2020,7 +2020,7 @@ sub remove_quoted_text {
    my ($string) = @_;
    $string =~ s/[^\\]`[^`]*[^\\]`//g; 
    $string =~ s/[^\\]"[^"]*[^\\]"//g; 
-   $string =~ s/[^\\]"[^"]*[^\\]"//g; 
+   $string =~ s/[^\\]'[^']*[^\\]'//g; 
    return $string;
 }
 
@@ -7156,10 +7156,16 @@ sub escape {
  
    return join($fields_separated_by, map {
       s/([\t\n\\])/\\$1/g if defined $_;  # Escape tabs etc
-      $_ = defined $_ ? $_ : '\N';             # NULL = \N
+      my $s = defined $_ ? $_ : '\N';             # NULL = \N
       # var & ~var will return 0 only for numbers
-      $_ =~ s/([^\\])"/$1\\"/g if ($_ !~ /^[0-9,.E]+$/  && $optionally_enclosed_by eq '"');
-      $_ = $optionally_enclosed_by && $_ & ~$_ ? $optionally_enclosed_by."$_".$optionally_enclosed_by : $_;
+      if ($s !~ /^[0-9,.E]+$/  && $optionally_enclosed_by eq '"') {
+          $s =~ s/([^\\])"/$1\\"/g;
+          $s = $optionally_enclosed_by."$s".$optionally_enclosed_by;
+      }
+      # $_ =~ s/([^\\])"/$1\\"/g if ($_ !~ /^[0-9,.E]+$/  && $optionally_enclosed_by eq '"');
+      # $_ = $optionally_enclosed_by && ($_ & ~$_) ? $optionally_enclosed_by."$_".$optionally_enclosed_by : $_;
+      chomp $s;
+      $s;
    } @$row);
 
 }
@@ -8579,6 +8585,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-archiver 3.0.9
+pt-archiver 3.0.11
 
 =cut

bin/pt-config-diff

@@ -43,7 +43,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -5884,6 +5884,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-config-diff 3.0.9
+pt-config-diff 3.0.11
 
 =cut

bin/pt-deadlock-logger

@@ -42,7 +42,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -5674,6 +5674,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-deadlock-logger 3.0.9
+pt-deadlock-logger 3.0.11
 
 =cut

bin/pt-diskstats

@@ -38,7 +38,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -5668,6 +5668,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-diskstats 3.0.9
+pt-diskstats 3.0.11
 
 =cut

bin/pt-duplicate-key-checker

@@ -39,7 +39,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -407,7 +407,7 @@ sub remove_quoted_text {
    my ($string) = @_;
    $string =~ s/[^\\]`[^`]*[^\\]`//g; 
    $string =~ s/[^\\]"[^"]*[^\\]"//g; 
-   $string =~ s/[^\\]"[^"]*[^\\]"//g; 
+   $string =~ s/[^\\]'[^']*[^\\]'//g; 
    return $string;
 }
 
@@ -5737,6 +5737,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-duplicate-key-checker 3.0.9
+pt-duplicate-key-checker 3.0.11
 
 =cut

bin/pt-fifo-split

@@ -1648,6 +1648,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-fifo-split 3.0.9
+pt-fifo-split 3.0.11
 
 =cut

bin/pt-find

@@ -35,7 +35,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -1935,7 +1935,7 @@ sub remove_quoted_text {
    my ($string) = @_;
    $string =~ s/[^\\]`[^`]*[^\\]`//g; 
    $string =~ s/[^\\]"[^"]*[^\\]"//g; 
-   $string =~ s/[^\\]"[^"]*[^\\]"//g; 
+   $string =~ s/[^\\]'[^']*[^\\]'//g; 
    return $string;
 }
 
@@ -5098,6 +5098,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-find 3.0.9
+pt-find 3.0.11
 
 =cut

bin/pt-fingerprint

@@ -2239,6 +2239,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-fingerprint 3.0.9
+pt-fingerprint 3.0.11
 
 =cut

bin/pt-fk-error-logger

@@ -37,7 +37,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -4660,6 +4660,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-fk-error-logger 3.0.9
+pt-fk-error-logger 3.0.11
 
 =cut

bin/pt-heartbeat

@@ -44,7 +44,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -3591,7 +3591,7 @@ sub remove_quoted_text {
    my ($string) = @_;
    $string =~ s/[^\\]`[^`]*[^\\]`//g; 
    $string =~ s/[^\\]"[^"]*[^\\]"//g; 
-   $string =~ s/[^\\]"[^"]*[^\\]"//g; 
+   $string =~ s/[^\\]'[^']*[^\\]'//g; 
    return $string;
 }
 
@@ -7339,6 +7339,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-heartbeat 3.0.9
+pt-heartbeat 3.0.11
 
 =cut

bin/pt-index-usage

@@ -45,7 +45,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -3174,7 +3174,7 @@ sub remove_quoted_text {
    my ($string) = @_;
    $string =~ s/[^\\]`[^`]*[^\\]`//g; 
    $string =~ s/[^\\]"[^"]*[^\\]"//g; 
-   $string =~ s/[^\\]"[^"]*[^\\]"//g; 
+   $string =~ s/[^\\]'[^']*[^\\]'//g; 
    return $string;
 }
 
@@ -7667,6 +7667,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-index-usage 3.0.9
+pt-index-usage 3.0.11
 
 =cut

bin/pt-ioprofile

@@ -1127,7 +1127,7 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-ioprofile 3.0.9
+pt-ioprofile 3.0.11
 
 =cut
 

bin/pt-kill

@@ -47,7 +47,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -3000,7 +3000,7 @@ sub remove_quoted_text {
    my ($string) = @_;
    $string =~ s/[^\\]`[^`]*[^\\]`//g; 
    $string =~ s/[^\\]"[^"]*[^\\]"//g; 
-   $string =~ s/[^\\]"[^"]*[^\\]"//g; 
+   $string =~ s/[^\\]'[^']*[^\\]'//g; 
    return $string;
 }
 
@@ -8495,6 +8495,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-kill 3.0.9
+pt-kill 3.0.11
 
 =cut

bin/pt-mext

@@ -599,9 +599,9 @@ for i in `_seq $NUM`; do
    NEXTFILE=$(($i + 1))
 
    # Sort each file and eliminate empty lines, so 'join' doesn't complain.
-   sort "$FILE$i" | grep . > "$FILE$i.tmp"
+   sort -s "$FILE$i" | grep . > "$FILE$i.tmp"
    mv "$FILE$i.tmp" "$FILE$i"
-   sort "$FILE${NEXTFILE}" | grep . > "$FILE${NEXTFILE}.tmp"
+   sort -s "$FILE${NEXTFILE}" | grep . > "$FILE${NEXTFILE}.tmp"
    mv "$FILE${NEXTFILE}.tmp" "$FILE${NEXTFILE}"
 
    # Join the files together.  This gets slow O(n^2) as we add more files, but
@@ -610,7 +610,7 @@ for i in `_seq $NUM`; do
 
    # Find the max length of the [numeric only] values in the file so we know how
    # wide to make the columns
-   MAXLEN=`awk '{print $2}' "$FILE${NEXTFILE}" | grep -v '[^0-9]' | awk '{print length($1)}' | sort -rn | head -n1`
+   MAXLEN=`awk '{print $2}' "$FILE${NEXTFILE}" | grep -v '[^0-9]' | awk '{print length($1)}' | sort -rns | head -n1`
    mv "$FILE" "$FILE${NEXTFILE}"
    SPEC="$SPEC %${MAXLEN}d";
 
@@ -804,7 +804,7 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-mext 3.0.9
+pt-mext 3.0.11
 
 =cut
 

bin/pt-mysql-summary

@@ -827,7 +827,7 @@ setup_data_dir () {
 get_var () {
    local varname="$1"
    local file="$2"
-   awk -v pattern="${varname}" '$1 == pattern { if (length($2)) { len = length($1); print substr($0, len+index(substr($0, len+1), $2)) } }' "${file}"
+   awk -v pattern="${varname}" '$1 == pattern { if (length($2)) { len = length($1); print substr($0, len+index(substr($0, len+1), $2)) } }' "${file}" | tr -d '\r'
 }
 
 # ###########################################################################
@@ -1185,6 +1185,10 @@ collect_encrypted_tables() {
     $CMD_MYSQL $EXT_ARGV --table -ss -e "SELECT TABLE_SCHEMA, TABLE_NAME, CREATE_OPTIONS FROM INFORMATION_SCHEMA.TABLES WHERE CREATE_OPTIONS LIKE '%ENCRYPTION=\"Y\"%';"
 }
 
+collect_encrypted_tablespaces() {
+    $CMD_MYSQL $EXT_ARGV --table -ss -e "SELECT SPACE, NAME, SPACE_TYPE from INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES where FLAG&8192 = 8192;"
+}
+
 
 
 _NO_FALSE_NEGATIVES=""
@@ -1630,6 +1634,14 @@ format_encrypted_tables() {
    fi
 }
 
+format_encrypted_tablespaces() {
+   local encrypted_tablespaces="$1"
+   if [ ! -z "$encrypted_tablespaces" ]; then
+       echo "Encrypted tablespaces:"
+       echo "$encrypted_tablespaces"
+   fi
+}
+
 format_mysql_roles() {
    local file=$1
    [ -e "$file" ] || return
@@ -2253,25 +2265,20 @@ report_jemalloc_enabled() {
   local GENERAL_JEMALLOC_STATUS=0
   local JEMALLOC_LOCATION=''
 
-  for PID in $(pidof mysqld); do
-     grep -qc jemalloc /proc/${PID}/environ || ldd $(which mysqld) 2>/dev/null | grep -qc jemalloc
-     JEMALLOC_STATUS=$?
-     if [ $JEMALLOC_STATUS = 1 ]; then
-       echo "jemalloc is not enabled in MySQL config for process with ID ${PID}" 
+  for pid in $(pidof mysqld); do
+     grep -qc jemalloc /proc/${pid}/environ || ldd $(which mysqld) 2>/dev/null | grep -qc jemalloc
+     jemalloc_status=$?
+     if [ $jemalloc_status = 1 ]; then
+       echo "jemalloc is not enabled in mysql config for process with id ${pid}" 
      else
-       echo "jemalloc enabled in MySQL config for process with ID ${PID}"
+       echo "jemalloc enabled in mysql config for process with id ${pid}"
        GENERAL_JEMALLOC_STATUS=1
      fi
   done
 
-  if [ $GENERAL_JEMALLOC_STATUS = 1 ]; then
-     for libjemall in "/usr/lib64" "/usr/lib/x86_64-linux-gnu" "/usr/lib"; do
-       if [ -r "$libjemall/libjemalloc.so.1" ]; then
-         JEMALLOC_LOCATION="$libjemall/libjemalloc.so.1"
-         break
-       fi
-     done
-     if [ -z $JEMALLOC_LOCATION ]; then
+  if [ $GENERAL_JEMALLOC_STATUS -eq 1 ]; then
+     JEMALLOC_LOCATION=$(find /usr/lib64/ /usr/lib/x86_64-linux-gnu /usr/lib -name "libjemalloc.*" 2>/dev/null | head -n 1)
+     if [ -z "$JEMALLOC_LOCATION" ]; then
        echo "Jemalloc library not found"
      else
        echo "Using jemalloc from $JEMALLOC_LOCATION"
@@ -2509,10 +2516,16 @@ report_mysql_summary () {
 
    section "Encryption"
    local keyring_plugins="$(collect_keyring_plugins)"
-   local encrypted_tables="$(collect_encrypted_tables)"
+   local encrypted_tables=""
+   local encrypted_tablespaces=""
+   if [ "${OPT_LIST_ENCRYPTED_TABLES}" = 'yes' ]; then 
+       encrypted_tables="$(collect_encrypted_tables)"
+       encrypted_tablespaces="$(collect_encrypted_tablespaces)"
+   fi
 
    format_keyring_plugins "$keyring_plugins" "$encrypted_tables"
    format_encrypted_tables "$encrypted_tables"
+   format_encrypted_tablespaces "$encrypted_tablespaces"
 
    section "Binary Logging"
 
@@ -2567,14 +2580,14 @@ check_mysql () {
    # Check that mysql and mysqldump are in PATH.  If not, we're
    # already dead in the water, so don't bother with cmd line opts,
    # just error and exit.
-   [ -n "$(mysql --help 2>/dev/null)" ] \
+   [ -n "$(${CMD_MYSQL} --help 2>/dev/null)" ] \
       || die "Cannot execute mysql.  Check that it is in PATH."
-   [ -n "$(mysqldump --help 2>/dev/null)" ] \
+   [ -n "$(${CMD_MYSQLDUMP} --help 2>/dev/null)" ] \
       || die "Cannot execute mysqldump.  Check that it is in PATH."
 
    # Now that we have the cmd line opts, check that we can actually
    # connect to MySQL.
-   [ -n "$(mysql $EXT_ARGV -e 'SHOW STATUS')" ] \
+   [ -n "$(${CMD_MYSQL} ${EXT_ARGV} -e 'SHOW STATUS')" ] \
       || die "Cannot connect to MySQL.  Check that MySQL is running and that the options after -- are correct."
 
 }
@@ -3132,6 +3145,13 @@ short form: -h; type: string
 
 Host to connect to.
 
+=item --list-encrypted-tables
+
+default: false
+
+Include a list of the encrypted tables in all databases. This can cause slowdowns since
+querying Information Schema tables can be slow.
+
 =item --password
 
 short form: -p; type: string
@@ -3267,7 +3287,7 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-mysql-summary 3.0.9
+pt-mysql-summary 3.0.11
 
 =cut
 

bin/pt-online-schema-change

@@ -56,7 +56,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -3364,7 +3364,7 @@ sub remove_quoted_text {
    my ($string) = @_;
    $string =~ s/[^\\]`[^`]*[^\\]`//g; 
    $string =~ s/[^\\]"[^"]*[^\\]"//g; 
-   $string =~ s/[^\\]"[^"]*[^\\]"//g; 
+   $string =~ s/[^\\]'[^']*[^\\]'//g; 
    return $string;
 }
 
@@ -5838,6 +5838,7 @@ sub can_nibble {
    my $one_nibble = !defined $args{one_nibble} || $args{one_nibble}
                   ? $row_est <= $chunk_size * $chunk_size_limit
                   : 0;
+
    PTDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no');
 
    if ( $args{resume}
@@ -5852,6 +5853,16 @@ sub can_nibble {
       die "There is no good index and the table is oversized.";
    }
 
+   if ($o->has('force-nibbling') && $o->get('force-nibbling')) {
+       my @nibbling_tables = split(/,/, $o->get('force-nibbling'));
+       my @table_in_list = grep (/$tbl->{tbl}/, @nibbling_tables);
+       if (@table_in_list && $index && $row_est >= 2) {
+           PTDEBUG && _d("Disabling one nibble for the entire table");
+           $one_nibble = 0;
+       }
+   }
+
+
    my $pause_file = ($o->has('pause-file') && $o->get('pause-file')) || undef;
    
    return {
@@ -9790,6 +9801,12 @@ sub main {
       }
    }
 
+   if ($server_version >= '8.0' && $alter_fk_method eq 'drop_swap') {
+       my $msg = "--alter-foreign-keys-method=drop_swap doesn't work with MySQL 8.0+\n".
+                 "See https://bugs.mysql.com/bug.php?id=89441";
+       die($msg);
+   }
+
    # --plugin hook
    if ( $plugin && $plugin->can('after_copy_rows') ) {
       $plugin->after_copy_rows();
@@ -13038,6 +13055,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-online-schema-change 3.0.9
+pt-online-schema-change 3.0.11
 
 =cut

bin/pt-pmp

@@ -897,7 +897,7 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-pmp 3.0.9
+pt-pmp 3.0.11
 
 =cut
 

bin/pt-query-digest

@@ -64,7 +64,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -2679,7 +2679,7 @@ sub any_unix_timestamp {
 
 sub make_checksum {
    my ( $val ) = @_;
-   my $checksum = uc substr(md5_hex($val), -16);
+   my $checksum = uc md5_hex($val);
    PTDEBUG && _d($checksum, 'checksum for', $val);
    return $checksum;
 }
@@ -7778,7 +7778,12 @@ sub explain_report {
             $explain .= "# *************************** $i. "
                       . "row ***************************\n";
             foreach my $j ( 0 .. $#row ) {
-               $explain .= sprintf "# %13s: %s\n", $sth->{NAME}->[$j],
+               # In some OSes/Perl versions, the filtered row can be reported with or without decimals.
+               # Example, in Ubuntu 16.04 it is being printed as 100.00 while in Ubuntu 18.04 it is
+               # being printed as 100.
+               # To make it testeable, we need to have a consistent format across versions.
+               my $value_format = $sth->{NAME}->[$j] eq 'filtered' ? "%.02f" : "%s";
+               $explain .= sprintf "# %13s: $value_format\n", $sth->{NAME}->[$j],
                   defined $row[$j] ? $row[$j] : 'NULL';
             }
             $i++;  # next row number
@@ -8937,7 +8942,7 @@ sub remove_quoted_text {
    my ($string) = @_;
    $string =~ s/[^\\]`[^`]*[^\\]`//g; 
    $string =~ s/[^\\]"[^"]*[^\\]"//g; 
-   $string =~ s/[^\\]"[^"]*[^\\]"//g; 
+   $string =~ s/[^\\]'[^']*[^\\]'//g; 
    return $string;
 }
 
@@ -9245,7 +9250,7 @@ sub new {
    my $sql = <<"      SQL";
       INSERT INTO $args{db_tbl}
       (checksum, fingerprint, sample, first_seen, last_seen)
-      VALUES(CONV(?, 16, 10), ?, ?, COALESCE(?, $now), COALESCE(?, $now))
+      VALUES(?, ?, ?, COALESCE(?, $now), COALESCE(?, $now))
       ON DUPLICATE KEY UPDATE
          first_seen = IF(
             first_seen IS NULL,
@@ -9262,8 +9267,8 @@ sub new {
    my @review_cols = grep { !$skip_cols{$_} } @{$args{tbl_struct}->{cols}};
    $sql = "SELECT "
         . join(', ', map { $args{quoter}->quote($_) } @review_cols)
-        . ", CONV(checksum, 10, 16) AS checksum_conv FROM $args{db_tbl}"
-        . " WHERE checksum=CONV(?, 16, 10)";
+        . ", checksum AS checksum_conv FROM $args{db_tbl}"
+        . " WHERE checksum=?";
    PTDEBUG && _d('SQL to select from review table:', $sql);
    my $select_sth = $args{dbh}->prepare($sql);
 
@@ -9393,7 +9398,7 @@ sub set_history_options {
    my $sql = "REPLACE INTO $args{table}("
       . join(', ',
          map { Quoter->quote($_) } ('checksum', 'sample', @cols))
-      . ') VALUES (CONV(?, 16, 10), ?'
+      . ') VALUES (?, ?'
       . (@cols ? ', ' : '')  # issue 1265
       . join(', ', map {
          $_ eq 'ts_min' || $_ eq 'ts_max'
@@ -15732,7 +15737,7 @@ pt-query-digest inspects the columns in the table.  The table must have at
 least the following columns:
 
   CREATE TABLE query_review_history (
-    checksum     BIGINT UNSIGNED NOT NULL,
+    checksum     CHAR(32) NOT NULL,
     sample       TEXT NOT NULL
   );
 
@@ -15762,7 +15767,7 @@ The following table definition is used for L<"--[no]create-history-table">:
 MAGIC_create_history_table
 
  CREATE TABLE IF NOT EXISTS query_history (
-   checksum             BIGINT UNSIGNED NOT NULL,
+   checksum             CHAR(32) NOT NULL,
    sample               TEXT NOT NULL,
    ts_min               DATETIME,
    ts_max               DATETIME,
@@ -15866,6 +15871,9 @@ MAGIC_create_history_table
 Note that we store the count (cnt) for the ts attribute only; it will be
 redundant to store this for other attributes.
 
+Starting from Percona Toolkit 3.0.11, the checksum function has been updated to use 32 chars in the MD5 sum.
+This causes the checksum field in the history table will have a different value than in the previous versions of the tool.
+
 =item --host
 
 short form: -h; type: string
@@ -16180,7 +16188,7 @@ by pt-query-digest.
 MAGIC_create_review_table:
 
   CREATE TABLE IF NOT EXISTS query_review (
-     checksum     BIGINT UNSIGNED NOT NULL PRIMARY KEY,
+     checksum     CHAR(32) NOT NULL PRIMARY KEY,
      fingerprint  TEXT NOT NULL,
      sample       TEXT NOT NULL,
      first_seen   DATETIME,
@@ -16901,6 +16909,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-query-digest 3.0.9
+pt-query-digest 3.0.11
 
 =cut

bin/pt-show-grants

@@ -1909,8 +1909,17 @@ sub main {
       . ($o->get('timestamp') ? ", MySQL $version at $ts" : ", MySQL $version"),
       ), "\n" if $o->get('header');
 
+   # MySQL 8 roles must be excluded from the regular users list.
+   # Roles can be identified because the user password is expired, the authentication 
+   # string is empty and the account is locked
+   my $mysql8_where = '';
+   if (VersionCompare::cmp($version, '8.0.0') >= 0) {
+       $mysql8_where = ' WHERE NOT ( `account_locked`="Y" AND ' .
+                                      ' `password_expired`="Y" AND ' .
+                                      ' `authentication_string`="" ) ';
+   }
    my $users = $o->get('only') || $dbh->selectall_arrayref(
-      'SELECT DISTINCT User, Host FROM mysql.user ORDER BY User, Host',
+      "SELECT DISTINCT User, Host FROM mysql.user $mysql8_where ORDER BY User, Host",
       { Slice => {} });
    if ( scalar @all_hosts ) {
       my $where = join(' OR ', map { "User='$_'" } @all_hosts);
@@ -1922,6 +1931,24 @@ sub main {
    my $ignore_users = $o->get('ignore');
 
    my $exit_status = 0;
+   my $roles = get_roles($dbh, $users);
+   if ($roles && scalar @$roles > 0) {
+       print "-- Roles\n";
+       my $count=0;
+
+       for my $role (@$roles) {
+           next if (!$o->get("include-unused-roles") && $role->{active} == 0);
+           unshift @$users, { Host => $role->{host}, User => $role->{name}, IsRole => 1};
+           $count++;
+           printf('CREATE ROLE IF NOT EXISTS `%s`;'."\n", $role->{name});
+       }
+
+       if ($count == 0) {
+           print "No active roles found\n";
+       }
+       print "-- End of roles listing\n";
+   }
+   
    USER:
    foreach my $u ( @$users ) {
       my $user_host = "'$u->{User}'\@'$u->{Host}'";
@@ -2040,7 +2067,7 @@ sub main {
             "\n";
       }
 
-      if ( $o->get('drop') ) {
+      if ( $o->get('drop') && !defined($u->{IsRole}) ) {
          print join("\n",
             "DROP USER $user_host;",
             "DELETE FROM `mysql`.`user` WHERE `User`='$u->{User}' AND `Host`='$u->{Host}';",
@@ -2084,6 +2111,42 @@ sub parse_user {
    return ( $user, $host );
 }
 
+sub get_roles {
+    my ($dbh, $users) = @_;
+    my $query = <<__EOQ;
+    SELECT DISTINCT user.user AS name, user.host, IF(from_user IS NULL,0, 1) AS active
+      FROM mysql.user
+ LEFT JOIN mysql.role_edges ON role_edges.from_user=user.user
+     WHERE `account_locked`='Y'
+       AND `password_expired`='Y'
+       AND `authentication_string`=''
+__EOQ
+    if (scalar $users > 0) {
+       my $user_names = join (", ", map { "'$_->{User}'" } @$users);
+       $query .= " AND to_user IN ($user_names)";
+    }
+    PTDEBUG && _d("Getting roles");
+    PTDEBUG && _d($query);                                      
+    my $roles;
+    eval { $roles = $dbh->selectall_arrayref($query, { Slice => {} }) };
+    if ($EVAL_ERROR) {
+        PTDEBUG && _d("Cannot list roles: $EVAL_ERROR");
+    }
+    return $roles;
+}
+
+sub is_role {
+    my ($users, $grant) = @_;
+    foreach my $u ( @$users ) {
+        my $user_host = "`$u->{User}`\@`$u->{Host}`";
+        warn "> user_host: $user_host";
+        if ($grant eq $user_host) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
 sub split_grants {
    my ($grants) = @_;
    return unless $grants;
@@ -2346,6 +2409,10 @@ example, specifying C<--set-vars wait_timeout=500> overrides the defaultvalue of
 
 The tool prints a warning and continues if a variable cannot be set.
 
+=item --[no]include-unused-roles
+
+When dumping MySQL 8+ roles, include unused roles.
+
 =item --socket
 
 short form: -S; type: string
@@ -2524,6 +2591,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-show-grants 3.0.9
+pt-show-grants 3.0.11
 
 =cut

bin/pt-sift

@@ -1245,7 +1245,7 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-sift 3.0.9
+pt-sift 3.0.11
 
 =cut
 

bin/pt-slave-delay

@@ -40,7 +40,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -4960,6 +4960,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-slave-delay 3.0.9
+pt-slave-delay 3.0.11
 
 =cut

bin/pt-slave-find

@@ -4482,6 +4482,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-slave-find 3.0.9
+pt-slave-find 3.0.11
 
 =cut

bin/pt-slave-restart

@@ -41,7 +41,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -6112,6 +6112,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-slave-restart 3.0.9
+pt-slave-restart 3.0.11
 
 =cut

bin/pt-stalk

@@ -916,7 +916,7 @@ collect() {
    local ps_instrumentation_enabled=$($CMD_MYSQL $EXT_ARGV -e 'SELECT ENABLED FROM performance_schema.setup_instruments WHERE NAME = "transaction";' \
                                       | sed "2q;d" | sed 'y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/')
 
-   if [ $ps_instrumentation_enabled != "yes" ]; then
+   if [ !$ps_instrumentation_enabled = "yes" ]; then
       log "Performance Schema instrumentation is disabled"
    fi
 
@@ -1162,26 +1162,25 @@ slave_status() {
    local outfile=$1
    local mysql_version=$2
 
-   if [ "${mysql_version}" '<' "5.7" ]; then
-      local sql="SHOW SLAVE STATUS\G"  
-      echo -e "\n$sql\n" >> $outfile
-      $CMD_MYSQL $EXT_ARGV -e "$sql" >> $outfile
-   else
+   local sql="SHOW SLAVE STATUS\G"   
+   echo -e "\n$sql\n" >> $outfile
+   $CMD_MYSQL $EXT_ARGV -e "$sql" >> $outfile
+   if [ "${mysql_version}" '>' "5.6" ]; then
       local sql="SELECT * FROM performance_schema.replication_connection_configuration JOIN performance_schema.replication_applier_configuration USING(channel_name)\G"
       echo -e "\n$sql\n" >> $outfile
       $CMD_MYSQL $EXT_ARGV -e "$sql" >> $outfile
 
-      sql="SELECT * FROM replication_connection_status\G"
+      sql="SELECT * FROM performance_schema.replication_connection_status\G"
       echo -e "\n$sql\n" >> $outfile
       $CMD_MYSQL $EXT_ARGV -e "$sql" >> $outfile
 
-      sql="SELECT * FROM replication_applier_status JOIN replication_applier_status_by_coordinator USING(channel_name)\G"
+      sql="SELECT * FROM performance_schema.replication_applier_status JOIN performance_schema.replication_applier_status_by_coordinator USING(channel_name)\G"
       echo -e "\n$sql\n" >> $outfile
       $CMD_MYSQL $EXT_ARGV -e "$sql" >> $outfile
    fi
-
 }
 
+
 collect_mysql_variables() {
    local outfile=$1 
 
@@ -2376,7 +2375,7 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-stalk 3.0.9
+pt-stalk 3.0.11
 
 =cut
 

bin/pt-summary

@@ -2723,7 +2723,7 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-summary 3.0.9
+pt-summary 3.0.11
 
 =cut
 

bin/pt-table-checksum

@@ -58,7 +58,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -4543,7 +4543,7 @@ sub remove_quoted_text {
    my ($string) = @_;
    $string =~ s/[^\\]`[^`]*[^\\]`//g; 
    $string =~ s/[^\\]"[^"]*[^\\]"//g; 
-   $string =~ s/[^\\]"[^"]*[^\\]"//g; 
+   $string =~ s/[^\\]'[^']*[^\\]'//g; 
    return $string;
 }
 
@@ -6238,9 +6238,9 @@ sub find_replication_differences {
    }
    my ($dbh, $repl_table) = @args{@required_args};
 
-    
    my $tries = $self->{'OptionParser'}->get('replicate-check-retries') || 1; 
    my $diffs;
+
    while ($tries--) {
       my $sql
          = "SELECT CONCAT(db, '.', tbl) AS `table`, "
@@ -6258,7 +6258,7 @@ sub find_replication_differences {
       if (!@$diffs || !$tries) { # if no differences are found OR we are out of tries left...
          last;                   # get out now
       }
-      sleep 1;            
+      sleep 1;
    }
    return $diffs;
 }
@@ -6689,6 +6689,7 @@ sub can_nibble {
    my $one_nibble = !defined $args{one_nibble} || $args{one_nibble}
                   ? $row_est <= $chunk_size * $chunk_size_limit
                   : 0;
+
    PTDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no');
 
    if ( $args{resume}
@@ -6703,6 +6704,16 @@ sub can_nibble {
       die "There is no good index and the table is oversized.";
    }
 
+   if ($o->has('force-nibbling') && $o->get('force-nibbling')) {
+       my @nibbling_tables = split(/,/, $o->get('force-nibbling'));
+       my @table_in_list = grep (/$tbl->{tbl}/, @nibbling_tables);
+       if (@table_in_list && $index && $row_est >= 2) {
+           PTDEBUG && _d("Disabling one nibble for the entire table");
+           $one_nibble = 0;
+       }
+   }
+
+
    my $pause_file = ($o->has('pause-file') && $o->get('pause-file')) || undef;
    
    return {
@@ -9847,6 +9858,7 @@ use sigtrap 'handler', \&sig_int, 'normal-signals';
 my $oktorun      = 1;
 my $print_header = 1;
 my $exit_status  = 0;
+my $original_qrt_plugin_master_status = undef;
 
 # "exit codes 1 - 2, 126 - 165, and 255 [1] have special meanings,
 # and should therefore be avoided for user-specified exit parameters"
@@ -10052,30 +10064,28 @@ sub main {
       # instead, it should check if it's already set to STATEMENT.
       # This is becase starting with MySQL 5.1.29, changing the format
       # requires a SUPER user.
-      if ( $o->get('check-binlog-format') ) {
-          if ( VersionParser->new($dbh) >= '5.1.5' ) {
-             $sql = 'SELECT @@binlog_format';
-             PTDEBUG && _d($dbh, $sql);
-             my ($original_binlog_format) = $dbh->selectrow_array($sql);
-             PTDEBUG && _d('Original binlog_format:', $original_binlog_format);
-             if ( $original_binlog_format !~ /STATEMENT/i ) {
-                $sql = q{/*!50108 SET @@binlog_format := 'STATEMENT'*/};
-                eval {
-                   PTDEBUG && _d($dbh, $sql);
-                   $dbh->do($sql);
-                };
-                if ( $EVAL_ERROR ) {
-                   die "Failed to $sql: $EVAL_ERROR\n"
-                      . "This tool requires binlog_format=STATEMENT, "
-                      . "but the current binlog_format is set to "
-                      ."$original_binlog_format and an error occurred while "
-                      . "attempting to change it.  If running MySQL 5.1.29 or newer, "
-                      . "setting binlog_format requires the SUPER privilege.  "
-                      . "You will need to manually set binlog_format to 'STATEMENT' "
-                      . "before running this tool.\n";
-                }
-             }
-          }
+      if ( VersionParser->new($dbh) >= '5.1.5' ) {
+         $sql = 'SELECT @@binlog_format';
+         PTDEBUG && _d($dbh, $sql);
+         my ($original_binlog_format) = $dbh->selectrow_array($sql);
+         PTDEBUG && _d('Original binlog_format:', $original_binlog_format);
+         if ( $original_binlog_format !~ /STATEMENT/i ) {
+            $sql = q{/*!50108 SET @@binlog_format := 'STATEMENT'*/};
+            eval {
+               PTDEBUG && _d($dbh, $sql);
+               $dbh->do($sql);
+            };
+            if ( $EVAL_ERROR ) {
+               die "Failed to $sql: $EVAL_ERROR\n"
+                  . "This tool requires binlog_format=STATEMENT, "
+                  . "but the current binlog_format is set to "
+                  ."$original_binlog_format and an error occurred while "
+                  . "attempting to change it.  If running MySQL 5.1.29 or newer, "
+                  . "setting binlog_format requires the SUPER privilege.  "
+                  . "You will need to manually set binlog_format to 'STATEMENT' "
+                  . "before running this tool.\n";
+            }
+         }
       }
 
       # Set transaction isolation level. We set binlog_format to STATEMENT,
@@ -10103,6 +10113,7 @@ sub main {
             . "level to REPEATABLE-READ.\n";
       }
 
+
       return;
    };
 
@@ -10131,6 +10142,20 @@ sub main {
    my $master_dbh = $master_cxn->dbh();  # just for brevity
    my $master_dsn = $master_cxn->dsn();  # just for brevity
 
+   if ($o->get('disable-qrt-plugin')) {
+       eval {
+           $master_dbh->selectrow_arrayref('SELECT @@query_response_time_session_stats' );
+       };
+       if ($EVAL_ERROR) {
+           $original_qrt_plugin_master_status = undef;
+           PTDEBUG && _d('QRT plugin is not installed: '.$EVAL_ERROR);
+       } else {
+           ($original_qrt_plugin_master_status) = $master_dbh->selectrow_arrayref('SELECT @@query_response_time_stats' );
+           PTDEBUG && _d("Disabling qrt plugin on master server");
+           $master_dbh->do('SET GLOBAL query_response_time_stats = off');
+       }
+   }
+
    my @ignored_engines = keys %{$o->get('ignore-engines')};
    my @rocksdb_ignored = grep(/^ROCKSDB$/i, @ignored_engines); 
    if (!@rocksdb_ignored) {
@@ -10451,6 +10476,23 @@ sub main {
          }
       }
 
+      for my $slave (@$slaves) {
+          my $qrt_plugin_status;
+          eval {
+              ($qrt_plugin_status) = $slave->{dbh}->selectrow_arrayref('SELECT @@QUERY_RESPONSE_TIME_SESSION_STATS' );
+          };
+          if ($EVAL_ERROR) {
+              PTDEBUG && _d('QRT plugin is not installed on slave '.$slave->{dsn_name});
+              $slave->{qrt_plugin_status} = undef;
+              next;
+          }
+          $slave->{qrt_plugin_status} = $qrt_plugin_status->[0];
+          if ($slave->{qrt_plugin_status}) {
+              PTDEBUG && _d("Disabling qrt plugin state on slave ".$slave->{dsn_name});
+              $slave->{dbh}->do('SET GLOBAL query_response_time_stats = off');
+          }
+      }
+
       if ( $o->get('check-slave-lag') ) {
          PTDEBUG && _d('Will use --check-slave-lag to check for slave lag');
          my $cxn = $make_cxn->(
@@ -11196,6 +11238,7 @@ sub main {
 
                # Update chunk-size based on rows/s checksum rate.
                $nibble_iter->set_chunk_size($tbl->{chunk_size});
+               PTDEBUG && _d('Updated chunk size: '.$tbl->{chunk_size});
             }
 
             # Every table should have a Progress obj; update it.
@@ -11239,6 +11282,10 @@ sub main {
 
             # Wait for the last checksum of this table to replicate
             # to each slave.
+            # MySQL 8+ replication is slower than 5.7 and the old wait_for_last_checksum alone 
+            # was failing. The new wait_for_slaves checks that Read_Master_Log_Pos on slaves is 
+            # greather or equal Position in the master
+            wait_for_slaves(master_dbh => $args{Cxn}->dbh(), master_slave => $ms, slaves => $slaves);
             wait_for_last_checksum(
                tbl          => $tbl,
                repl_table   => $repl_table,
@@ -11269,6 +11316,12 @@ sub main {
                      map { $diff_chunks{ $_->{chunk} }++ } @$diffs;
                      $exit_status |= $PTC_EXIT_STATUS{TABLE_DIFF};
                   }
+                  my $max_cnt_diff=0;
+                  for my $diff (@$diffs) {
+                     if ( $diff->{cnt_diff} > $max_cnt_diff ) {
+                         $tbl->{checksum_results}->{max_rows_cnt_diff} = $diff->{cnt_diff};
+                     }
+                  }
                };
                if ($EVAL_ERROR) {
                   if ( $o->get('quiet') < 2 ) {
@@ -11453,6 +11506,25 @@ sub main {
       }
    }
 
+   # Restore origin QRT pligin state
+   if ($o->get('disable-qrt-plugin')) {
+       eval {
+           if ($original_qrt_plugin_master_status) {
+               PTDEBUG && _d("Restoring qrt plugin state on master server");
+               $master_dbh->do("SET GLOBAL query_response_time_stats = $original_qrt_plugin_master_status->[0]");
+           }
+           for my $slave (@$slaves) {
+               if ($slave->{qrt_plugin_status}) {
+                   PTDEBUG && _d("Restoring qrt plugin state on slave ".$slave->{dsn_name});
+                   $slave->{dbh}->do("SET GLOBAL query_response_time_stats = $slave->{qrt_plugin_status}");
+               }
+           }
+       };
+       if ($EVAL_ERROR) {
+           warn "Cannot restore qrt_plugin status: $EVAL_ERROR";
+       }
+   }
+
    PTDEBUG && _d('Exit status', $exit_status,
                  'oktorun',     $oktorun,
                  'have time',   $have_time->());
@@ -11703,8 +11775,8 @@ sub exec_nibble {
 }
 
 {
-my $line_fmt = "%14s %6s %6s %8s %7s %7s %7s %-s\n";
-my @headers  =  qw(TS ERRORS DIFFS ROWS CHUNKS SKIPPED TIME TABLE);
+my $line_fmt = "%14s %6s %6s %8s % 10s %7s %7s %7s %-s\n";
+my @headers  =  qw(TS ERRORS DIFFS ROWS DIFF_ROWS CHUNKS SKIPPED TIME TABLE);
 
 sub print_checksum_results {
    my (%args) = @_;
@@ -11724,7 +11796,8 @@ sub print_checksum_results {
       ts(),
       $res->{errors}   || 0,
       $res->{diffs}    || 0,
-      $res->{n_rows}   || 0,
+      $res->{n_rows}   || 0,            
+      $tbl->{checksum_results}->{max_rows_cnt_diff} || 0,
       $res->{n_chunks} || 0,
       $res->{skipped}  || 0,
       sprintf('%.3f', $res->{start_time} ? time - $res->{start_time} : 0),
@@ -12279,6 +12352,20 @@ sub have_more_chunks {
    return 1; # more chunks
 }
 
+sub wait_for_slaves {
+   my (%args) = @_;
+   my @required_args = qw(master_dbh master_slave slaves);
+   foreach my $arg ( @required_args ) {
+       die "I need a $arg argument" unless $args{$arg};
+   }
+   my ($master_dbh, $ms, $slaves) = @args{@required_args};
+
+   my $master_status = $ms->get_master_status($master_dbh);
+   foreach my $slave ( @$slaves ) {
+       $ms->wait_for_master(master_status => $master_status, slave_dbh => $slave->dbh());
+   }
+}
+
 sub wait_for_last_checksum {
    my (%args) = @_;
    my @required_args = qw(tbl repl_table slaves max_chunk have_time OptionParser);
@@ -13138,6 +13225,10 @@ short form: -F; type: string; group: Connection
 Only read mysql options from the given file.  You must give an absolute
 pathname.
 
+=item --disable-qrt-plugin
+
+Disable the QRT (Query Response Time) plugin if it is enabled.
+
 =item --[no]empty-replicate-table
 
 default: yes
@@ -13182,6 +13273,16 @@ the values are converted to strings by the CONCAT() function, and MySQL chooses
 the string representation.  If you specify a value of 2, for example, then the
 values 1.008 and 1.009 will be rounded to 1.01, and will checksum as equal.
 
+=item --force-nibbling
+
+type: string
+
+Comma sepatered list of tables on which we want to force nibbling instead of
+checksuming the whole table in only one chunk,
+In some cases, MySQL returns a wrong estimate about number of rows in a table 
+and NibbleIterator is trying to process huge tables in a single chunk.
+This parameter disable one nibble checksums for the specified tables.
+
 =item --function
 
 type: string
@@ -13950,6 +14051,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-table-checksum 3.0.9
+pt-table-checksum 3.0.11
 
 =cut

bin/pt-table-sync

@@ -55,7 +55,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -2934,7 +2934,7 @@ sub remove_quoted_text {
    my ($string) = @_;
    $string =~ s/[^\\]`[^`]*[^\\]`//g; 
    $string =~ s/[^\\]"[^"]*[^\\]"//g; 
-   $string =~ s/[^\\]"[^"]*[^\\]"//g; 
+   $string =~ s/[^\\]'[^']*[^\\]'//g; 
    return $string;
 }
 
@@ -13044,6 +13044,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-table-sync 3.0.9
+pt-table-sync 3.0.11
 
 =cut

bin/pt-table-usage

@@ -6850,7 +6850,7 @@ sub remove_quoted_text {
    my ($string) = @_;
    $string =~ s/[^\\]`[^`]*[^\\]`//g; 
    $string =~ s/[^\\]"[^"]*[^\\]"//g; 
-   $string =~ s/[^\\]"[^"]*[^\\]"//g; 
+   $string =~ s/[^\\]'[^']*[^\\]'//g; 
    return $string;
 }
 
@@ -8487,6 +8487,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-table-usage 3.0.9
+pt-table-usage 3.0.11
 
 =cut

bin/pt-upgrade

@@ -61,7 +61,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -11414,6 +11414,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-upgrade 3.0.9
+pt-upgrade 3.0.11
 
 =cut

bin/pt-variable-advisor

@@ -44,7 +44,7 @@ BEGIN {
 {
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';
@@ -6229,6 +6229,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-variable-advisor 3.0.9
+pt-variable-advisor 3.0.11
 
 =cut

bin/pt-visual-explain

@@ -3281,6 +3281,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-pt-visual-explain 3.0.9
+pt-visual-explain 3.0.11
 
 =cut

config/deb/changelog

@@ -1,3 +1,34 @@
+percona-toolkit (3.0.11-1) unstable; urgency=low
+
+ * Improvement PT-1571 : Improved hostname recognition in pt-secure-collect
+ * Fixed bug   PT-1570 : pt-archiver fails to detect columns with the word GENERATED as part of the comment
+ * Improvement PT-1569 : Disabled --alter-foreign-keys-method=drop_swap in pt-osc
+ * Fixed bug   PT-1563 : Fixed pt-show-grants for MySQL 5.6
+ * Improvement PT-1562 : pt-mysql-summary: Fix mysqld command for Travis
+ * Fixed bug   PT-1551 : pt-table-checksum fails on MySQL 8.0.11
+ * Improvement  PT-242 : (pt-stalk) Include SHOW SLAVE STATUS on 5.7 (Thanks Marcelo Altmann)
+ * Fixed bug    PT-241 : (pt-stalk) Slave queries doesn't run on 5.7 (Thanks Marcelo Altmann)
+
+ -- Percona Toolkit Developers <toolkit-dev@percona.com>  Fri, 06 Jul 2018 15:07:41 +0000
+
+percona-toolkit (3.0.11-1) unstable; urgency=low
+
+ * Improvement PT-1571 : Improved hostname recognition in pt-secure-collect
+ * Fixed bug   PT-1570 : pt-archiver fails to detect columns with the word GENERATED as part of the comment
+ * Improvement PT-1569 : Disabled --alter-foreign-keys-method=drop_swap in pt-osc
+ * Fixed bug   PT-1563 : Fixed pt-show-grants for MySQL 5.6
+ * Improvement PT-1562 : pt-mysql-summary: Fix mysqld command for Travis
+ * Fixed bug   PT-1551 : pt-table-checksum fails on MySQL 8.0.11
+ * Improvement  PT-242 : (pt-stalk) Include SHOW SLAVE STATUS on 5.7 (Thanks Marcelo Altmann)
+ * Fixed bug    PT-241 : (pt-stalk) Slave queries doesn't run on 5.7 (Thanks Marcelo Altmann)
+
+ -- Percona Toolkit Developers <toolkit-dev@percona.com>  Tue, 03 Jul 2018 12:54:53 +0000
+
+percona-toolkit (3.0.10-1) unstable; urgency=low
+
+
+ -- Percona Toolkit Developers <toolkit-dev@percona.com>  Mon, 21 May 2018 17:36:10 +0000
+
 percona-toolkit (3.0.9-1) unstable; urgency=low
 
 

config/sphinx-build/conf.py

@@ -50,7 +50,7 @@ copyright = u'2017, Percona LLC and/or its affiliates'
 # The short X.Y version.
 version = '3.0'
 # The full version, including alpha/beta/rc tags.
-release = '3.0.9'
+release = '3.0.11'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/percona-toolkit.pod

@@ -102,10 +102,6 @@ Aggregate GDB stack traces for a selected program.
 
 Analyze MySQL queries from logs, processlist, and tcpdump.
 
-=item pt-secure-collect
-
-Collect, sanitize, pack and encrypt pt-tools outputs.
-
 =item pt-show-grants
 
 Canonicalize and print MySQL grants so you can effectively replicate, compare and version-control them.
@@ -564,6 +560,6 @@ Place, Suite 330, Boston, MA  02111-1307  USA.
 
 =head1 VERSION
 
-Percona Toolkit v3.0.9 released 2018-04-17
+Percona Toolkit v3.0.11 released 2018-07-06
 
 =cut

docs/rn.3-0-1.txt

@@ -0,0 +1,26 @@
+
+v3.0.1 released 2017-02-20
+==========================
+
+Percona Toolkit 3.0.1 GA includes the following changes:
+
+* Added requirement to run ``pt-mongodb-summary`` as a user with the ``clusterAdmin`` or ``root`` built-in roles.
+
+v3.0 released 2017-02-06
+========================
+
+Percona Toolkit 3.0.0 RC includes the following changes:
+
+New Features
+
+* Added ``pt-mongodb-summary`` tool
+
+* Added ``pt-mongodb-query-digest`` tool
+
+Bug fixes
+
+* 1402776: Updated ``MySQLProtocolParser`` to fix error when parsing ``tcpdump`` capture with ``pt-query-digest``
+
+* 1632522: Fixed failure of ``pt-online-schema-change`` when altering a table with a self-referencing foreign key (Thanks Amiel Marqeta)
+
+* 1654668: Fixed failure of ``pt-summary`` on Red Hat and derivatives (Thanks Marcelo Altmann)

docs/rn.3-0-10.txt

@@ -0,0 +1,19 @@
+
+v3.0.10 released 2018-05-21
+===========================
+
+New Features
+
+* :jirabug:`PT-131`: ``pt-table-checksum`` disables the QRT plugin
+* :jirabug:`PT-118`: ``pt-table-checksum`` report the number of rows
+  of difference between master and slave.
+
+Improvements
+
+* :jirabug:`PT-1546`: Improved support of MySQL 8 roles
+* :jirabug:`PT-1543`: The encrypted table status query causes high load over multiple minutes
+* :jirabug:`PT-1536`: Added info about encrypted tablespaces in ``pt-mysql-summary``
+
+Bug Fixes
+
+* :jirabug:`PT-1556`: ``pt-table-checksum`` 3.0.9 does not change ``binlog_format`` to statement any more.

docs/rn.3-0-11.txt

@@ -0,0 +1,30 @@
+v3.0.11 released 2018-07-06
+===========================
+
+New features
+
+* :jirabug:`PT-1571`: Improved hostname recognition in ``pt-secure-collect``
+* :jirabug:`PT-1569`: Disabled ``--alter-foreign-keys-method=drop_swap`` in ``pt-online-schema-change``
+* :jirabug:`PT-242`: (``pt-stalk``) Include ``SHOW SLAVE STATUS`` on MySQL 5.7 (Thanks `Marcelo Altmann <https://www.percona.com/blog/author/marcelo-altmann/>`_)
+
+Fixed bugs
+
+* :jirabug:`PT-1570`: ``pt-archiver`` fails to detect columns with the word *GENERATED* as part of the comment
+* :jirabug:`PT-1563`: ``pt-show-grants`` fails for MySQL 5.6 producing an error which reports that an unknown column **account_locked** has been detected.
+* :jirabug:`PT-1551`: ``pt-table-checksum`` fails on MySQL 8.0.11
+* :jirabug:`PT-241`: (``pt-stalk``) Slave queries don\'t run on MySQL 5.7 because the FQDN is missing (Thanks `Marcelo Altmann <https://www.percona.com/blog/author/marcelo-altmann/>`_)
+
+Breaking changes:
+
+Starting with this version, the queries checksum in ``pt-query-digest`` will
+store the full MD5 field as a CHAR(32) field instead of storing just the least
+significant bytes of the checksum as a BIGINT field.  The reason for this
+change is that storing only the least significant bytes as a BIGINT was
+producing inconsistent results in MySQL 8 compared to MySQL 5.6+.
+
+``pt-online-schema-change`` in MySQL 8:
+
+Due to a `bug in MySQL 8.0+ <https://bugs.mysql.com/bug.php?id=89441>`_, it
+is not possible to use the ``drop_swap`` method to rebuild constraints because
+renaming a table will result in losing the foreign keys. You must specify a
+different method explicitly.

docs/rn.3-0-2.txt

@@ -0,0 +1,31 @@
+
+v3.0.2 released 2017-03-27
+==========================
+
+Percona Toolkit 3.0.2 includes the following changes:
+
+New Features
+
+* PT-73: Added support for SSL connections to ``pt-mongodb-summary`` and ``pt-mongodb-query-digest``
+
+* 1642751: Enabled gathering of information about locks and transactions by ``pt-stalk`` using Performance Schema if it is enabled (Thanks Agustin Gallego)
+
+Bug Fixes
+
+* PT-74: Fixed gathering of security settings when running ``pt-mongodb-summary`` on a mongod instance that is specified as the host
+
+* PT-75: Changed the default sort order in ``pt-mongodb-query-digest`` output to descending
+
+* PT-76: Added support of ``&`` and ``#`` symbols in passwords for ``pt-mysql-summary``
+
+* PT-77: Updated ``Makefile`` to support new MongoDB tools
+
+* PT-89: Fixed ``pt-stalk`` to run ``top`` more than once to collect useful CPU usage
+
+* PT-93: Fixed ``pt-mongodb-query-digest`` to make query ID match query key (Thanks Kamil Dziedzic)
+
+* PT-94: Fixed ``pt-online-schema-change`` to not make duplicate rows in ``_t_new`` when updating primary key. Also see 1646713.
+
+* PT-101: Fixed ``pt-table-checksum`` to correctly use the ``--slave-user`` and ``--slave-password`` options. Also see 1651002.
+
+* PT-105: Fixed ``pt-table-checksum`` to continue running if a database is dropped in the process

docs/rn.3-0-3.txt

@@ -0,0 +1,29 @@
+
+v3.0.3 released 2017-05-18
+==========================
+
+Percona Toolkit 3.0.3 includes the following changes:
+
+New Features
+
+* Added the ``--skip-check-slave-lag`` option for ``pt-table-checksum``, ``pt-online-schema-change``, and ``pt-archiver``.
+
+  This option can be used to specify list of servers where to skip checking for slave lag.
+
+* 1642754: Added support for collecting replication slave information in ``pt-stalk``.
+
+* PT-111: Added support for collecting information about variables from Performance Schema in ``pt-stalk``. For more information, see 1642753.
+
+* PT-116: Added the ``--[no]use-insert-ignore`` option for ``pt-online-schema-change`` to force or prevent using ``IGNORE`` on ``INSERT`` statements. For more information, see 1545129.
+
+Bug Fixes
+
+* PT-115: Fixed ``OptionParser`` to accept repeatable DSNs.
+
+* PT-126: Fixed ``pt-online-schema-change`` to correctly parse comments. For more information, see 1592072.
+
+* PT-128: Fixed ``pt-stalk`` to include memory usage information. For more information, see 1510809.
+
+* PT-130: Fixed ``pt-mext`` to work with non-empty RSA public key. For more information, see 1587404.
+
+* PT-132: Fixed ``pt-online-schema-change`` to enable ``--no-drop-new-table`` when ``--no-swap-tables`` and ``--no-drop-triggers`` are used.

docs/rn.3-0-4.txt

@@ -0,0 +1,81 @@
+
+v3.0.4 released 2017-08-02
+==========================
+
+Percona Toolkit 3.0.4 includes the following changes:
+
+New Features
+
+* :jirabug:`PT-90`: Added collection of information about prepared statements
+  by ``pt-stalk`` when Performance Schema is enabled.
+  For more information, see :lpbug:`1642750`.
+
+* :jirabug:`PT-91`: Added the ``--preserve-triggers`` option
+  for ``pt-online-schema-change`` to support ``AFTER`` triggers.
+
+* :jirabug:`PT-138`: Added ``--output-format`` option
+  for ``pt-mongodb-summary`` to choose between JSON format
+  and the default plain text.
+
+* :jirabug:`PT-141`: Added the ``--output-format=csv`` parameter
+  for ``pt-archiver`` to archive rows in CSV format.
+
+* :jirabug:`PT-142`: Added the ``--only-same-schema-fks`` option
+  for ``pt-online-schema-change`` to check foreigns keys only on tables
+  with the same schema as the original table.
+  This should speed up the tool's execution,
+  but keep in mind that if you have foreign keys
+  referencing tables in other schemas,
+  they won't be detected.
+  For more information, see :lpbug:`1690122`.
+
+* :jirabug:`PT-153`: Added the ``--check-unique-key-change`` option
+  for ``pt-online-schema-change`` to abort
+  if the specified statement for ``--alter`` is trying to add a unique index.
+  This is supposed to avoid adding duplicate keys
+  that might lead to silently losing data.
+
+* :jirabug:`PT-173`: Added the ``--truncate-replicate-table`` option
+  for ``pt-table-checksum`` to ensure stale data is removed.
+
+Bug fixes
+
+* :jirabug:`PT-136`: Fixed ``pt-table-checksum`` to support tables
+  that have columns with different collations or charsets.
+  For more information, see :lpbug:`1674266`.
+
+* :jirabug:`PT-143`: Fixed primary key handling by ``pt-archiver``.
+  For more information, see :lpbug:`1691630`.
+
+* :jirabug:`PT-144`: Limited constraint name in the new table
+  when running ``pt-online-schema-change``.
+  For more information, see :lpbug:`1491674`.
+
+* :jirabug:`PT-146`: Fixed the ``--no-check-binlog-format`` option
+  for ``pt-table-checksum`` to work as expected.
+
+* :jirabug:`PT-148`: Fixed the use of uninitialized value in ``printf()``
+  for ``pt-online-schema-change``.
+  For more information, see :lpbug:`1693614`.
+
+* :jirabug:`PT-151`: Fixed ``pt-table-sync`` to prevent field type ``point``
+  to be taken as decimal.
+
+* :jirabug:`PT-154`: Reverted :jirabug:`PT-116`
+  to remove the ``--use-insert-ignore`` option
+  from ``pt-online-schema-change``.
+
+* :jirabug:`PT-161`: Fixed the ``--skip-check-slave-lag`` feature
+  for ``pt-table-checksum`` to safely check for undefined values.
+
+* :jirabug:`PT-178`: Fixed regression in ``--check-slave-lag`` option
+  for ``pt-online-schema-change``.
+
+* :jirabug:`PT-180`: Fixed regression in ``--skip-check-slave-lag`` option
+  for ``pt-online-schema-change``.
+
+* :jirabug:`PT-181`: Fixed syntax error in ``pt-online-schema-change``.
+
+Other Improvements
+
+* :jirabug:`PT-162`: Updated list of tables ignored by ``pt-table-checksum``.

docs/rn.3-0-5.txt

@@ -0,0 +1,28 @@
+
+v3.0.5 released 2017-11-20
+==========================
+
+Percona Toolkit 3.0.5 includes the following changes:
+
+New Features
+
+ * :jirabug:`PMM-1590`: Improve MongoDB Profiler for ``PMM`` and ``PT``
+ * :jirabug:`PT-216`: The ``mongodb-query-digest`` supports ``MongoDB`` versions lower than 3.2; incorrect output was fixed.
+ * :jirabug:`PT-182`: The ``pt-summary``, ``pt-mysql-summary``,  ``pt-mongodb-summary`` commands provide output in the the JSON format.
+ * :jirabug:`PT-152`: ``pt-mysql-summary`` shows the output of the *Show Slave Hosts* command.
+ * :jirabug:`PT-139`: ``pt-table-sync`` supports replication channels (requires MySQL version 5.7.6 or higher)
+
+Bug fixes
+
+ * :jirabug:`PT-211`: ``pt-mext`` fails if the ``Rsa_public_key`` variable is empty.
+ * :jirabug:`PT-212`: ``pt-mongodb-query-digest --version`` produced incorrect values.
+ * :jirabug:`PT-202`: ``pt-online-schema-change`` incorrectly processed virtual columns.
+ * :jirabug:`PT-200`: ``pt-online-schema-change`` command reported an error when the name of an index contained 'unique' as as the prefix or suffix.
+ * :jirabug:`PT-199`: ``pt-table-checksum`` did not detect differences on a system with the ROW based replication active.
+ * :jirabug:`PT-196`: ``pt-onine-schema-change --max-load`` paused if a status variable was passed **0** as the value.
+ * :jirabug:`PT-193`: ``pt-table-checksum`` reported a misleading error if a column comment contained an apostrophe. For more information, see :lpbug:`1708749`.
+ * :jirabug:`PT-187`: In some cases, ``pt-table-checksum`` did not report that the same table contained different values on the master and slave.
+ * :jirabug:`PT-186`: ``pt-online-schema-change --alter`` could fail if field names contained upper case characters. For more information, see :lpbug:`1705998`.
+ * :jirabug:`PT-183`: In some cases ``pt-mongodb-query-digest`` could not connect to a db using authentication.
+ * :jirabug:`PT-167`: In some cases, ``pt-kill`` could ignore the value of the ``--busy-time`` parameter.  For more information, see :lpbug:`1016272`.
+ * :jirabug:`PT-161`: When run with the ``--skip-check-slave-lag``, the ``pt-table-checksum`` could fail in some cases.

docs/rn.3-0-6.txt

@@ -0,0 +1,25 @@
+
+v3.0.6 released 2018-01-04
+==========================
+
+Percona Toolkit 3.0.6 includes the following changes:
+
+New Features
+
+* :jirabug:`PT-221`: Improve ``pt-table-sync`` support for ``MyRocks``
+* :jirabug:`PT-218`: ``pt-stalk`` now checks the ``RocksDB`` status
+* :jirabug:`PT-214`: ``pt-mysql-summary`` contains the ``RocksDB`` section
+* :jirabug:`PT-205`: ``pt-osc`` shows a message if trying to set the engine to
+  ``rocksdb`` and ``binlog_format != row``.
+* :jirabug:`PT-204`: ``pt-table-checksum`` skips ``RocksDB`` tables.
+
+Known Issues
+
+* :jirabug:`PT-238`: The information message implemented for :jirabug:`PT-204` has a typo when referring to the ``--ignore-engines`` parameter. This problem is planned to be fixed in the next release.
+* :jirabug:`PT-240`: It has been detected that the implementation of :jirabug:`PT-205` is not complete and ``pt-osc`` may fail when altering tables. This problem is planned to be fixed in the next release.
+
+Bug Fixes     
+
+* :jirabug:`PT-234`: The general log parser cannot handle timestamps which include time zones
+* :jirabug:`PT-229`: ``pt-online-schema-change`` does not retry on a deadlock error when using ``Percona Server`` 5.7
+* :jirabug:`PT-225`: ``pt-table-checksum`` ignores generated columns

docs/rn.3-0-7.txt

@@ -0,0 +1,21 @@
+
+v3.0.7 released 2018-03-01
+==========================
+
+New Features
+
+* :jirabug:`PT-633`: Added --mysql-only option to ``pt-stalk`` for RDS
+
+Bug Fixes     
+
+* :jirabug:`PT-244`: The ``--data-dir`` option of ``pt-online-schema-change``  is broken for partitioned table
+* :jirabug:`PT-1256`: ``pt-table-sync`` does not use the character set for the table it is synchronizing
+* :jirabug:`PT-1455`: ``pt-osc`` is stuck when filtering out on the slave the table that is being altered
+* :jirabug:`PT-1485`: The *Security* section of ``pt-mysql-summary`` is broken  in versions bigger than 5.6
+* :jirabug:`PMM-1905`: ``Explain`` fails if it encounters a negative ``ntoreturn``
+
+Known Issues:
+
+* pt-online-schema-change will lock forever if using ``--drop-swap`` under MySQL 8.0.3-rc and 8.0.4-rc due to an error in MySQL: https://bugs.mysql.com/bug.php?id=8948
+* pt-online-schema-change will lose FK constraints under MySQL 8.0.2-dmr, 8.0.3-rc 8.0.4-rc due to an error in MySQL: https://bugs.mysql.com/bug.php?id=89441
+* pt-show-grants can't handle MySQL 8 roles yet

docs/rn.3-0-8.txt

@@ -0,0 +1,12 @@
+
+v3.0.8 released 2018-03-13
+==========================
+
+New Features
+
+* :jirabug:`PT-1500`: Added --output=secure-slowlog option to ``pt-query-digest``
+
+Buf Fixes
+
+* :jirabug:`PT-1503`: The post-install script fails on VM due to improper UUID file detection
+* :jirabug:`PT-1492`: ``pt-kill`` in version 3.0.7 ignores the value of the  ``--busy-time``

docs/rn.3-0-9.txt

@@ -0,0 +1,25 @@
+
+v3.0.9 released 2018-04-20
+==========================
+
+New Tools
+
+* :jirabug:`PT-1501`: ``pt-secure-collect`` - New tool to collect and sanitize pt-tools outputs
+
+New Features 
+
+* :jirabug:`PT-1530`: Add support for encryption status to ``pt-mysql-summary``
+* :jirabug:`PT-1526`: Add ndb status to ``pt-mysql-summary`` (Thanks Fernando Ipar)
+* :jirabug:`PT-1525`: Add support for MySQL 8 roles into ``pt-mysql-summary``
+* :jirabug:`PT-1509`: Make ``pt-table-sync`` only set binlog_format when necessary (Thanks Moritz Lenz)
+* :jirabug:`PT-1508`: Add ``--read-only-interval`` and ``--fail-successive-errors`` flags to ``pt-heartbeat`` (Thanks Shlomi Noach)
+* :jirabug:`PT-243`: Add ``--max-hostname-length`` and ``--max-line-length`` flags to ``pt-query-digest``
+
+Bug Fixes     
+
+* :jirabug:`PT-1527`: Fixed ``pt-table-checksum`` ignores ``--nocheck-binlog-format``
+
+Improvements
+
+* :jirabug:`PT-1507`: ``pt-summary`` does not reliably read in the transparent huge pages setting (Thanks Nick Veenhof)
+* :jirabug:`PT-1488`: ``pt-show-grants`` support for MySQL 8.0

lib/CompareResults.pm

@@ -532,12 +532,12 @@ sub diff_rows {
 
    $left_dbh->do("DROP TABLE IF EXISTS $left_tbl");
    $left_dbh->do("CREATE TABLE $left_tbl $table_ddl");
-   $left_dbh->do("LOAD DATA LOCAL INFILE '$left_outfile' "
+   $left_dbh->do("LOAD DATA INFILE '$left_outfile' "
       . "INTO TABLE $left_tbl");
 
    $right_dbh->do("DROP TABLE IF EXISTS $right_tbl");
    $right_dbh->do("CREATE TABLE $right_tbl $table_ddl");
-   $right_dbh->do("LOAD DATA LOCAL INFILE '$right_outfile' "
+   $right_dbh->do("LOAD DATA INFILE '$right_outfile' "
       . "INTO TABLE $right_tbl");
 
    PTDEBUG && _d('Loaded', $left_outfile, 'into table', $left_tbl, 'and',

lib/NibbleIterator.pm

@@ -495,6 +495,8 @@ sub can_nibble {
    my $one_nibble = !defined $args{one_nibble} || $args{one_nibble}
                   ? $row_est <= $chunk_size * $chunk_size_limit
                   : 0;
+                  #
+
    PTDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no');
 
    # Special case: we're resuming and there's no boundaries, so the table
@@ -512,6 +514,23 @@ sub can_nibble {
       die "There is no good index and the table is oversized.";
    }
 
+   # In some cases, MySQL returns a wrong estimate about number of rows in a table 
+   # and NibbleIterator is trying to process huge tables in a single chunk.
+   # This parameter disable one nibble.
+   # See https://jira.percona.com/browse/PT-1585
+   # This function should work on all scenarios so, to prevent trying to nibble on empty
+   # (or really really small) tables, we need to ensure tha table has an index and it has
+   # at least 2 rows, otherwise there is no way to split only one row into chunks.
+   if ($o->has('force-nibbling') && $o->get('force-nibbling')) {
+       my @nibbling_tables = split(/,/, $o->get('force-nibbling'));
+       my @table_in_list = grep (/$tbl->{tbl}/, @nibbling_tables);
+       if (@table_in_list && $index && $row_est >= 2) {
+           PTDEBUG && _d("Disabling one nibble for the entire table");
+           $one_nibble = 0;
+       }
+   }
+
+
    # The table can be nibbled if this point is reached, else we would have
    # died earlier.  Return some values about nibbling the table.
    my $pause_file = ($o->has('pause-file') && $o->get('pause-file')) || undef;

lib/Percona/Test.pm

@@ -656,14 +656,15 @@ sub test_bash_tool {
 }
 
 my %checksum_result_col = (
-   ts      => 0,
-   errors  => 1,
-   diffs   => 2,
-   rows    => 3,
-   chunks  => 4,
-   skipped => 5,
-   time    => 6,
-   table   => 7,
+   ts        => 0,
+   errors    => 1,
+   diffs     => 2,
+   rows      => 3,
+   diff_rows => 4,
+   chunks    => 5,
+   skipped   => 5,
+   time      => 6,
+   table     => 7,
 );
 sub count_checksum_results {
    my ($output, $column, $table) = @_;
@@ -696,7 +697,7 @@ sub normalize_checksum_results {
    open my $fh, ">", $tmp_file or die "Cannot open $tmp_file: $OS_ERROR";
    printf $fh $output;
    close $fh;
-   my $normal_output = `cat $tmp_file | awk '/^[0-9 ]/ {print \$2 " " \$3 " " \$4 " " \$5 " " \$6 " " \$8} /^[A-Z]/ {print \$0}'`;
+   my $normal_output = `cat $tmp_file | awk '/^[0-9 ]/ {print \$2 " " \$3 " " \$4 " " \$5 " " \$6 " " \$7 " " \$9} /^[A-Z]/ {print \$0}'`;
    `rm $tmp_file >/dev/null`;
    return $normal_output;
 }

lib/Percona/Toolkit.pm

@@ -18,7 +18,7 @@
 # ###########################################################################
 package Percona::Toolkit;
 
-our $VERSION = '3.0.9';
+our $VERSION = '3.0.11-dev';
 
 use strict;
 use warnings FATAL => 'all';

lib/PerconaTest.pm

@@ -692,14 +692,15 @@ sub test_bash_tool {
 }
 
 my %checksum_result_col = (
-   ts      => 0,
-   errors  => 1,
-   diffs   => 2,
-   rows    => 3,
-   chunks  => 4,
-   skipped => 5,
-   time    => 6,
-   table   => 7,
+   ts        => 0,
+   errors    => 1,
+   diffs     => 2,
+   rows      => 3,
+   diff_rows => 4,
+   chunks    => 5,
+   skipped   => 6,
+   time      => 7,
+   table     => 7,
 );
 sub count_checksum_results {
    my ($output, $column, $table) = @_;
@@ -732,7 +733,7 @@ sub normalize_checksum_results {
    open my $fh, ">", $tmp_file or die "Cannot open $tmp_file: $OS_ERROR";
    printf $fh $output;
    close $fh;
-   my $normal_output = `cat $tmp_file | awk '/^[0-9 ]/ {print \$2 " " \$3 " " \$4 " " \$5 " " \$6 " " \$8} /^[A-Z]/ {print \$0}'`;
+   my $normal_output = `cat $tmp_file | awk '/^[0-9 ]/ {print \$2 " " \$3 " " \$4 " " \$5 " " \$6 " " \$7 " " \$9} /^[A-Z]/ {print \$0}'`;
    if ( wantarray ) {
       my $original_output = `cat $tmp_file`;
       return $normal_output, $original_output;

lib/QueryReportFormatter.pm

@@ -1332,7 +1332,8 @@ sub explain_report {
             $explain .= "# *************************** $i. "
                       . "row ***************************\n";
             foreach my $j ( 0 .. $#row ) {
-               $explain .= sprintf "# %13s: %s\n", $sth->{NAME}->[$j],
+               my $value_format = $sth->{NAME}->[$j] eq 'filtered' ? "%.02f" : "%s";
+               $explain .= sprintf "# %13s: $value_format\n", $sth->{NAME}->[$j],
                   defined $row[$j] ? $row[$j] : 'NULL';
             }
             $i++;  # next row number

lib/Sandbox.pm

@@ -211,6 +211,7 @@ sub wipe_clean {
 
    $self->wait_for_slaves();
 
+   $self->clear_genlogs();
    return;
 }
 
@@ -391,7 +392,7 @@ sub verify_test_data {
    my @diffs;
    foreach my $c ( @checksums ) {
       next unless $c->{checksum};
-      if ( $c->{checksum} ne $ref->{$c->{table}}->{checksum} ) {
+      if ( $ref->{$c->{table}} && $c->{checksum} ne $ref->{$c->{table}}->{checksum} ) {
          push @diffs, $c->{table};
       }
    }

lib/TableParser.pm

@@ -220,7 +220,7 @@ sub remove_quoted_text {
    my ($string) = @_;
    $string =~ s/[^\\]`[^`]*[^\\]`//g; 
    $string =~ s/[^\\]"[^"]*[^\\]"//g; 
-   $string =~ s/[^\\]"[^"]*[^\\]"//g; 
+   $string =~ s/[^\\]'[^']*[^\\]'//g; 
    return $string;
 }
 

lib/bash/report_mysql_info.sh

@@ -115,6 +115,12 @@ collect_encrypted_tables() {
     $CMD_MYSQL $EXT_ARGV --table -ss -e "SELECT TABLE_SCHEMA, TABLE_NAME, CREATE_OPTIONS FROM INFORMATION_SCHEMA.TABLES WHERE CREATE_OPTIONS LIKE '%ENCRYPTION=\"Y\"%';"
 }
 
+collect_encrypted_tablespaces() {
+# I_S.INNODB_SYS_TABLESPACES has a "flag" field. Encrypted tablespace has bit 14 set. You can check it with "flag & 8192". 
+# And seems like MySQL is capable of bitwise operations. https://dev.mysql.com/doc/refman/5.7/en/bit-functions.html
+    $CMD_MYSQL $EXT_ARGV --table -ss -e "SELECT SPACE, NAME, SPACE_TYPE from INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES where FLAG&8192 = 8192;"
+}
+
 
 # ##############################################################################
 # Functions for parsing specific files and getting desired info from them.
@@ -597,6 +603,14 @@ format_encrypted_tables() {
    fi
 }
 
+format_encrypted_tablespaces() {
+   local encrypted_tablespaces="$1"
+   if [ ! -z "$encrypted_tablespaces" ]; then
+       echo "Encrypted tablespaces:"
+       echo "$encrypted_tablespaces"
+   fi
+}
+
 format_mysql_roles() {
    local file=$1
    [ -e "$file" ] || return
@@ -1255,27 +1269,20 @@ report_jemalloc_enabled() {
   local GENERAL_JEMALLOC_STATUS=0
   local JEMALLOC_LOCATION=''
 
-  for PID in $(pidof mysqld); do
-     grep -qc jemalloc /proc/${PID}/environ || ldd $(which mysqld) 2>/dev/null | grep -qc jemalloc
-     JEMALLOC_STATUS=$?
-     if [ $JEMALLOC_STATUS = 1 ]; then
-       echo "jemalloc is not enabled in MySQL config for process with ID ${PID}" 
+  for pid in $(pidof mysqld); do
+     grep -qc jemalloc /proc/${pid}/environ || ldd $(which mysqld) 2>/dev/null | grep -qc jemalloc
+     jemalloc_status=$?
+     if [ $jemalloc_status = 1 ]; then
+       echo "jemalloc is not enabled in mysql config for process with id ${pid}" 
      else
-       echo "jemalloc enabled in MySQL config for process with ID ${PID}"
+       echo "jemalloc enabled in mysql config for process with id ${pid}"
        GENERAL_JEMALLOC_STATUS=1
      fi
   done
 
-  if [ $GENERAL_JEMALLOC_STATUS = 1 ]; then
-     # Check location for libjemalloc.so.1
-     #for libjemall in "${SCRIPT_PWD}/lib/mysql" "/usr/lib64" "/usr/lib/x86_64-linux-gnu" "/usr/lib"; do
-     for libjemall in "/usr/lib64" "/usr/lib/x86_64-linux-gnu" "/usr/lib"; do
-       if [ -r "$libjemall/libjemalloc.so.1" ]; then
-         JEMALLOC_LOCATION="$libjemall/libjemalloc.so.1"
-         break
-       fi
-     done
-     if [ -z $JEMALLOC_LOCATION ]; then
+  if [ $GENERAL_JEMALLOC_STATUS -eq 1 ]; then
+     JEMALLOC_LOCATION=$(find /usr/lib64/ /usr/lib/x86_64-linux-gnu /usr/lib -name "libjemalloc.*" 2>/dev/null | head -n 1)
+     if [ -z "$JEMALLOC_LOCATION" ]; then
        echo "Jemalloc library not found"
      else
        echo "Using jemalloc from $JEMALLOC_LOCATION"
@@ -1567,10 +1574,16 @@ report_mysql_summary () {
 
    section "Encryption"
    local keyring_plugins="$(collect_keyring_plugins)"
-   local encrypted_tables="$(collect_encrypted_tables)"
+   local encrypted_tables=""
+   local encrypted_tablespaces=""
+   if [ "${OPT_LIST_ENCRYPTED_TABLES}" = 'yes' ]; then 
+       encrypted_tables="$(collect_encrypted_tables)"
+       encrypted_tablespaces="$(collect_encrypted_tablespaces)"
+   fi
 
    format_keyring_plugins "$keyring_plugins" "$encrypted_tables"
    format_encrypted_tables "$encrypted_tables"
+   format_encrypted_tablespaces "$encrypted_tablespaces"
 
    # ########################################################################
    # Binary Logging

sandbox/prove2junit.pl

@@ -5,21 +5,52 @@ use warnings;
 
 my $file = $ARGV[0];
 my $testcase = "";
+my $testsuite = "";
 my $error_collect = "";
-my $error_print = "";
+my $open_error = 0;
 
 if (not defined $file) {
   die "Need filename as parameter!\n";
 }
 
+sub close_error {
+  if ( $open_error == 1 ) {
+    print "$error_collect ]]></system-err></testcase>\n";
+    $error_collect=""; $open_error=0;
+  }
+
+  return;
+}
+
 open (my $info, $file) or die "Could not open $file: $!";
 
-print "<testsuite name=\"PT MySQL Test\">\n";
+print "<testsuites name=\"PT-MySQL\">\n";
 while(my $line = <$info>) {
-  if ($line =~ /^(t\/\S+).* (\.*) (skipped:) (.*)$/) { print "<testcase name=\"$1\"><skipped/><system-out>Skip reason:<![CDATA[ $4 ]]></system-out></testcase>\n"; }
-  elsif ($line =~ /^ok (\d+) - (.*)$/) { print "<testcase name=\"$testcase - test $1\"><system-out>Test description:<![CDATA[ $2 ]]></system-out></testcase>\n"; }
-  elsif ($line =~ /^not ok (\d+) - (.*)$/) { print "<testcase name=\"$testcase - test $1\"><failure/><system-out>Test description:<![CDATA[ $2 ]]></system-out><system-err><![CDATA[ $error_print ]]></system-err></testcase>\n"; }
-  elsif ($line =~ /^(t\/\S+).* (\.*) $/) { $testcase = "$1"; $error_print = $error_collect; $error_collect = ""; }
-  elsif ($line !~ /^ok$/ && $line !~ /^\d+..\d+$/) { $error_collect = $error_collect . $line; }
+  if ($line =~ /^(t\/)(\S+)(\/)(\S+).* (\.*) (skipped:) (.*)$/) {
+    close_error();
+    print "<testcase name=\"$4\"><skipped/><system-out>Skip reason:<![CDATA[ $7 ]]></system-out></testcase>\n";
+  }
+  elsif ($line =~ /^ok (\d+) - (.*)$/) {
+    close_error();
+    print "<testcase name=\"$testcase - test $1\"><system-out>Test description:<![CDATA[ $2 ]]></system-out></testcase>\n";
+  }
+  elsif ($line =~ /^not ok (\d+) - (.*)$/) {
+    close_error();
+    print "<testcase name=\"$testcase - test $1\"><failure/><system-out>Test description:<![CDATA[ $2 ]]></system-out><system-err><![CDATA[ ";
+    $open_error=1;
+  }
+  elsif ($line =~ /^(t\/)(\S+)(\/)(\S+).* (\.*) $/) {
+    close_error();
+    if ( "$2" eq "$testsuite" ) {
+      $testcase="$4"; $error_collect="";
+    }
+    else {
+      if ( "$testsuite" ne "" ) { print "</testsuite>\n"; }
+      $testsuite="$2"; $testcase="$4"; $error_collect=""; print "<testsuite name=\"$testsuite\">\n";
+    }
+  }
+  elsif ($line !~ /^ok$/ && $line !~ /^\d+..\d+$/) {
+    $error_collect=$error_collect . $line;
+  }
 }
-print "</testsuite>\n"
+print "</testsuite>\n</testsuites>\n";

sandbox/servers/5.5/my.sandbox.cnf

@@ -24,5 +24,4 @@ report-host                = 127.0.0.1
 report-port                = PORT
 log-error                  = /tmp/PORT/data/mysqld.log
 innodb_lock_wait_timeout   = 3
-general_log
-general_log_file           = genlog
+secure-file-priv           = 

sandbox/servers/5.7/my.sandbox.cnf

@@ -26,16 +26,16 @@ log-error                  = /tmp/PORT/data/mysqld.log
 innodb_lock_wait_timeout   = 3
 general_log
 general_log_file           = genlog
-lower_case_table_names     = 0
-slow-query-log             = 0
-slow-query-log-file        = /tmp/PORT/data/slow.log
-log_slow_admin_statements  = 1
-long_query_time            = 0
+#lower_case_table_names     = 0
+#slow-query-log             = 0
+#slow-query-log-file        = /tmp/PORT/data/slow.log
+#log_slow_admin_statements  = 1
+#long_query_time            = 0
 #character-set-server       = utf8
 
 # fkc test
 binlog_format              = STATEMENT
-performance_schema         = ON
+#performance_schema         = ON
 #performance-schema-instrument='wait/lock/metadata/sql/mdl=ON'
 #performance-schema-instrument='transaction=ON'
 secure-file-priv           = 

sandbox/slave_channels_t.sql

@@ -0,0 +1,15 @@
+STOP SLAVE FOR CHANNEL '';
+SET GLOBAL master_info_repository = 'TABLE';
+SET @@GLOBAL.relay_log_info_repository = 'TABLE';
+SET @@GLOBAL.ENFORCE_GTID_CONSISTENCY=ON;
+SET @@GLOBAL.GTID_MODE = OFF_PERMISSIVE;
+SET @@GLOBAL.GTID_MODE = ON_PERMISSIVE;
+SET @@GLOBAL.GTID_MODE = ON;
+
+CHANGE MASTER TO master_host='127.0.0.1', master_port=2900, master_user='msandbox', master_password='msandbox', master_auto_position=1 FOR CHANNEL 'masterchan1';  
+
+CHANGE MASTER TO master_host='127.0.0.1', master_port=2901, master_user='msandbox', master_password='msandbox', master_auto_position=1 FOR CHANNEL 'masterchan2';  
+
+START SLAVE for channel 'masterchan1';
+START SLAVE for channel 'masterchan2';
+

sandbox/start-sandbox

@@ -122,8 +122,12 @@ make_sandbox() {
     fi
 
     if [ -n "$GTID" ]; then
-        #echo "gtid_mode=on" >> ${TMP_DIR}/$port/my.sandbox.cnf
-        echo "gtid_mode=ON_PERMISSIVE" >> ${TMP_DIR}/$port/my.sandbox.cnf
+        if [ "$version" "<" '5.7' ]; then
+            echo "gtid_mode=on" >> ${TMP_DIR}/$port/my.sandbox.cnf
+        fi
+        if [ ! "$version" "<" '5.7' ]; then
+            echo "gtid_mode=ON_PERMISSIVE" >> ${TMP_DIR}/$port/my.sandbox.cnf
+        fi
         echo "enforce_gtid_consistency" >> ${TMP_DIR}/$port/my.sandbox.cnf
     fi
     if [ -n "$REPLICATION_THREADS" ]; then
@@ -143,7 +147,7 @@ make_sandbox() {
 
     if [ $generating_database -eq 1 ]; then
         echo "Creating default databases ..."
-        if [ "$version" > "5.6" ]; then
+        if [ "$version" ">" "5.6" ]; then
             rm -f ${TMP_DIR}/empty-defaults.txt
             touch ${TMP_DIR}/empty-defaults.txt
             rm -rf ${TMP_DIR}/$port/data

src/go/Makefile

@@ -1,6 +1,6 @@
 GO := go
 pkgs   = $(shell find . -type d -name "pt-*" -exec basename {} \;)
-VERSION="3.0.9" 
+VERSION="3.0.11" 
 BUILD=$(shell date +%FT%T%z)
 GOVERSION=$(shell go version | cut --delimiter=" " -f3)
 GOUTILSDIR ?= $(GOPATH)/bin

src/go/pt-secure-collect/sanitize/sanitize.go

@@ -8,7 +8,7 @@ import (
 )
 
 var (
-	hostnameRE    = regexp.MustCompile(`(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)+([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-][A-Za-z0-9]){2,3}(?:\W)`)
+	hostnameRE    = regexp.MustCompile(`(([a-zA-Z0-9]|[a-zA-Z0-9\-]*[a-zA-Z0-9])\.)+([A-Za-z]+)|([0-9]{1,3}\.){3}[0-9]{1,3}`)
 	queryLineRe   []*regexp.Regexp
 	queryInLineRe []*regexp.Regexp
 )

src/go/pt-secure-collect/sanitize/sanitize_test.go

@@ -0,0 +1,54 @@
+package sanitize
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/kr/pretty"
+)
+
+func TestSanitizeHostnames(t *testing.T) {
+
+	want := []string{
+		"top - 20:05:17 up 10 days, 16:27, 1 user, load average: 0.01, 0.15, 0.19",
+		"Tasks: 115 total, 1 running, 114 sleeping, 0 stopped, 0 zombie",
+		"%Cpu(s): 1.0 us, 0.3 sy, 0.0 ni, 98.6 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st",
+		"KiB Mem : 3881748 total, 147324 free, 1892824 used, 1841600 buff/cache",
+		"KiB Swap: 1572860 total, 1572748 free, 112 used. 1609372 avail Mem",
+		"",
+		"PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND",
+		"5304 vagrant 20 0 1983280 1.327g 12272 S 2.3 35.8 456:34.90 prometheus",
+		"5313 root 20 0 142100 16952 5428 S 1.0 0.4 189:16.81 node_exporter		",
+	}
+	lines := make([]string, len(want))
+	copy(lines, want)
+	sanitizeHostnames(lines)
+	if !reflect.DeepEqual(lines, want) {
+		pretty.Println(want)
+		pretty.Println(lines)
+		t.Error("structures don't match")
+	}
+
+	lines = []string{
+		"top - 20:05:17 up 10 days, 16:27, 1 user, load average: 0.01, 0.15, 0.19",
+		"Tasks: 115 total, 1 running, 114 sleeping, 0 stopped, 0 zombie",
+		"%Cpu(s): 1.0 us, 0.3 sy, 0.0 ni, 98.6 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st",
+		"lets put a host name here: domain.com",
+		"and put an ip address here: 10.0.0.1",
+		"and put a non ip address here: 10.0",
+	}
+	want = []string{
+		"top - 20:05:17 up 10 days, 16:27, 1 user, load average: 0.01, 0.15, 0.19",
+		"Tasks: 115 total, 1 running, 114 sleeping, 0 stopped, 0 zombie",
+		"%Cpu(s): 1.0 us, 0.3 sy, 0.0 ni, 98.6 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st",
+		"lets put a host name here: hostname",
+		"and put an ip address here: hostname",
+		"and put a non ip address here: 10.0",
+	}
+	sanitizeHostnames(lines)
+	if !reflect.DeepEqual(lines, want) {
+		t.Error("structures don't match")
+		pretty.Println(want)
+		pretty.Println(lines)
+	}
+}

t/lib/DSNParser.t

@@ -256,9 +256,11 @@ SKIP: {
    is($d->{S}, '/tmp/12345/mysql_sandbox12345.sock', 'Filled in socket');
    is($d->{h}, '127.0.0.1', 'Left hostname alone');
 
+   my $want = $sandbox_version lt '8.0' ? [ qw(utf8 utf8 utf8) ]: [ qw(utf8mb4 utf8mb4 utf8mb4) ];
+   warn Data::Dumper::Dumper($want);
    is_deeply(
       $dbh->selectrow_arrayref('select @@character_set_client, @@character_set_connection, @@character_set_results'),
-      [qw(utf8 utf8 utf8)],
+      $want,
       'Set charset'
    );
    $dbh->disconnect();
@@ -276,7 +278,7 @@ SKIP: {
    };
    is_deeply(
       $dbh->selectrow_arrayref('select @@character_set_client, @@character_set_connection, @@character_set_results'),
-      [qw(utf8 utf8 utf8)],
+      $want,
       'Set utf8 charset case-insensitively (issue 1282)'
    );
 };

t/lib/ExplainAnalyzer.t

@@ -73,20 +73,27 @@ if ( $sandbox_version gt '5.6' ) {
       ];
 }
 
-is_deeply(
-   $exa->explain_query(
+my $got = $exa->explain_query(
       dbh   => $dbh,
       query => 'select * from actor where actor_id = 5',
-   ),
+);
+
+$got->[0]->{filtered} = int($got->[0]->{filtered}) if (defined($got->[0]->{filtered}));
+
+is_deeply(
+   $got,
    $want,
    'Got a simple EXPLAIN result',
 );
 
-is_deeply(
-   $exa->explain_query(
+$got = $exa->explain_query(
       dbh   => $dbh,
       query => 'delete from actor where actor_id = 5',
-   ),
+);
+$got->[0]->{filtered} = int($got->[0]->{filtered}) if (defined($got->[0]->{filtered}));
+
+is_deeply(
+   $got,
    $want,
    'Got EXPLAIN result for a DELETE',
 );

t/lib/MockSyncStream.t

@@ -173,9 +173,9 @@ SKIP: {
             d  => $DBD::mysql::VERSION ge '4.001' ? undef : '(7)',
             dt => undef,
             ts => undef,
-            c  => '(1)',
-            c2 => '(15)',
-            v  => '(32)',
+            c  => $sandbox_version ge '8.0' ? '(4)' : '(3)',
+            c2 => $sandbox_version ge '8.0' ? '(60)' : '(45)',
+            v  => $sandbox_version ge '8.0' ? '(128)' : '(96)',
             t  => undef,
          },
       },

t/lib/QueryReportFormatter.t

@@ -1220,9 +1220,11 @@ SKIP: {
       : $sandbox_version ge '5.1' ? "t/lib/samples/QueryReportFormatter/report025.txt"
       :                             "t/lib/samples/QueryReportFormatter/report026.txt");
 
-   # 30
+   # 32
+   my $explain_report = $qrf->explain_report("select * from qrf.t where i=2", 'qrf');
+   $explain_report =~ s/filtered: 100(\s+)/filtered: 100.00$1/;
    is(
-      $qrf->explain_report("select * from qrf.t where i=2", 'qrf'),
+      $explain_report,
       $explain,
       "explain_report()"
    );

t/lib/Quoter.t

@@ -16,6 +16,7 @@ use Quoter;
 use PerconaTest;
 use DSNParser;
 use Sandbox;
+
 my $dp  = new DSNParser(opts=>$dsn_opts);
 my $sb  = new Sandbox(basedir => '/tmp', DSNParser => $dp);
 my $dbh = $sb->get_dbh_for('master');
@@ -173,6 +174,12 @@ SKIP: {
    $dbh->do('CREATE DATABASE IF NOT EXISTS serialize_test');
    $dbh->do('DROP TABLE IF EXISTS serialize_test.serialize');
    $dbh->do('CREATE TABLE serialize_test.serialize (id INT, textval TEXT, blobval BLOB)');
+   # Ensure we are using lantin1 as the default for the connection
+   # From the documentation:
+   # This statement sets the three session system variables character_set_client, 
+   # character_set_connection, and character_set_results to the given character set. 
+   $dbh->do("SET NAMES 'latin1'"); 
+   warn Data::Dumper::Dumper($dbh);
 
    my $sth = $dbh->prepare(
       "INSERT INTO serialize_test.serialize VALUES (?, ?, ?)"

t/lib/SqlModes.t

@@ -23,8 +23,9 @@ my $dbh = $sb->get_dbh_for('master');
 
 if ( !$dbh ) {
    plan skip_all => "Cannot connect to sandbox master";
-}
-else {
+} elsif ($sandbox_version ge '8.0') {
+   plan skip_all => "There is no NO_AUTO_CREATE_USER in MySQL 8.0+";
+} else {
    plan tests => 4;
 }
 

t/lib/TableChecksum.t

@@ -624,9 +624,10 @@ is_deeply(
    'Type and length of CRC32'
 );
 
+my $want = $sandbox_version ge '8.0' ? [qw(varchar 128)] : [qw(varchar 96)];
 is_deeply(
    [$c->get_crc_type($dbh, 'MD5')],
-   [qw(varchar 32)],
+   $want,
    'Type and length of MD5'
 );
 

t/lib/TableParser.t

@@ -1188,6 +1188,85 @@ SKIP: {
    ) or die Data::Dumper::Dumper($tbl);
 }
 
+# Test that the GENERATED word in a column comment doesn't make that column
+# to be detected as a MySQL 5.7+ generated column.
+$tbl = $tp->parse( load_file('t/lib/samples/generated_cols_comments.sql') );
+is_deeply(
+   $tbl,
+   {
+     charset => 'latin1',
+     clustered_key => 'PRIMARY',
+     col_posn => {
+       id => 0,
+       source => 1,
+       tso_id => 2
+     },
+     cols => [
+       'id',
+       'source',
+       'tso_id'
+     ],
+     defs => {
+       id => '  `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT \'The unique id of the audit record.\'',
+       source => '  `source` enum(\'val1\',\'val2\') NOT NULL COMMENT \'Transaction originator\'',
+       tso_id => '  `tso_id` int(11) unsigned NOT NULL DEFAULT \'0\' COMMENT \'An internally generated transaction.\''
+     },
+     engine => 'InnoDB',
+     is_autoinc => {
+       id => 1,
+       source => 0,
+       tso_id => 0
+     },
+     is_col => {
+       id => 1,
+       source => 1,
+       tso_id => 1
+     },
+     is_generated => {},
+     is_nullable => {},
+     is_numeric => {
+       id => 1,
+       tso_id => 1
+     },
+     keys => {
+       PRIMARY => {
+         col_prefixes => [
+           undef
+         ],
+         colnames => '`id`',
+         cols => [
+           'id'
+         ],
+         ddl => 'PRIMARY KEY (`id`),',
+         is_col => {
+           id => 1
+         },
+         is_nullable => 0,
+         is_unique => 1,
+         name => 'PRIMARY',
+         type => 'BTREE'
+       }
+     },
+     name => 't1',
+     non_generated_cols => [
+       'id',
+       'source',
+       'tso_id'
+     ],
+     null_cols => [],
+     numeric_cols => [
+       'id',
+       'tso_id'
+     ],
+     type_for => {
+       id => 'int',
+       source => 'enum',
+       tso_id => 'int'
+     }
+   },
+   'Column having the word "generated" as part of the comment is OK',
+) or diag Data::Dumper::Dumper($tbl);
+
 # #############################################################################
 # Done.
 # #############################################################################

t/lib/samples/QueryReportFormatter/report031-5.7.txt

@@ -9,5 +9,5 @@
 #       key_len: 4
 #           ref: const
 #          rows: 1
-#      filtered: 100
+#      filtered: 100.00
 #         Extra: NULL

t/lib/samples/SchemaIterator/all-dbs-tbls-8.0.txt

@@ -6,7 +6,7 @@ CREATE TABLE `columns_priv` (
   `Table_name` char(64) COLLATE utf8_bin NOT NULL DEFAULT '',
   `Column_name` char(64) COLLATE utf8_bin NOT NULL DEFAULT '',
   `Timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-  `Column_priv` set('Select','Insert','Update','References') CHARACTER SET utf8 NOT NULL DEFAULT '',
+  `Column_priv` set('Select','Insert','Update','References') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '',
   PRIMARY KEY (`Host`,`Db`,`User`,`Table_name`,`Column_name`)
 ) /*!50100 TABLESPACE `mysql` */ ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin STATS_PERSISTENT=0 COMMENT='Column privileges'
 
@@ -23,25 +23,25 @@ CREATE TABLE `db` (
   `Host` char(60) COLLATE utf8_bin NOT NULL DEFAULT '',
   `Db` char(64) COLLATE utf8_bin NOT NULL DEFAULT '',
   `User` char(32) COLLATE utf8_bin NOT NULL DEFAULT '',
-  `Select_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Insert_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Update_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Delete_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Drop_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Grant_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `References_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Index_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Alter_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_tmp_table_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Lock_tables_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_view_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Show_view_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_routine_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Alter_routine_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Execute_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Event_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Trigger_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
+  `Select_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Insert_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Update_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Delete_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Drop_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Grant_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `References_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Index_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Alter_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_tmp_table_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Lock_tables_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_view_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Show_view_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_routine_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Alter_routine_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Execute_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Event_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Trigger_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
   PRIMARY KEY (`Host`,`Db`,`User`),
   KEY `User` (`User`)
 ) /*!50100 TABLESPACE `mysql` */ ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin STATS_PERSISTENT=0 COMMENT='Database privileges'
@@ -63,7 +63,7 @@ CREATE TABLE `engine_cost` (
   `cost_value` float DEFAULT NULL,
   `last_update` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
   `comment` varchar(1024) DEFAULT NULL,
-  `default_value` float GENERATED ALWAYS AS ((case `cost_name` when _utf8'io_block_read_cost' then 1.0 when _utf8'memory_block_read_cost' then 0.25 else NULL end)) VIRTUAL,
+  `default_value` float GENERATED ALWAYS AS ((case `cost_name` when _utf8mb3'io_block_read_cost' then 1.0 when _utf8mb3'memory_block_read_cost' then 0.25 else NULL end)) VIRTUAL,
   PRIMARY KEY (`cost_name`,`engine_name`,`device_type`)
 ) /*!50100 TABLESPACE `mysql` */ ENGINE=InnoDB DEFAULT CHARSET=utf8 STATS_PERSISTENT=0
 
@@ -72,7 +72,7 @@ CREATE TABLE `func` (
   `name` char(64) COLLATE utf8_bin NOT NULL DEFAULT '',
   `ret` tinyint(1) NOT NULL DEFAULT '0',
   `dl` char(128) COLLATE utf8_bin NOT NULL DEFAULT '',
-  `type` enum('function','aggregate') CHARACTER SET utf8 NOT NULL,
+  `type` enum('function','aggregate') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL,
   PRIMARY KEY (`name`)
 ) /*!50100 TABLESPACE `mysql` */ ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin STATS_PERSISTENT=0 COMMENT='User defined functions'
 
@@ -80,8 +80,8 @@ mysql.global_grants
 CREATE TABLE `global_grants` (
   `USER` char(32) COLLATE utf8_bin NOT NULL DEFAULT '',
   `HOST` char(60) COLLATE utf8_bin NOT NULL DEFAULT '',
-  `PRIV` char(32) CHARACTER SET utf8 NOT NULL DEFAULT '',
-  `WITH_GRANT_OPTION` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
+  `PRIV` char(32) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '',
+  `WITH_GRANT_OPTION` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
   PRIMARY KEY (`USER`,`HOST`,`PRIV`)
 ) /*!50100 TABLESPACE `mysql` */ ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin STATS_PERSISTENT=0 COMMENT='Extended global grants'
 
@@ -122,29 +122,6 @@ CREATE TABLE `help_topic` (
   UNIQUE KEY `name` (`name`)
 ) /*!50100 TABLESPACE `mysql` */ ENGINE=InnoDB DEFAULT CHARSET=utf8 STATS_PERSISTENT=0 COMMENT='help topics'
 
-mysql.innodb_ddl_log
-CREATE TABLE `innodb_ddl_log` (
-  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
-  `thread_id` bigint(20) unsigned NOT NULL,
-  `type` int(10) unsigned NOT NULL,
-  `space_id` int(10) unsigned DEFAULT NULL,
-  `page_no` int(10) unsigned DEFAULT NULL,
-  `index_id` bigint(20) unsigned DEFAULT NULL,
-  `table_id` bigint(20) unsigned DEFAULT NULL,
-  `old_file_path` varchar(512) COLLATE utf8_bin DEFAULT NULL,
-  `new_file_path` varchar(512) COLLATE utf8_bin DEFAULT NULL,
-  PRIMARY KEY (`id`),
-  KEY `thread_id` (`thread_id`)
-) /*!50100 TABLESPACE `mysql` */ ENGINE=InnoDB AUTO_INCREMENT=478 DEFAULT CHARSET=utf8 COLLATE=utf8_bin STATS_PERSISTENT=0 ROW_FORMAT=DYNAMIC
-
-mysql.innodb_dynamic_metadata
-CREATE TABLE `innodb_dynamic_metadata` (
-  `table_id` bigint(20) unsigned NOT NULL,
-  `version` bigint(20) unsigned NOT NULL,
-  `metadata` blob NOT NULL,
-  PRIMARY KEY (`table_id`)
-) /*!50100 TABLESPACE `mysql` */ ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin STATS_PERSISTENT=0 ROW_FORMAT=DYNAMIC
-
 mysql.password_history
 CREATE TABLE `password_history` (
   `Host` char(60) COLLATE utf8_bin NOT NULL DEFAULT '',
@@ -166,10 +143,10 @@ CREATE TABLE `procs_priv` (
   `Host` char(60) COLLATE utf8_bin NOT NULL DEFAULT '',
   `Db` char(64) COLLATE utf8_bin NOT NULL DEFAULT '',
   `User` char(32) COLLATE utf8_bin NOT NULL DEFAULT '',
-  `Routine_name` char(64) CHARACTER SET utf8 NOT NULL DEFAULT '',
+  `Routine_name` char(64) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '',
   `Routine_type` enum('FUNCTION','PROCEDURE') COLLATE utf8_bin NOT NULL,
   `Grantor` char(93) COLLATE utf8_bin NOT NULL DEFAULT '',
-  `Proc_priv` set('Execute','Alter Routine','Grant') CHARACTER SET utf8 NOT NULL DEFAULT '',
+  `Proc_priv` set('Execute','Alter Routine','Grant') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '',
   `Timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
   PRIMARY KEY (`Host`,`Db`,`User`,`Routine_name`,`Routine_type`),
   KEY `Grantor` (`Grantor`)
@@ -194,7 +171,7 @@ CREATE TABLE `role_edges` (
   `FROM_USER` char(32) COLLATE utf8_bin NOT NULL DEFAULT '',
   `TO_HOST` char(60) COLLATE utf8_bin NOT NULL DEFAULT '',
   `TO_USER` char(32) COLLATE utf8_bin NOT NULL DEFAULT '',
-  `WITH_ADMIN_OPTION` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
+  `WITH_ADMIN_OPTION` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
   PRIMARY KEY (`FROM_HOST`,`FROM_USER`,`TO_HOST`,`TO_USER`)
 ) /*!50100 TABLESPACE `mysql` */ ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin STATS_PERSISTENT=0 COMMENT='Role hierarchy and role grants'
 
@@ -204,7 +181,7 @@ CREATE TABLE `server_cost` (
   `cost_value` float DEFAULT NULL,
   `last_update` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
   `comment` varchar(1024) DEFAULT NULL,
-  `default_value` float GENERATED ALWAYS AS ((case `cost_name` when _utf8'disk_temptable_create_cost' then 20.0 when _utf8'disk_temptable_row_cost' then 0.5 when _utf8'key_compare_cost' then 0.05 when _utf8'memory_temptable_create_cost' then 1.0 when _utf8'memory_temptable_row_cost' then 0.1 when _utf8'row_evaluate_cost' then 0.1 else NULL end)) VIRTUAL,
+  `default_value` float GENERATED ALWAYS AS ((case `cost_name` when _utf8mb3'disk_temptable_create_cost' then 20.0 when _utf8mb3'disk_temptable_row_cost' then 0.5 when _utf8mb3'key_compare_cost' then 0.05 when _utf8mb3'memory_temptable_create_cost' then 1.0 when _utf8mb3'memory_temptable_row_cost' then 0.1 when _utf8mb3'row_evaluate_cost' then 0.1 else NULL end)) VIRTUAL,
   PRIMARY KEY (`cost_name`)
 ) /*!50100 TABLESPACE `mysql` */ ENGINE=InnoDB DEFAULT CHARSET=utf8 STATS_PERSISTENT=0
 
@@ -230,8 +207,8 @@ CREATE TABLE `tables_priv` (
   `Table_name` char(64) COLLATE utf8_bin NOT NULL DEFAULT '',
   `Grantor` char(93) COLLATE utf8_bin NOT NULL DEFAULT '',
   `Timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-  `Table_priv` set('Select','Insert','Update','Delete','Create','Drop','Grant','References','Index','Alter','Create View','Show view','Trigger') CHARACTER SET utf8 NOT NULL DEFAULT '',
-  `Column_priv` set('Select','Insert','Update','References') CHARACTER SET utf8 NOT NULL DEFAULT '',
+  `Table_priv` set('Select','Insert','Update','Delete','Create','Drop','Grant','References','Index','Alter','Create View','Show view','Trigger') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '',
+  `Column_priv` set('Select','Insert','Update','References') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '',
   PRIMARY KEY (`Host`,`Db`,`User`,`Table_name`),
   KEY `Grantor` (`Grantor`)
 ) /*!50100 TABLESPACE `mysql` */ ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin STATS_PERSISTENT=0 COMMENT='Table privileges'
@@ -239,7 +216,7 @@ CREATE TABLE `tables_priv` (
 mysql.time_zone
 CREATE TABLE `time_zone` (
   `Time_zone_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
-  `Use_leap_seconds` enum('Y','N') NOT NULL DEFAULT 'N',
+  `Use_leap_seconds` enum('Y','N') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
   PRIMARY KEY (`Time_zone_id`)
 ) /*!50100 TABLESPACE `mysql` */ ENGINE=InnoDB DEFAULT CHARSET=utf8 STATS_PERSISTENT=0 COMMENT='Time zones'
 
@@ -279,36 +256,36 @@ mysql.user
 CREATE TABLE `user` (
   `Host` char(60) COLLATE utf8_bin NOT NULL DEFAULT '',
   `User` char(32) COLLATE utf8_bin NOT NULL DEFAULT '',
-  `Select_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Insert_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Update_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Delete_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Drop_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Reload_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Shutdown_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Process_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `File_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Grant_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `References_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Index_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Alter_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Show_db_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Super_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_tmp_table_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Lock_tables_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Execute_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Repl_slave_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Repl_client_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_view_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Show_view_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_routine_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Alter_routine_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_user_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Event_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Trigger_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_tablespace_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `ssl_type` enum('','ANY','X509','SPECIFIED') CHARACTER SET utf8 NOT NULL DEFAULT '',
+  `Select_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Insert_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Update_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Delete_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Drop_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Reload_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Shutdown_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Process_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `File_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Grant_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `References_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Index_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Alter_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Show_db_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Super_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_tmp_table_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Lock_tables_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Execute_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Repl_slave_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Repl_client_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_view_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Show_view_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_routine_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Alter_routine_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_user_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Event_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Trigger_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_tablespace_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `ssl_type` enum('','ANY','X509','SPECIFIED') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '',
   `ssl_cipher` blob NOT NULL,
   `x509_issuer` blob NOT NULL,
   `x509_subject` blob NOT NULL,
@@ -318,12 +295,12 @@ CREATE TABLE `user` (
   `max_user_connections` int(11) unsigned NOT NULL DEFAULT '0',
   `plugin` char(64) COLLATE utf8_bin NOT NULL DEFAULT 'caching_sha2_password',
   `authentication_string` text COLLATE utf8_bin,
-  `password_expired` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
+  `password_expired` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
   `password_last_changed` timestamp NULL DEFAULT NULL,
   `password_lifetime` smallint(5) unsigned DEFAULT NULL,
-  `account_locked` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_role_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Drop_role_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
+  `account_locked` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_role_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Drop_role_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
   `Password_reuse_history` smallint(5) unsigned DEFAULT NULL,
   `Password_reuse_time` smallint(5) unsigned DEFAULT NULL,
   PRIMARY KEY (`Host`,`User`)
@@ -334,19 +311,19 @@ CREATE TABLE `checksums` (
   `db_tbl` varchar(128) NOT NULL,
   `checksum` int(10) unsigned NOT NULL,
   PRIMARY KEY (`db_tbl`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
 
 percona_test.load_data
 CREATE TABLE `load_data` (
   `i` int(11) DEFAULT NULL
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
 
 percona_test.sentinel
 CREATE TABLE `sentinel` (
   `id` int(11) NOT NULL,
   `ping` varchar(64) NOT NULL DEFAULT '',
   PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
 
 sakila.actor
 CREATE TABLE `actor` (
@@ -571,5 +548,5 @@ CREATE TABLE `sys_config` (
   `set_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
   `set_by` varchar(128) DEFAULT NULL,
   PRIMARY KEY (`variable`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
 

t/lib/samples/SchemaIterator/mysql-user-ddl-8.0.txt

@@ -2,36 +2,36 @@ mysql.user
 CREATE TABLE `user` (
   `Host` char(60) COLLATE utf8_bin NOT NULL DEFAULT '',
   `User` char(32) COLLATE utf8_bin NOT NULL DEFAULT '',
-  `Select_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Insert_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Update_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Delete_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Drop_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Reload_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Shutdown_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Process_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `File_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Grant_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `References_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Index_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Alter_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Show_db_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Super_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_tmp_table_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Lock_tables_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Execute_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Repl_slave_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Repl_client_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_view_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Show_view_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_routine_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Alter_routine_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_user_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Event_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Trigger_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_tablespace_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `ssl_type` enum('','ANY','X509','SPECIFIED') CHARACTER SET utf8 NOT NULL DEFAULT '',
+  `Select_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Insert_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Update_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Delete_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Drop_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Reload_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Shutdown_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Process_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `File_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Grant_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `References_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Index_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Alter_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Show_db_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Super_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_tmp_table_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Lock_tables_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Execute_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Repl_slave_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Repl_client_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_view_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Show_view_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_routine_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Alter_routine_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_user_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Event_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Trigger_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_tablespace_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `ssl_type` enum('','ANY','X509','SPECIFIED') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '',
   `ssl_cipher` blob NOT NULL,
   `x509_issuer` blob NOT NULL,
   `x509_subject` blob NOT NULL,
@@ -41,12 +41,12 @@ CREATE TABLE `user` (
   `max_user_connections` int(11) unsigned NOT NULL DEFAULT '0',
   `plugin` char(64) COLLATE utf8_bin NOT NULL DEFAULT 'caching_sha2_password',
   `authentication_string` text COLLATE utf8_bin,
-  `password_expired` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
+  `password_expired` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
   `password_last_changed` timestamp NULL DEFAULT NULL,
   `password_lifetime` smallint(5) unsigned DEFAULT NULL,
-  `account_locked` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Create_role_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
-  `Drop_role_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
+  `account_locked` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Create_role_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
+  `Drop_role_priv` enum('N','Y') CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT 'N',
   `Password_reuse_history` smallint(5) unsigned DEFAULT NULL,
   `Password_reuse_time` smallint(5) unsigned DEFAULT NULL,
   PRIMARY KEY (`Host`,`User`)

t/lib/samples/generated_cols_comments.sql

@@ -0,0 +1,6 @@
+CREATE TABLE `t1` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT 'The unique id of the audit record.',
+  `source` enum('val1','val2') NOT NULL COMMENT 'Transaction originator',
+  `tso_id` int(11) unsigned NOT NULL DEFAULT '0' COMMENT 'An internally generated transaction.',
+  PRIMARY KEY (`id`),
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 COMMENT='some comment here generated'

t/pt-archiver/file.t

@@ -46,7 +46,6 @@ is($output, <<EOF
 2\t\\N\t3\t4
 3\t2\t3\t\\\t
 4\t2\t3\t\\
-
 EOF
 , 'File has the right stuff');
 `rm -f archive.test.table_1`;
@@ -124,8 +123,8 @@ $output = output(
 );
 $output = `cat archive.test.table_2`;
 is($output, <<EOF
-1, 2, 3, "4"
-2, "\\N", 3, "4"
+1, 2, 3, 4
+2, "\\N", 3, 4
 3, 2, 3, "\\\t"
 4, 2, 3, "\\\n"
 5, 2, 3, "Zapp \\"Brannigan"

t/pt-archiver/issue_1225.t

@@ -10,6 +10,7 @@ use strict;
 use warnings FATAL => 'all';
 use English qw(-no_match_vars);
 use Test::More;
+use utf8;
 
 use PerconaTest;
 use Sandbox;

t/pt-archiver/issue_1229.t

@@ -14,6 +14,8 @@ use Data::Dumper;
 
 use PerconaTest;
 use Sandbox;
+use utf8;
+
 require "$trunk/bin/pt-archiver";
 
 my $dp  = new DSNParser(opts=>$dsn_opts);

t/pt-duplicate-key-checker/basics.t

@@ -140,10 +140,11 @@ ok(
 
 $sb->load_file('master', 't/lib/samples/dupekeys/simple_dupe_bug_1217013.sql', 'test');
 
+my $want = $sandbox_version lt '8.0' ? "$sample/simple_dupe_bug_1217013.txt" : "$sample/simple_dupe_bug_1217013_80.txt";
 ok(
    no_diff(
       sub { pt_duplicate_key_checker::main(@args, qw(-t test.domains -v)) },
-      "$sample/simple_dupe_bug_1217013.txt"),
+      $want),
    'Exact unique dupes (bug 1217013)'
 ) or diag($test_diff);
 
@@ -152,10 +153,11 @@ ok(
 # https://bugs.launchpad.net/percona-toolkit/+bug/1402730
 # #############################################################################
 
+$want = $sandbox_version lt '8.0' ? "$sample/simple_dupe_bug_1402730.txt" : "$sample/simple_dupe_bug_1402730_80.txt";
 ok(
    no_diff(
       sub { pt_duplicate_key_checker::main(@args, qw(-t test.domains --verbose)) },
-      "$sample/simple_dupe_bug_1217013.txt", keep_output=>1),
+      "$want", keep_output=>1),
    q[--verbose option doesn't skip dupes reporting (bug 1402730)]
 ) or diag($test_diff);
 

t/pt-duplicate-key-checker/samples/simple_dupe_bug_1217013_80.txt

@@ -0,0 +1,25 @@
+# ########################################################################
+# test.domains                                                            
+# ########################################################################
+
+# PRIMARY (`id`)
+# domain (`domain`)
+# unique_key_domain (`domain`)
+
+# Uniqueness of domain ignored because unique_key_domain is a duplicate constraint
+# domain is a duplicate of unique_key_domain
+# Key definitions:
+#   UNIQUE KEY `domain` (`domain`),
+#   UNIQUE KEY `unique_key_domain` (`domain`)
+# Column types:
+#	  `domain` varchar(175) character set utf8 collate utf8_bin not null
+# To remove this duplicate index, execute:
+ALTER TABLE `test`.`domains` DROP INDEX `domain`;
+
+# ########################################################################
+# Summary of indexes                                                      
+# ########################################################################
+
+# Size Duplicate Indexes   527
+# Total Duplicate Indexes  1
+# Total Indexes            3

t/pt-duplicate-key-checker/samples/simple_dupe_bug_1402730.txt

@@ -0,0 +1,25 @@
+# ########################################################################
+# test.domains                                                            
+# ########################################################################
+
+# PRIMARY (`id`)
+# domain (`domain`)
+# unique_key_domain (`domain`)
+
+# Uniqueness of domain ignored because unique_key_domain is a duplicate constraint
+# domain is a duplicate of unique_key_domain
+# Key definitions:
+#   UNIQUE KEY `domain` (`domain`),
+#   UNIQUE KEY `unique_key_domain` (`domain`)
+# Column types:
+#	  `domain` varchar(175) collate utf8_bin not null
+# To remove this duplicate index, execute:
+ALTER TABLE `test`.`domains` DROP INDEX `domain`;
+
+# ########################################################################
+# Summary of indexes                                                      
+# ########################################################################
+
+# Size Duplicate Indexes   527
+# Total Duplicate Indexes  1
+# Total Indexes            3

t/pt-duplicate-key-checker/samples/simple_dupe_bug_1402730_80.txt

@@ -0,0 +1,25 @@
+# ########################################################################
+# test.domains                                                            
+# ########################################################################
+
+# PRIMARY (`id`)
+# domain (`domain`)
+# unique_key_domain (`domain`)
+
+# Uniqueness of domain ignored because unique_key_domain is a duplicate constraint
+# domain is a duplicate of unique_key_domain
+# Key definitions:
+#   UNIQUE KEY `domain` (`domain`),
+#   UNIQUE KEY `unique_key_domain` (`domain`)
+# Column types:
+#	  `domain` varchar(175) character set utf8 collate utf8_bin not null
+# To remove this duplicate index, execute:
+ALTER TABLE `test`.`domains` DROP INDEX `domain`;
+
+# ########################################################################
+# Summary of indexes                                                      
+# ########################################################################
+
+# Size Duplicate Indexes   527
+# Total Duplicate Indexes  1
+# Total Indexes            3

t/pt-find/pt-find.t

@@ -191,13 +191,13 @@ SKIP: {
       is(
          $output,
    "`sakila`.`actor_info`
-   `sakila`.`customer_list`
-   `sakila`.`film_list`
-   `sakila`.`nicer_but_slower_film_list`
-   `sakila`.`sales_by_film_category`
-   `sakila`.`sales_by_store`
-   `sakila`.`staff_list`
-   ",
+`sakila`.`customer_list`
+`sakila`.`film_list`
+`sakila`.`nicer_but_slower_film_list`
+`sakila`.`sales_by_film_category`
+`sakila`.`sales_by_store`
+`sakila`.`staff_list`
+",
          '--datasize NULL',
       );
    };

t/pt-heartbeat/basics.t

@@ -260,7 +260,12 @@ is(
 # --check-read-only
 # #############################################################################
 
-diag(`/tmp/12345/use -u root -e "GRANT ALL ON *.* TO 'bob'\@'%' IDENTIFIED BY 'msandbox'"`);
+if ($sandbox_version ge '8.0') {
+    diag(`/tmp/12345/use -u root -e "CREATE USER 'bob'\@'%' IDENTIFIED WITH mysql_native_password BY 'msandbox'"`);
+} else {
+    diag(`/tmp/12345/use -u root -e "CREATE USER 'bob'\@'%' IDENTIFIED BY 'msandbox'"`);
+}
+diag(`/tmp/12345/use -u root -e "GRANT ALL ON *.* TO 'bob'\@'%'"`);
 diag(`/tmp/12345/use -u root -e "REVOKE SUPER ON *.* FROM 'bob'\@'%'"`);
 if ($sandbox_version ge '8.0') {
     diag(`/tmp/12345/use -u root -e "REVOKE CONNECTION_ADMIN ON *.* FROM 'bob'\@'%'"`);

t/pt-heartbeat/pt-1508.t

@@ -65,7 +65,12 @@ my $create_table_sql = <<__EOQ;
 __EOQ
 
 $sb->do_as_root('master', "$create_table_sql");
-$sb->do_as_root('slave1', 'GRANT SELECT, INSERT, UPDATE, REPLICATION CLIENT ON *.* TO "unprivileged"@"localhost" IDENTIFIED BY "password"');
+if ($sandbox_version ge '8.0') {
+    $sb->do_as_root('slave1', 'CREATE USER "unprivileged"@"localhost" IDENTIFIED WITH mysql_native_password BY "password"');
+} else {
+    $sb->do_as_root('slave1', 'CREATE USER "unprivileged"@"localhost" IDENTIFIED BY "password"');
+}
+$sb->do_as_root('slave1', 'GRANT SELECT, INSERT, UPDATE, REPLICATION CLIENT ON *.* TO "unprivileged"@"localhost"');
 $sb->do_as_root('slave1', "FLUSH TABLES WITH READ LOCK;");
 $sb->do_as_root('slave1', "SET GLOBAL read_only = 1;");
 

t/pt-mext/samples/mext-001-result.txt

@@ -47,6 +47,7 @@ Com_lock_tables                              37    0    0    0
 Com_optimize                                  1    0    0    0
 Com_preload_keys                              0    0    0    0
 Com_prepare_sql                               0    0    0    0
+Compression                                   0    0    0    0
 Com_purge                                     0    0    0    0
 Com_purge_before_date                         0    0    0    0
 Com_rename_table                              0    0    0    0
@@ -107,7 +108,6 @@ Com_xa_prepare                                0    0    0    0
 Com_xa_recover                                0    0    0    0
 Com_xa_rollback                               0    0    0    0
 Com_xa_start                                  0    0    0    0
-Compression                                   0    0    0    0
 Connections                                1008    0    0    0
 Created_tmp_disk_tables                     350    0    0    0
 Created_tmp_files                             6    0    0    0
@@ -160,8 +160,8 @@ Innodb_os_log_fsyncs                       2505    0    0    0
 Innodb_os_log_pending_fsyncs                  0    0    0    0
 Innodb_os_log_pending_writes                  0    0    0    0
 Innodb_os_log_written                  11063296    0    0    0
-Innodb_page_size                          16384    0    0    0
 Innodb_pages_created                        542    0    0    0
+Innodb_page_size                          16384    0    0    0
 Innodb_pages_read                             0    0    0    0
 Innodb_pages_written                       1071    0    0    0
 Innodb_row_lock_current_waits                 0    0    0    0
@@ -187,10 +187,10 @@ Ndb_config_from_host                          0    0    0    0
 Ndb_config_from_port                          0    0    0    0
 Ndb_number_of_data_nodes                      0    0    0    0
 Not_flushed_delayed_rows                      0    0    0    0
+Opened_tables                              1698    0    0    0
 Open_files                                    5    0    0    0
 Open_streams                                  0    0    0    0
 Open_tables                                   1    0    0    0
-Opened_tables                              1698    0    0    0
 Prepared_stmt_count                           0    0    0    0
 Qcache_free_blocks                            0    0    0    0
 Qcache_free_memory                            0    0    0    0

t/pt-mext/samples/mext-002-result.txt

@@ -47,6 +47,7 @@ Com_lock_tables                              37        0        0        0
 Com_optimize                                  1        0        0        0
 Com_preload_keys                              0        0        0        0
 Com_prepare_sql                               0        0        0        0
+Compression                                   0        0        0        0
 Com_purge                                     0        0        0        0
 Com_purge_before_date                         0        0        0        0
 Com_rename_table                              0        0        0        0
@@ -107,7 +108,6 @@ Com_xa_prepare                                0        0        0        0
 Com_xa_recover                                0        0        0        0
 Com_xa_rollback                               0        0        0        0
 Com_xa_start                                  0        0        0        0
-Compression                                   0        0        0        0
 Connections                                1009        0        0        0
 Created_tmp_disk_tables                     350        0        0        0
 Created_tmp_files                             6        0        0        0
@@ -160,8 +160,8 @@ Innodb_os_log_fsyncs                       2505        0        0        0
 Innodb_os_log_pending_fsyncs                  0        0        0        0
 Innodb_os_log_pending_writes                  0        0        0        0
 Innodb_os_log_written                  11063296        0        0        0
-Innodb_page_size                          16384        0        0        0
 Innodb_pages_created                        542        0        0        0
+Innodb_page_size                          16384        0        0        0
 Innodb_pages_read                             0        0        0        0
 Innodb_pages_written                       1071        0        0        0
 Innodb_row_lock_current_waits                 0        0        0        0
@@ -187,10 +187,10 @@ Ndb_config_from_host                          0        0        0        0
 Ndb_config_from_port                          0        0        0        0
 Ndb_number_of_data_nodes                      0        0        0        0
 Not_flushed_delayed_rows                      0        0        0        0
+Opened_tables                              1698        0        0        0
 Open_files                                    5        0        0        0
 Open_streams                                  0        0        0        0
 Open_tables                                   1        0        0        0
-Opened_tables                              1698        0        0        0
 Prepared_stmt_count                           0        0        0        0
 Qcache_free_blocks                            0        0        0        0
 Qcache_free_memory                            0        0        0        0

t/pt-mysql-summary/pt-mysql-summary.t

@@ -9,16 +9,37 @@ BEGIN {
 use strict;
 use warnings FATAL => 'all';
 use English qw(-no_match_vars);
-
 use PerconaTest;
-
-my ($tool) = $PROGRAM_NAME =~ m/([\w-]+)\.t$/;
-
+use Sandbox;
+use DSNParser;
+require VersionParser;
 use Test::More;
 use File::Temp qw( tempdir );
 
 local $ENV{PTDEBUG} = "";
 
+my $dp         = new DSNParser(opts=>$dsn_opts);
+my $sb         = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $master_dbh = $sb->get_dbh_for('master');
+my $has_keyring_plugin;
+
+my $db_flavor = VersionParser->new($master_dbh)->flavor();
+if ( $db_flavor =~ m/Percona Server/ ) {
+    my $rows = $master_dbh->selectall_hashref("SHOW PLUGINS", "name");
+    while (my ($key, $values) = each %$rows) {
+        if ($key =~ m/^keyring_/) {
+            $has_keyring_plugin=1;
+            last;
+        }
+    }
+}
+
+if ($has_keyring_plugin) {
+    plan skip_all => 'Keyring plugins are enabled.';
+}
+
+my ($tool) = $PROGRAM_NAME =~ m/([\w-]+)\.t$/;
+
 # mysqldump from earlier versions doesn't seem to work with 5.6,
 # so use the actual mysqldump from each MySQL bin which should
 # always be compatible with itself.

t/pt-mysql-summary/pt-mysql-summary_encryption.t

@@ -0,0 +1,161 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use PerconaTest;
+use Sandbox;
+use DSNParser;
+require VersionParser;
+use Test::More;
+use File::Temp qw( tempdir );
+
+local $ENV{PTDEBUG} = "";
+
+my $dp         = new DSNParser(opts=>$dsn_opts);
+my $sb         = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $master_dbh = $sb->get_dbh_for('master');
+my $has_keyring_plugin;
+
+my $db_flavor = VersionParser->new($master_dbh)->flavor();
+if ( $db_flavor =~ m/Percona Server/ ) {
+    my $rows = $master_dbh->selectall_hashref("SHOW PLUGINS", "name");
+    while (my ($key, $values) = each %$rows) {
+        if ($key =~ m/^keyring_/) {
+            $has_keyring_plugin=1;
+            last;
+        }
+    }
+}
+
+if (!$has_keyring_plugin) {
+    plan skip_all => 'Keyring plugins are not enabled.';
+} elsif ( $sandbox_version lt '5.7' || $db_flavor !~ m/Percona Server/) {
+    plan skip_all => 'These tests need Percona Server 5.7+';
+}
+
+my ($tool) = $PROGRAM_NAME =~ m/([\w-]+)_encryption\.t$/;
+
+# mysqldump from earlier versions doesn't seem to work with 5.6,
+# so use the actual mysqldump from each MySQL bin which should
+# always be compatible with itself.
+my $env = qq\CMD_MYSQLDUMP="$ENV{PERCONA_TOOLKIT_SANDBOX}/bin/mysqldump"\;
+
+#
+# --save-samples
+#
+
+my $dir = tempdir( "percona-testXXXXXXXX", CLEANUP => 1 );
+
+`$env $trunk/bin/$tool --sleep 1 --save-samples $dir -- --defaults-file=/tmp/12345/my.sandbox.cnf`;
+
+ok(
+   -e $dir,
+   "Using --save-samples doesn't mistakenly delete the target dir"
+);
+
+# If the box has a default my.cnf (e.g. /etc/my.cnf) there
+# should be 15 files, else 14.
+my @files = glob("$dir/*");
+my $n_files = scalar @files;
+ok(
+   $n_files >= 15 && $n_files <= 18,
+   "And leaves all files in there"
+) or diag($n_files, `ls -l $dir`);
+
+undef($dir);  # rm the dir because CLEANUP => 1
+
+#
+# --databases
+#
+
+my $out = `$env $trunk/bin/$tool --sleep 1 --databases mysql 2>/dev/null -- --defaults-file=/tmp/12345/my.sandbox.cnf`;
+
+like(
+   $out,
+   qr/Database Tables Views SPs Trigs Funcs   FKs Partn\s+\Qmysql\E/,
+   "--databases works"
+);
+
+like(
+   $out,
+   qr/# InnoDB #.*Version.*# MyISAM #/s,
+   "InnoDB section present"
+);
+
+like(
+   $out,
+   qr/Users \| 2/,
+   "Security works"
+);
+
+# --read-samples
+for my $i (2..7) {
+   ok(
+      no_diff(
+         sub {
+            local $ENV{_NO_FALSE_NEGATIVES} = 1;
+            print `$env $trunk/bin/$tool --read-samples $trunk/t/pt-mysql-summary/samples/temp00$i  -- --defaults-file=/tmp/12345/my.sandbox.cnf | tail -n+3 | perl -wlnpe 's/Skipping schema analysis.*/Specify --databases or --all-databases to dump and summarize schemas/' | grep -v jemalloc`
+         },
+         "t/pt-mysql-summary/samples/expected_output_temp_enc00$i.txt",
+      ),
+      "--read-samples works for t/pt-mysql-summary/temp_enc00$i",
+   ) or diag($test_diff);
+}
+
+# Test that --help works under sh
+
+my $sh   = `sh   $trunk/bin/$tool --help`;
+my $bash = `bash $trunk/bin/$tool --help`;
+
+is(
+   $sh,
+   $bash,
+   "--help works under sh and bash"
+);
+
+$master_dbh->do("DROP DATABASE IF EXISTS test");
+$master_dbh->do("CREATE DATABASE test");
+$master_dbh->do("CREATE TABLE test.t1(a INT PRIMARY KEY) ENCRYPTION='Y'");
+$master_dbh->do("CREATE TABLESPACE foo ADD DATAFILE 'foo.ibd' ENCRYPTION='Y'");
+$master_dbh->do("ALTER TABLE test.t1 TABLESPACE=foo");
+$master_dbh->do("CREATE TABLE test.t2(a INT PRIMARY KEY) ENCRYPTION='Y'");
+
+$out = `bash $trunk/bin/$tool --list-encrypted-tables`;
+
+like(
+   $out,
+   qr/Encryption/,
+   "Encryption section included in report"
+) or diag $out;
+
+like(
+   $out,
+   qr/Keyring plugins/,
+   "Keyring plugins included in report"
+) or diag $out;
+
+like(
+   $out,
+   qr/Encrypted tables/,
+   "Encrypted tables included in report"
+) or diag $out;
+
+like(
+   $out,
+   qr/Encrypted tablespaces/,
+   "Encrypted tablespaces included in report"
+) or diag $out;
+
+$master_dbh->do("DROP TABLE IF EXISTS test.t1");
+$master_dbh->do("DROP TABLE IF EXISTS test.t2");
+$master_dbh->do("DROP DATABASE IF EXISTS test");
+$master_dbh->do("DROP TABLESPACE foo");
+
+done_testing;

t/pt-mysql-summary/samples/expected_output_temp002.txt

@@ -222,6 +222,8 @@ Uptime                                    90000           1           1
 # Security ###################################################
                     Users | 6 users, 0 anon, 1 w/o pw, 1 old pw
             Old Passwords | OFF
+# Encryption #################################################
+No keyring plugins found
 # Binary Logging #############################################
 # Noteworthy Variables #######################################
      Auto-Inc Incr/Offset | 1/1

t/pt-mysql-summary/samples/expected_output_temp003.txt

@@ -156,6 +156,8 @@ Specify --databases or --all-databases to dump and summarize schemas
 # Security ###################################################
                     Users | 2 users, 0 anon, 0 w/o pw, 0 old pw
             Old Passwords | OFF
+# Encryption #################################################
+No keyring plugins found
 # Binary Logging #############################################
                   Binlogs | 1
                Zero-Sized | 0

t/pt-mysql-summary/samples/expected_output_temp004.txt

@@ -155,6 +155,8 @@ Specify --databases or --all-databases to dump and summarize schemas
 # Security ###################################################
                     Users | 2 users, 0 anon, 0 w/o pw, 0 old pw
             Old Passwords | OFF
+# Encryption #################################################
+No keyring plugins found
 # Binary Logging #############################################
                   Binlogs | 2
                Zero-Sized | 0

t/pt-mysql-summary/samples/expected_output_temp005.txt

@@ -228,6 +228,8 @@ Uptime                                    90000           1           1
 # Security ###################################################
                     Users | 2 users, 0 anon, 0 w/o pw, 0 old pw
             Old Passwords | OFF
+# Encryption #################################################
+No keyring plugins found
 # Binary Logging #############################################
                   Binlogs | 1
                Zero-Sized | 0

t/pt-mysql-summary/samples/expected_output_temp006.txt

@@ -304,6 +304,8 @@ wsrep_cluster_size                          100
 # Security ###################################################
                     Users | 2 users, 0 anon, 0 w/o pw, 0 old pw
             Old Passwords | OFF
+# Encryption #################################################
+No keyring plugins found
 # Binary Logging #############################################
                   Binlogs | 1
                Zero-Sized | 0